* [PATCH v2 1/2] fetch2: Add gomod fetcher
@ 2024-09-06 9:27 Christian Lindeberg
2024-09-06 9:27 ` [PATCH v2 2/2] fetch2: Add gomodgit fetcher Christian Lindeberg
0 siblings, 1 reply; 4+ messages in thread
From: Christian Lindeberg @ 2024-09-06 9:27 UTC (permalink / raw)
To: bitbake-devel
From: Christian Lindeberg <christian.lindeberg@axis.com>
Add a go module fetcher for downloading module dependencies to the
module cache from a module proxy. The fetcher can be used with the
go-mod class in OE-Core.
A module dependency can be specified with:
SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com>
---
Changes in V2:
- Use GO_MOD_CACHE_DIR variable or default instead of requiring the
GOMODCACHE environment variable to be set.
- Add more documentation to make it cleared what is downloaded and what
is unpacked.
lib/bb/fetch2/__init__.py | 4 +-
lib/bb/fetch2/gomod.py | 128 ++++++++++++++++++++++++++++++++++++++
lib/bb/tests/fetch.py | 65 +++++++++++++++++++
3 files changed, 196 insertions(+), 1 deletion(-)
create mode 100644 lib/bb/fetch2/gomod.py
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 5bf2c4b8c..f84ce5999 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1317,7 +1317,7 @@ class FetchData(object):
if checksum_name in self.parm:
checksum_expected = self.parm[checksum_name]
- elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
+ elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod"]:
checksum_expected = None
else:
checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
@@ -2088,6 +2088,7 @@ from . import npmsw
from . import az
from . import crate
from . import gcp
+from . import gomod
methods.append(local.Local())
methods.append(wget.Wget())
@@ -2110,3 +2111,4 @@ methods.append(npmsw.NpmShrinkWrap())
methods.append(az.Az())
methods.append(crate.Crate())
methods.append(gcp.GCP())
+methods.append(gomod.GoMod())
diff --git a/lib/bb/fetch2/gomod.py b/lib/bb/fetch2/gomod.py
new file mode 100644
index 000000000..fe025e367
--- /dev/null
+++ b/lib/bb/fetch2/gomod.py
@@ -0,0 +1,128 @@
+"""
+BitBake 'Fetch' implementation for Go modules
+
+The gomod fetcher is used to download Go modules to the module cache from a
+module proxy.
+
+Example SRC_URI:
+
+SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
+
+Required SRC_URI parameters:
+
+- version
+ The version of the module.
+
+Optional SRC_URI parameters:
+
+- mod
+ Fetch and unpack the go.mod file only instead of the complete module.
+ The go command may need to download go.mod files for many different modules
+ when computing the build list, and go.mod files are much smaller than
+ module zip files.
+ The default is "0", set mod=1 for the go.mod file only.
+
+- sha256sum
+ The checksum of the module zip file, or the go.mod file in case of fetching
+ only the go.mod file. Alternatively, set the SRC_URI varible flag for
+ "module@version.sha256sum".
+
+Related variables:
+
+- GO_MOD_PROXY
+ The module proxy used by the fetcher.
+
+- GO_MOD_CACHE_DIR
+ The directory where the module cache is located.
+ This must match the exported GOMODCACHE variable for the go command to find
+ the downloaded modules.
+
+See the Go modules reference, https://go.dev/ref/mod, for more information
+about the module cache, module proxies and version control systems.
+"""
+
+import os
+import re
+import shutil
+import zipfile
+
+import bb
+from bb.fetch2 import FetchError
+from bb.fetch2 import MissingParameterError
+from bb.fetch2.wget import Wget
+
+
+def escape(path):
+ """Escape capital letters using exclamation points."""
+ return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
+
+
+class GoMod(Wget):
+ """Class to fetch Go modules from a Go module proxy via wget"""
+
+ def supports(self, ud, d):
+ """Check to see if a given URL is for this fetcher."""
+ return ud.type == 'gomod'
+
+ def urldata_init(self, ud, d):
+ """Set up to download the module from the module proxy.
+
+ Set up to download the module zip file to the module cache directory
+ and unpack the go.mod file (unless downloading only the go.mod file):
+
+ cache/download/<module>/@v/<version>.zip: The module zip file.
+ cache/download/<module>/@v/<version>.mod: The go.mod file.
+ """
+
+ proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
+ moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+
+ if 'version' not in ud.parm:
+ raise MissingParameterError('version', ud.url)
+
+ module = ud.host + ud.path
+ ud.parm['module'] = module
+
+ # Set URL and filename for wget download
+ path = escape(module + '/@v/' + ud.parm['version'])
+ if ud.parm.get('mod', '0') == '1':
+ path += '.mod'
+ else:
+ path += '.zip'
+ ud.parm['unpack'] = '0'
+ ud.url = bb.fetch2.encodeurl(
+ ('https', proxy, '/' + path, None, None, None))
+ ud.parm['downloadfilename'] = path
+
+ # Set name parameter if sha256sum is set in recipe
+ name = f"{module}@{ud.parm['version']}"
+ if d.getVarFlag('SRC_URI', name + '.sha256sum'):
+ ud.parm['name'] = name
+
+ # Set subdir for unpack
+ ud.parm['subdir'] = os.path.join(moddir, 'cache/download',
+ os.path.dirname(path))
+
+ super().urldata_init(ud, d)
+
+ def unpack(self, ud, rootdir, d):
+ """Unpack the module in the module cache."""
+
+ # Unpack the module zip file or go.mod file
+ super().unpack(ud, rootdir, d)
+
+ if ud.localpath.endswith('.zip'):
+ # Unpack the go.mod file from the zip file
+ module = ud.parm['module']
+ unpackdir = os.path.join(rootdir, ud.parm['subdir'])
+ name = os.path.basename(ud.localpath).rsplit('.', 1)[0] + '.mod'
+ bb.note(f"Unpacking {name} to {unpackdir}/")
+ with zipfile.ZipFile(ud.localpath) as zf:
+ with open(os.path.join(unpackdir, name), mode='wb') as mf:
+ try:
+ f = module + '@' + ud.parm['version'] + '/go.mod'
+ shutil.copyfileobj(zf.open(f), mf)
+ except KeyError:
+ # If the module does not have a go.mod file, synthesize
+ # one containing only a module statement.
+ mf.write(f'module {module}\n'.encode())
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 2ef206343..2365a5096 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -3390,3 +3390,68 @@ class FetchPremirroronlyBrokenTarball(FetcherTest):
fetcher.download()
output = "".join(logs.output)
self.assertFalse(" not a git repository (or any parent up to mount point /)" in output)
+
+class GoModTest(FetcherTest):
+
+ @skipIfNoNetwork()
+ def test_gomod_url(self):
+ urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
+ 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip')
+ self.assertNotIn('name', ud.parm)
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomod_url_go_mod_only(self):
+ urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;'
+ 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod')
+ self.assertNotIn('name', ud.parm)
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomod_url_sha256sum_varflag(self):
+ urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0']
+ self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6')
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
+ self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0')
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomod_url_no_go_mod_in_module(self):
+ urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;'
+ 'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
+ self.assertNotIn('name', ud.parm)
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
--
2.39.2
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH v2 2/2] fetch2: Add gomodgit fetcher
2024-09-06 9:27 [PATCH v2 1/2] fetch2: Add gomod fetcher Christian Lindeberg
@ 2024-09-06 9:27 ` Christian Lindeberg
2024-09-07 7:27 ` [bitbake-devel] " Vyacheslav Yurkov
0 siblings, 1 reply; 4+ messages in thread
From: Christian Lindeberg @ 2024-09-06 9:27 UTC (permalink / raw)
To: bitbake-devel
From: Christian Lindeberg <christian.lindeberg@axis.com>
Add a go module fetcher for downloading module dependencies to the
module cache directly from a git repository. The fetcher can be used
with the go-mod class in OE-Core.
A module dependency can be specified with:
SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;srcrev=..."
Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com>
---
Changes in V2:
- Use separate gomodgit:// type instead of the direct=git parameter.
- Use GO_MOD_CACHE_DIR variable or default instead of requiring the
GOMODCACHE environment variable to be set.
- Add more documentation to make it cleared what is downloaded and what
is unpacked.
lib/bb/fetch2/__init__.py | 1 +
lib/bb/fetch2/gomod.py | 140 +++++++++++++++++++++++++++++++++++++-
lib/bb/tests/fetch.py | 89 ++++++++++++++++++++++++
3 files changed, 228 insertions(+), 2 deletions(-)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index f84ce5999..ddee4400b 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -2112,3 +2112,4 @@ methods.append(az.Az())
methods.append(crate.Crate())
methods.append(gcp.GCP())
methods.append(gomod.GoMod())
+methods.append(gomod.GoModGit())
diff --git a/lib/bb/fetch2/gomod.py b/lib/bb/fetch2/gomod.py
index fe025e367..1b532d03f 100644
--- a/lib/bb/fetch2/gomod.py
+++ b/lib/bb/fetch2/gomod.py
@@ -1,12 +1,13 @@
"""
BitBake 'Fetch' implementation for Go modules
-The gomod fetcher is used to download Go modules to the module cache from a
-module proxy.
+The gomod/gomodgit fetchers are used to download Go modules to the module cache
+from a module proxy or directly from a version control repository.
Example SRC_URI:
SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
+SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
Required SRC_URI parameters:
@@ -27,6 +28,23 @@ Optional SRC_URI parameters:
only the go.mod file. Alternatively, set the SRC_URI varible flag for
"module@version.sha256sum".
+- protocol
+ The method used when fetching directly from a version control repository.
+ The default is "https" for git.
+
+- repo
+ The URL when fetching directly from a version control repository. Required
+ when the URL is different from the module path.
+
+- srcrev
+ The revision identifier used when fetching directly from a version control
+ repository. Alternatively, set the SRCREV varible for "module@version".
+
+- subdir
+ The module subdirectory when fetching directly from a version control
+ repository. Required when the module is not located in the root of the
+ repository.
+
Related variables:
- GO_MOD_PROXY
@@ -41,14 +59,19 @@ See the Go modules reference, https://go.dev/ref/mod, for more information
about the module cache, module proxies and version control systems.
"""
+import hashlib
import os
import re
import shutil
+import subprocess
import zipfile
import bb
from bb.fetch2 import FetchError
from bb.fetch2 import MissingParameterError
+from bb.fetch2 import runfetchcmd
+from bb.fetch2 import subprocess_setup
+from bb.fetch2.git import Git
from bb.fetch2.wget import Wget
@@ -126,3 +149,116 @@ class GoMod(Wget):
# If the module does not have a go.mod file, synthesize
# one containing only a module statement.
mf.write(f'module {module}\n'.encode())
+
+
+class GoModGit(Git):
+ """Class to fetch Go modules directly from a git repository"""
+
+ def supports(self, ud, d):
+ """Check to see if a given URL is for this fetcher."""
+ return ud.type == 'gomodgit'
+
+ def urldata_init(self, ud, d):
+ """Set up to download the module from the git repository.
+
+ Set up to download the git repository to the module cache directory and
+ unpack the module zip file and the go.mod file:
+
+ cache/vcs/<hash>: The bare git repository.
+ cache/download/<module>/@v/<version>.zip: The module zip file.
+ cache/download/<module>/@v/<version>.mod: The go.mod file.
+ """
+
+ moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+
+ if 'version' not in ud.parm:
+ raise MissingParameterError('version', ud.url)
+
+ module = ud.host + ud.path
+ ud.parm['module'] = module
+
+ # Set host, path and srcrev for git download
+ if 'repo' in ud.parm:
+ repo = ud.parm['repo']
+ idx = repo.find('/')
+ if idx != -1:
+ ud.host = repo[:idx]
+ ud.path = repo[idx:]
+ else:
+ ud.host = repo
+ ud.path = ''
+ if 'protocol' not in ud.parm:
+ ud.parm['protocol'] = 'https'
+ name = f"{module}@{ud.parm['version']}"
+ ud.names = [name]
+ srcrev = d.getVar('SRCREV_' + name)
+ if srcrev:
+ if 'srcrev' not in ud.parm:
+ ud.parm['srcrev'] = srcrev
+ else:
+ if 'srcrev' in ud.parm:
+ d.setVar('SRCREV_' + name, ud.parm['srcrev'])
+ if 'branch' not in ud.parm:
+ ud.parm['nobranch'] = '1'
+
+ # Set subpath, subdir and bareclone for git unpack
+ if 'subdir' in ud.parm:
+ ud.parm['subpath'] = ud.parm['subdir']
+ key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
+ ud.parm['key'] = key
+ ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
+ hashlib.sha256(key).hexdigest())
+ ud.parm['bareclone'] = '1'
+
+ super().urldata_init(ud, d)
+
+ def unpack(self, ud, rootdir, d):
+ """Unpack the module in the module cache."""
+
+ # Unpack the bare git repository
+ super().unpack(ud, rootdir, d)
+
+ moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+
+ # Create the info file
+ module = ud.parm['module']
+ repodir = os.path.join(rootdir, ud.parm['subdir'])
+ with open(repodir + '.info', 'wb') as f:
+ f.write(ud.parm['key'])
+
+ # Unpack the go.mod file from the repository
+ unpackdir = os.path.join(rootdir, moddir, 'cache/download',
+ escape(module), '@v')
+ bb.utils.mkdirhier(unpackdir)
+ srcrev = ud.parm['srcrev']
+ version = ud.parm['version']
+ escaped_version = escape(version)
+ cmd = f"git ls-tree -r --name-only '{srcrev}'"
+ if 'subpath' in ud.parm:
+ cmd += f" '{ud.parm['subpath']}'"
+ files = runfetchcmd(cmd, d, workdir=repodir).split()
+ name = escaped_version + '.mod'
+ bb.note(f"Unpacking {name} to {unpackdir}/")
+ with open(os.path.join(unpackdir, name), mode='wb') as mf:
+ f = 'go.mod'
+ if 'subpath' in ud.parm:
+ f = os.path.join(ud.parm['subpath'], f)
+ if f in files:
+ cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+ subprocess.check_call(cmd, stdout=mf, cwd=repodir,
+ preexec_fn=subprocess_setup)
+ else:
+ # If the module does not have a go.mod file, synthesize one
+ # containing only a module statement.
+ mf.write(f'module {module}\n'.encode())
+
+ # Synthesize the module zip file from the repository
+ name = escaped_version + '.zip'
+ bb.note(f"Unpacking {name} to {unpackdir}/")
+ with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
+ prefix = module + '@' + version + '/'
+ for f in files:
+ cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+ data = subprocess.check_output(cmd, cwd=repodir,
+ preexec_fn=subprocess_setup)
+ zf.writestr(prefix + f, data)
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 2365a5096..832e0dd6a 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -3455,3 +3455,92 @@ class GoModTest(FetcherTest):
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+
+class GoModGitTest(FetcherTest):
+
+ @skipIfNoNetwork()
+ def test_gomodgit_url_repo(self):
+ urls = ['gomodgit://golang.org/x/net;version=v0.9.0;'
+ 'repo=go.googlesource.com/net;'
+ 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'go.googlesource.com')
+ self.assertEqual(ud.path, '/net')
+ self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0'])
+ self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
+
+ fetcher.download()
+ self.assertTrue(os.path.exists(ud.localpath))
+
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomodgit_url_subdir(self):
+ urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
+ 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;'
+ 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'github.com')
+ self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
+ self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
+ self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'])
+ self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
+
+ fetcher.download()
+ self.assertTrue(os.path.exists(ud.localpath))
+
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomodgit_url_srcrev_var(self):
+ urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0']
+ self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'gopkg.in')
+ self.assertEqual(ud.path, '/ini.v1')
+ self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
+ self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomodgit_url_no_go_mod_in_module(self):
+ urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;'
+ 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'gopkg.in')
+ self.assertEqual(ud.path, '/ini.v1')
+ self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
+ self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
--
2.39.2
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [bitbake-devel] [PATCH v2 2/2] fetch2: Add gomodgit fetcher
2024-09-06 9:27 ` [PATCH v2 2/2] fetch2: Add gomodgit fetcher Christian Lindeberg
@ 2024-09-07 7:27 ` Vyacheslav Yurkov
2024-09-10 9:32 ` Christian Lindeberg
0 siblings, 1 reply; 4+ messages in thread
From: Vyacheslav Yurkov @ 2024-09-07 7:27 UTC (permalink / raw)
To: christian.lindeberg, bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 14753 bytes --]
A few questions on the patchset.
- What if the module doesn't use any proxy? (E.g. hosted in a private
repository)
- The current recipetool approach (
https://git.openembedded.org/openembedded-core/tree/scripts/lib/recipetool/create_go.py
) uses vendoring approach and all dependencies go into the SRC_URI. Thus
it's required to populate the required license information for the
manifest generation. Would that be possible with this fetcher as well?
Slava
On 06.09.2024 11:27, Christian Lindeberg via lists.openembedded.org wrote:
> From: Christian Lindeberg<christian.lindeberg@axis.com>
>
> Add a go module fetcher for downloading module dependencies to the
> module cache directly from a git repository. The fetcher can be used
> with the go-mod class in OE-Core.
>
> A module dependency can be specified with:
>
> SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;srcrev=..."
>
> Signed-off-by: Christian Lindeberg<christian.lindeberg@axis.com>
> ---
>
> Changes in V2:
> - Use separate gomodgit:// type instead of the direct=git parameter.
> - Use GO_MOD_CACHE_DIR variable or default instead of requiring the
> GOMODCACHE environment variable to be set.
> - Add more documentation to make it cleared what is downloaded and what
> is unpacked.
>
> lib/bb/fetch2/__init__.py | 1 +
> lib/bb/fetch2/gomod.py | 140 +++++++++++++++++++++++++++++++++++++-
> lib/bb/tests/fetch.py | 89 ++++++++++++++++++++++++
> 3 files changed, 228 insertions(+), 2 deletions(-)
>
> diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
> index f84ce5999..ddee4400b 100644
> --- a/lib/bb/fetch2/__init__.py
> +++ b/lib/bb/fetch2/__init__.py
> @@ -2112,3 +2112,4 @@ methods.append(az.Az())
> methods.append(crate.Crate())
> methods.append(gcp.GCP())
> methods.append(gomod.GoMod())
> +methods.append(gomod.GoModGit())
> diff --git a/lib/bb/fetch2/gomod.py b/lib/bb/fetch2/gomod.py
> index fe025e367..1b532d03f 100644
> --- a/lib/bb/fetch2/gomod.py
> +++ b/lib/bb/fetch2/gomod.py
> @@ -1,12 +1,13 @@
> """
> BitBake 'Fetch' implementation for Go modules
>
> -The gomod fetcher is used to download Go modules to the module cache from a
> -module proxy.
> +The gomod/gomodgit fetchers are used to download Go modules to the module cache
> +from a module proxy or directly from a version control repository.
>
> Example SRC_URI:
>
> SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
> +SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
>
> Required SRC_URI parameters:
>
> @@ -27,6 +28,23 @@ Optional SRC_URI parameters:
> only the go.mod file. Alternatively, set the SRC_URI varible flag for
> "module@version.sha256sum".
>
> +- protocol
> + The method used when fetching directly from a version control repository.
> + The default is "https" for git.
> +
> +- repo
> + The URL when fetching directly from a version control repository. Required
> + when the URL is different from the module path.
> +
> +- srcrev
> + The revision identifier used when fetching directly from a version control
> + repository. Alternatively, set the SRCREV varible for "module@version".
> +
> +- subdir
> + The module subdirectory when fetching directly from a version control
> + repository. Required when the module is not located in the root of the
> + repository.
> +
> Related variables:
>
> - GO_MOD_PROXY
> @@ -41,14 +59,19 @@ See the Go modules reference,https://go.dev/ref/mod, for more information
> about the module cache, module proxies and version control systems.
> """
>
> +import hashlib
> import os
> import re
> import shutil
> +import subprocess
> import zipfile
>
> import bb
> from bb.fetch2 import FetchError
> from bb.fetch2 import MissingParameterError
> +from bb.fetch2 import runfetchcmd
> +from bb.fetch2 import subprocess_setup
> +from bb.fetch2.git import Git
> from bb.fetch2.wget import Wget
>
>
> @@ -126,3 +149,116 @@ class GoMod(Wget):
> # If the module does not have a go.mod file, synthesize
> # one containing only a module statement.
> mf.write(f'module {module}\n'.encode())
> +
> +
> +class GoModGit(Git):
> + """Class to fetch Go modules directly from a git repository"""
> +
> + def supports(self, ud, d):
> + """Check to see if a given URL is for this fetcher."""
> + return ud.type == 'gomodgit'
> +
> + def urldata_init(self, ud, d):
> + """Set up to download the module from the git repository.
> +
> + Set up to download the git repository to the module cache directory and
> + unpack the module zip file and the go.mod file:
> +
> + cache/vcs/<hash>: The bare git repository.
> + cache/download/<module>/@v/<version>.zip: The module zip file.
> + cache/download/<module>/@v/<version>.mod: The go.mod file.
> + """
> +
> + moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
> +
> + if 'version' not in ud.parm:
> + raise MissingParameterError('version', ud.url)
> +
> + module = ud.host + ud.path
> + ud.parm['module'] = module
> +
> + # Set host, path and srcrev for git download
> + if 'repo' in ud.parm:
> + repo = ud.parm['repo']
> + idx = repo.find('/')
> + if idx != -1:
> + ud.host = repo[:idx]
> + ud.path = repo[idx:]
> + else:
> + ud.host = repo
> + ud.path = ''
> + if 'protocol' not in ud.parm:
> + ud.parm['protocol'] = 'https'
> + name = f"{module}@{ud.parm['version']}"
> + ud.names = [name]
> + srcrev = d.getVar('SRCREV_' + name)
> + if srcrev:
> + if 'srcrev' not in ud.parm:
> + ud.parm['srcrev'] = srcrev
> + else:
> + if 'srcrev' in ud.parm:
> + d.setVar('SRCREV_' + name, ud.parm['srcrev'])
> + if 'branch' not in ud.parm:
> + ud.parm['nobranch'] = '1'
> +
> + # Set subpath, subdir and bareclone for git unpack
> + if 'subdir' in ud.parm:
> + ud.parm['subpath'] = ud.parm['subdir']
> + key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
> + ud.parm['key'] = key
> + ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
> + hashlib.sha256(key).hexdigest())
> + ud.parm['bareclone'] = '1'
> +
> + super().urldata_init(ud, d)
> +
> + def unpack(self, ud, rootdir, d):
> + """Unpack the module in the module cache."""
> +
> + # Unpack the bare git repository
> + super().unpack(ud, rootdir, d)
> +
> + moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
> +
> + # Create the info file
> + module = ud.parm['module']
> + repodir = os.path.join(rootdir, ud.parm['subdir'])
> + with open(repodir + '.info', 'wb') as f:
> + f.write(ud.parm['key'])
> +
> + # Unpack the go.mod file from the repository
> + unpackdir = os.path.join(rootdir, moddir, 'cache/download',
> + escape(module), '@v')
> + bb.utils.mkdirhier(unpackdir)
> + srcrev = ud.parm['srcrev']
> + version = ud.parm['version']
> + escaped_version = escape(version)
> + cmd = f"git ls-tree -r --name-only '{srcrev}'"
> + if 'subpath' in ud.parm:
> + cmd += f" '{ud.parm['subpath']}'"
> + files = runfetchcmd(cmd, d, workdir=repodir).split()
> + name = escaped_version + '.mod'
> + bb.note(f"Unpacking {name} to {unpackdir}/")
> + with open(os.path.join(unpackdir, name), mode='wb') as mf:
> + f = 'go.mod'
> + if 'subpath' in ud.parm:
> + f = os.path.join(ud.parm['subpath'], f)
> + if f in files:
> + cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
> + subprocess.check_call(cmd, stdout=mf, cwd=repodir,
> + preexec_fn=subprocess_setup)
> + else:
> + # If the module does not have a go.mod file, synthesize one
> + # containing only a module statement.
> + mf.write(f'module {module}\n'.encode())
> +
> + # Synthesize the module zip file from the repository
> + name = escaped_version + '.zip'
> + bb.note(f"Unpacking {name} to {unpackdir}/")
> + with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
> + prefix = module + '@' + version + '/'
> + for f in files:
> + cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
> + data = subprocess.check_output(cmd, cwd=repodir,
> + preexec_fn=subprocess_setup)
> + zf.writestr(prefix + f, data)
> diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
> index 2365a5096..832e0dd6a 100644
> --- a/lib/bb/tests/fetch.py
> +++ b/lib/bb/tests/fetch.py
> @@ -3455,3 +3455,92 @@ class GoModTest(FetcherTest):
> downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
> self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
> self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
> +
> +class GoModGitTest(FetcherTest):
> +
> + @skipIfNoNetwork()
> + def test_gomodgit_url_repo(self):
> + urls = ['gomodgit://golang.org/x/net;version=v0.9.0;'
> + 'repo=go.googlesource.com/net;'
> + 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1']
> +
> + fetcher = bb.fetch2.Fetch(urls, self.d)
> + ud = fetcher.ud[urls[0]]
> + self.assertEqual(ud.host, 'go.googlesource.com')
> + self.assertEqual(ud.path, '/net')
> + self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0'])
> + self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
> +
> + fetcher.download()
> + self.assertTrue(os.path.exists(ud.localpath))
> +
> + fetcher.unpack(self.unpackdir)
> + vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
> + self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca')))
> + downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip')))
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')))
> +
> + @skipIfNoNetwork()
> + def test_gomodgit_url_subdir(self):
> + urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
> + 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;'
> + 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3']
> +
> + fetcher = bb.fetch2.Fetch(urls, self.d)
> + ud = fetcher.ud[urls[0]]
> + self.assertEqual(ud.host, 'github.com')
> + self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
> + self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
> + self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'])
> + self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
> +
> + fetcher.download()
> + self.assertTrue(os.path.exists(ud.localpath))
> +
> + fetcher.unpack(self.unpackdir)
> + vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
> + self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3')))
> + downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
> +
> + @skipIfNoNetwork()
> + def test_gomodgit_url_srcrev_var(self):
> + urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0']
> + self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951')
> +
> + fetcher = bb.fetch2.Fetch(urls, self.d)
> + ud = fetcher.ud[urls[0]]
> + self.assertEqual(ud.host, 'gopkg.in')
> + self.assertEqual(ud.path, '/ini.v1')
> + self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
> + self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
> +
> + fetcher.download()
> + fetcher.unpack(self.unpackdir)
> + vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
> + self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
> + downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
> +
> + @skipIfNoNetwork()
> + def test_gomodgit_url_no_go_mod_in_module(self):
> + urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;'
> + 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951']
> +
> + fetcher = bb.fetch2.Fetch(urls, self.d)
> + ud = fetcher.ud[urls[0]]
> + self.assertEqual(ud.host, 'gopkg.in')
> + self.assertEqual(ud.path, '/ini.v1')
> + self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
> + self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
> +
> + fetcher.download()
> + fetcher.unpack(self.unpackdir)
> + vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
> + self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
> + downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
> + self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
>
[-- Attachment #2: Type: text/html, Size: 16864 bytes --]
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH v2 2/2] fetch2: Add gomodgit fetcher
2024-09-07 7:27 ` [bitbake-devel] " Vyacheslav Yurkov
@ 2024-09-10 9:32 ` Christian Lindeberg
0 siblings, 0 replies; 4+ messages in thread
From: Christian Lindeberg @ 2024-09-10 9:32 UTC (permalink / raw)
To: bitbake-devel
[-- Attachment #1: Type: text/plain, Size: 1003 bytes --]
On Sat, Sep 7, 2024 at 09:27 AM, Vyacheslav Yurkov wrote:
>
> A few questions on the patchset.
>
> - What if the module doesn't use any proxy? (E.g. hosted in a private
> repository)
>
The gomodgit fetcher is for git repositories that needs to use the direct mode.
(Cf. https://go.dev/ref/mod#private-module-proxy-direct) ( https://go.dev/ref/mod#private-module-proxy-direct= )
>
> - The current recipetool approach ( https://git.openembedded.org/openembedded-core/tree/scripts/lib/recipetool/create_go.py
> ) uses vendoring approach and all dependencies go into the SRC_URI. Thus
> it's required to populate the required license information for the
> manifest generation. Would that be possible with this fetcher as well?
>
Yes, there would be no change when creating or updating the ${BPN}-licenses.inc
file compared to the vendoring approach. And then the gomod:// and gomodgit://
URLs would go into ${BPN}-modules.inc.
>
>
> Slava
>
>
Thanks,
Christian
[-- Attachment #2: Type: text/html, Size: 1465 bytes --]
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2024-09-10 9:32 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-09-06 9:27 [PATCH v2 1/2] fetch2: Add gomod fetcher Christian Lindeberg
2024-09-06 9:27 ` [PATCH v2 2/2] fetch2: Add gomodgit fetcher Christian Lindeberg
2024-09-07 7:27 ` [bitbake-devel] " Vyacheslav Yurkov
2024-09-10 9:32 ` Christian Lindeberg
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.