Re: [OE-Core][PATCH v2 1/4] classes: go-vendor: Add go-vendor class

["Ulrich Ölmann" <u.oelmann@pengutronix.de>]

Hi Lukas,

just two small typos I stumbled over.

On Tue, Oct 17 2023 at 15:26 +0200, "Lukas Funke" <lukas.funke-oss@weidmueller.com> wrote:
> From: Lukas Funke <lukas.funke@weidmueller.com>
>
> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
> ---
>  meta/classes/go-vendor.bbclass | 135 +++++++++++++++++++++++++++++++++
>  1 file changed, 135 insertions(+)
>  create mode 100644 meta/classes/go-vendor.bbclass
>
> diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
> new file mode 100644
> index 0000000000..13f1b8b2be
> --- /dev/null
> +++ b/meta/classes/go-vendor.bbclass
> @@ -0,0 +1,135 @@
> +#
> +# Copyright 2023 (C) Weidmueller GmbH & Co KG
> +# Author: Lukas Funke <lukas.funke@weidmueller.com>
> +#
> +# Handle Go vendor support for offline builds
> +#
> +# When importing Go modules, Go downloads the imported module using

s/imported module/imported modules/

> +# a network (proxy) connection ahead of the compile stage. This contradicts
> +# the yocto build concept of fetching every source ahead of build-time
> +# and supporting offline builds.
> +#
> +# To support offline builds, we use Go 'vendoring': module dependencies are
> +# downloaded during the fetch-phase and unpacked into the modules 'vendor'
> +# folder. Additinally a manifest file is generated for the 'vendor' folder

s/Additinally/Additionally/

Best regards
Ulrich


> +#
> +
> +inherit go-mod
> +
> +def go_src_uri(repo, version, path=None, subdir=None, \
> +                vcs='git', replaces=None, pathmajor=None):
> +
> +    destsuffix = "git/src/import/vendor.fetch"
> +    go_module_path = repo if not path else path
> +
> +    src_uri = "{}://{}" \
> +              ";name={}" \
> +              "".format(vcs, repo, \
> +                        go_module_path.replace('/', '.'))
> +
> +    src_uri += ";destsuffix={}/{}@{}".format(destsuffix, \
> +                                                go_module_path, \
> +                                                version)
> +
> +    if vcs == "git":
> +        src_uri += ";nobranch=1;protocol=https"
> +    if replaces:
> +        src_uri += ";go_module_replacement={}".format(replaces)
> +    if subdir:
> +        src_uri += ";go_subdir={}".format(subdir)
> +    if pathmajor:
> +        src_uri += ";go_pathmajor={}".format(pathmajor)
> +
> +    return src_uri
> +
> +
> +python do_go_vendor() {
> +    import shutil
> +
> +    src_uri = (d.getVar('SRC_URI') or "").split()
> +
> +    if len(src_uri) == 0:
> +        bb.error("SRC_URI is empty")
> +        return
> +
> +    default_destsuffix = "git/src/import/vendor.fetch"
> +    fetcher = bb.fetch2.Fetch(src_uri, d)
> +    go_import = d.getVar('GO_IMPORT')
> +    source_dir = d.getVar('S')
> +
> +    vendor_dir = os.path.join(source_dir, *['src', go_import, 'vendor'])
> +    import_dir = os.path.join(source_dir, *['src', 'import', 'vendor.fetch'])
> +
> +    bb.utils.mkdirhier(vendor_dir)
> +    modules = {}
> +
> +    for url in fetcher.urls:
> +        srcuri = fetcher.ud[url].host + fetcher.ud[url].path
> +
> +        # Skip main module for which the recipe is actually created
> +        if srcuri == go_import:
> +            continue
> +
> +        # Skip local files
> +        if fetcher.ud[url].type == 'file':
> +            continue
> +
> +        destsuffix = fetcher.ud[url].parm.get('destsuffix')
> +        # We derive the module path / version in the following manner (exmaple):
> +        #
> +        # destsuffix = git/src/import/vendor.fetch/github.com/foo/bar@v1.2.3
> +        # p = github.com/foo/bar@v1.2.3
> +        # path = github.com/foo/bar
> +        # version = v1.2.3
> +
> +        p = destsuffix[len(default_destsuffix)+1:]
> +        path, version = p.split('@')
> +
> +        subdir = fetcher.ud[url].parm.get('go_subdir')
> +        subdir = "" if not subdir else subdir
> +
> +        pathMajor = fetcher.ud[url].parm.get('go_pathmajor')
> +        pathMajor = "" if not pathMajor else pathMajor
> +
> +        base = path[:-(len(subdir)+len(pathMajor))-1]
> +        r = fetcher.ud[url].parm.get('go_module_replacement')
> +
> +        if not path in modules and not r:
> +            modules[path] =   {
> +                                "version": version,
> +                                "src": os.path.join(import_dir, *[p, subdir]),
> +                                "subdir": subdir,
> +                                "pathMajor": pathMajor,
> +                              }
> +
> +    for module_key in sorted(modules):
> +
> +        # only take the version which is explicitly listed
> +        # as a dependency in the go.mod
> +        module = modules[module_key]
> +        src = module['src']
> +
> +        # If the module is released at major version 2 or higher, the module
> +        # path must end with a major version suffix like /v2.
> +        # This may or may not be part of the subdirectory name
> +        #
> +        # https://go.dev/ref/mod#modules-overview
> +        srcMajorVersion = os.path.join(src, module['pathMajor'].strip('/'))
> +        if os.path.exists(srcMajorVersion):
> +            src = srcMajorVersion
> +        dst = os.path.join(vendor_dir, module_key)
> +        if os.path.exists(dst):
> +            shutil.rmtree(dst)
> +
> +        bb.debug(1, "cp %s --> %s" % (src, dst))
> +        shutil.copytree(src, dst, symlinks=True, \
> +            ignore=shutil.ignore_patterns(".git", \
> +                                            "vendor", \
> +                                            "*.md", \
> +                                            "*._test.go"))
> +    # Copy vendor manifest
> +    bb.debug(1, "cp %s --> %s" % (os.path.join(d.getVar('WORKDIR'), "modules.txt"), vendor_dir))
> +    shutil.copy2(os.path.join(d.getVar('WORKDIR'), "modules.txt"), vendor_dir)
> +}
> +
> +addtask go_vendor before do_populate_lic after do_unpack
-- 
Pengutronix e.K.                           | Ulrich Ölmann               |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
Peiner Str. 6-8, 31137 Hildesheim, Germany | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |