Re: [OE-core] [PATCH 4/5] recipetool: add go recipe generator

[Mark Asselstine <mark.asselstine@windriver.com>]

On 2022-05-06 03:15, Alexander Kanavin wrote:
> This is a lot of code. Can you add some documentation for it, what it
> does and how it works? If someone would want to understand it, how
> would they go about it?
> 
> Alex
> 
> On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
> <stefan.herbrechtsmeier-oss@weidmueller.com> wrote:
>>
>> From: Lukas Funke <lukas.funke@weidmueller.com>
>>
>> Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
>> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
>> ---
>>
>>   scripts/lib/recipetool/create_go.py | 394 ++++++++++++++++++++++++++++
>>   1 file changed, 394 insertions(+)
>>   create mode 100644 scripts/lib/recipetool/create_go.py
>>
>> diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
>> new file mode 100644
>> index 0000000000..4552e9b470
>> --- /dev/null
>> +++ b/scripts/lib/recipetool/create_go.py
>> @@ -0,0 +1,394 @@
>> +# Recipe creation tool - go support plugin
>> +#
>> +# Copyright (C) 2022 Weidmueller GmbH & Co KG
>> +# Author: Lukas Funke <lukas.funke@weidmueller.com>
>> +#
>> +# Copyright (c) 2009 The Go Authors. All rights reserved.
>> +#
>> +#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
>> +#
>> +import bb.utils
>> +from collections import namedtuple
>> +from enum import Enum
>> +from html.parser import HTMLParser
>> +import json
>> +import logging
>> +import os
>> +import re
>> +import subprocess
>> +import sys
>> +import tempfile
>> +import shutil
>> +from urllib.error import URLError, HTTPError
>> +import urllib.parse
>> +import urllib.request
>> +
>> +from recipetool.create import RecipeHandler, handle_license_vars, ensure_native_cmd
>> +
>> +GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
>> +logger = logging.getLogger('recipetool')
>> +
>> +tinfoil = None
>> +
>> +re_pseudo_semver = re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P<utc>[0-9]+)-(?P<sha1_abbrev>[0-9Aa-zA-Z]+)")
>> +re_semver = re.compile(r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
>> +
>> +def tinfoil_init(instance):
>> +    global tinfoil
>> +    tinfoil = instance
>> +
>> +class GoRecipeHandler(RecipeHandler):
>> +
>> +    def _resolve_repository_static(self, modulepath):
>> +        _rootpath = None
>> +        _vcs = None
>> +        _repourl = None
>> +        _suffix = None
>> +
>> +        host, _, path = modulepath.partition('/')
>> +
>> +        class vcs(Enum):
>> +            pathprefix = "pathprefix"
>> +            regexp = "regexp"
>> +            vcs = "vcs"
>> +            repo = "repo"
>> +            check = "check"
>> +            schemelessRepo = "schemelessRepo"
>> +
>> +        # GitHub
>> +        vcsGitHub = {}
>> +        vcsGitHub[vcs.pathprefix] = "github.com"
>> +        vcsGitHub[vcs.regexp] = re.compile(r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsGitHub[vcs.vcs] = "git"
>> +        vcsGitHub[vcs.repo] = "https://\g<root>"
>> +
>> +        # Bitbucket
>> +        vcsBitbucket = {}
>> +        vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
>> +        vcsBitbucket[vcs.regexp] = re.compile(r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsBitbucket[vcs.vcs] = "git"
>> +        vcsBitbucket[vcs.repo] = "https://\g<root>"
>> +
>> +        # IBM DevOps Services (JazzHub)
>> +        vcsIBMDevOps = {}
>> +        vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
>> +        vcsIBMDevOps[vcs.regexp] = re.compile(r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsIBMDevOps[vcs.vcs] = "git"
>> +        vcsIBMDevOps[vcs.repo] = "https://\g<root>"
>> +
>> +        # Git at Apache
>> +        vcsApacheGit = {}
>> +        vcsApacheGit[vcs.pathprefix] = "git.apache.org"
>> +        vcsApacheGit[vcs.regexp] = re.compile(r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsApacheGit[vcs.vcs] = "git"
>> +        vcsApacheGit[vcs.repo] = "https://\g<root>"
>> +
>> +        # Git at OpenStack
>> +        vcsOpenStackGit = {}
>> +        vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
>> +        vcsOpenStackGit[vcs.regexp] = re.compile(r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
>> +        vcsOpenStackGit[vcs.vcs] = "git"
>> +        vcsOpenStackGit[vcs.repo] = "https://\g<root>"
>> +
>> +        # chiselapp.com for fossil
>> +        vcsChiselapp = {}
>> +        vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
>> +        vcsChiselapp[vcs.regexp] = re.compile(r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
>> +        vcsChiselapp[vcs.vcs] = "fossil"
>> +        vcsChiselapp[vcs.repo] = "https://\g<root>"
>> +
>> +        # General syntax for any server.
>> +        # Must be last.
>> +        vcsGeneralServer = {}
>> +        vcsGeneralServer[vcs.regexp] = re.compile("(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\-]+))*$")
>> +        vcsGeneralServer[vcs.schemelessRepo] = True
>> +
>> +        vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, vcsApacheGit, vcsOpenStackGit, vcsChiselapp, vcsGeneralServer]
>> +
>> +        if modulepath.startswith("example.net") or modulepath == "rsc.io":
>> +            logger.warning("Suspicious module path %s" % modulepath)
>> +            return None
>> +        if modulepath.startswith("http:") or modulepath.startswith("https:"):
>> +            logger.warning("Import path should not start with %s %s" % ("http", "https"))
>> +            return None
>> +
>> +        for srv in vcsPaths:
>> +            m = srv[vcs.regexp].match(modulepath)
>> +            if vcs.pathprefix in srv:
>> +                if host == srv[vcs.pathprefix]:
>> +                    _rootpath = m.group('root')
>> +                    _vcs = srv[vcs.vcs]
>> +                    _repourl = m.expand(srv[vcs.repo])
>> +                    _suffix = m.group('suffix')
>> +                    break
>> +            elif m and srv[vcs.schemelessRepo]:
>> +                _rootpath = m.group('root')
>> +                _vcs = m[vcs.vcs]
>> +                _repourl = m[vcs.repo]
>> +                _suffix = m.group('suffix')
>> +                break
>> +
>> +        return GoImport(_rootpath, _vcs, _repourl, _suffix)
>> +
>> +    def _resolve_repository_dynamic(self, modulepath):
>> +
>> +        url = urllib.parse.urlparse("https://" + modulepath)
>> +
>> +        class GoImportHTMLParser(HTMLParser):
>> +
>> +            def __init__(self):
>> +                super().__init__()
>> +                self.__srv = []
>> +
>> +            def handle_starttag(self, tag, attrs):
>> +                if tag == 'meta' and list(filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
>> +                    content = list(filter(lambda a: (a[0] == 'content'), attrs))
>> +                    if content:
>> +                        self.__srv = content[0][1].split()
>> +
>> +            @property
>> +            def rootpath(self):
>> +                return self.__srv[0]
>> +
>> +            @property
>> +            def vcs(self):
>> +                return self.__srv[1]
>> +
>> +            @property
>> +            def repourl(self):
>> +                return self.__srv[2]
>> +
>> +        req = urllib.request.Request(url.geturl() + "?go-get=1")
>> +
>> +        try:
>> +            resp = urllib.request.urlopen(req)
>> +        except URLError as url_err:
>> +            logger.error("Error while fetching redirect page: %s", str(url_err))
>> +            return None
>> +        except HTTPError as http_err:
>> +            logger.error("Error while fetching redirect page: %s", str(http_err))
>> +            return None
>> +
>> +        parser = GoImportHTMLParser()
>> +        parser.feed(resp.read().decode('utf-8'))
>> +        parser.close()
>> +
>> +        return GoImport(parser.rootpath, parser.vcs, parser.repourl, None)

Attempting this on minio client

recipetool create -o ./maa https://github.com/minio/mc \
   -S b5a0640899f8f8653bcacd19791c92ca22066ba3

I am seeing

   File "/home/mark/git/poky/scripts/lib/recipetool/create_go.py", line 
151, in rootpath
     return self.__srv[0]
IndexError: list index out of range

Is more error checking required?

MarkA

>> +
>> +    def _resolve_repository(self, modulepath):
>> +        """
>> +        Resolves src uri from go module-path
>> +        """
>> +        repodata = self._resolve_repository_static(modulepath)
>> +        if not repodata.repourl:
>> +            repodata = self._resolve_repository_dynamic(modulepath)
>> +
>> +        if repodata:
>> +            logger.info("Resolved download path for import '%s' => %s", modulepath, repodata.repourl)
>> +
>> +        return repodata
>> +
>> +    def _resolve_pseudo_semver(self, d, repo, module_version):
>> +        hash = None
>> +
>> +        def vcs_fetch_all():
>> +            tmpdir = tempfile.mkdtemp()
>> +            clone_cmd = "%s clone --bare %s %s" % ('git', repo, tmpdir)
>> +            bb.process.run(clone_cmd)
>> +            log_cmd = "git log --all --pretty='%H %d' --decorate=short"
>> +            output, errors = bb.process.run(log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
>> +            bb.utils.prunedir(tmpdir)
>> +            return output.strip().split('\n')
>> +
>> +        def vcs_fetch_remote(search=""):
>> +            ls_remote_cmd = "git ls-remote --tags {} {}".format(repo, search)
>> +            output, errors = bb.process.run(ls_remote_cmd)
>> +            return output.strip().split('\n')
>> +
>> +        m_pseudo_semver = re_pseudo_semver.match(module_version)
>> +        if m_pseudo_semver:
>> +            remote_refs = vcs_fetch_all()
>> +            short_commit = m_pseudo_semver.group('sha1_abbrev')
>> +            for l in remote_refs:
>> +                r = l.split(maxsplit=1)
>> +                sha1 = r[0] if len(r) else None
>> +                if not sha1:
>> +                    logger.error("Ups: could not resolve abbref commit for %s" % short_commit)
>> +
>> +                elif sha1.startswith(short_commit):
>> +                    hash = sha1
>> +                    break
>> +        else:
>> +            m_semver = re_semver.match(module_version)
>> +            if m_semver:
>> +
>> +                def get_sha1_remote(re, groupId):
>> +                    for l in remote_refs:
>> +                        r = l.split(maxsplit=1)
>> +                        sha1 = r[0] if len(r) else None
>> +                        ref = r[1] if len(r) == 2 else None
>> +                        if ref:
>> +                            m = re.match(ref)
>> +                            if m and semver_tag in m.group(groupId).split(','):
>> +                                return sha1
>> +                    return None
>> +
>> +                re_tags_remote = re.compile(r"refs/tags/(?P<tag>[0-9A-Za-z-_\.]+)")
>> +                re_tags_all = re.compile(r"\((HEAD -> (.*), )?tag: *((?:([0-9A-Za-z-_\.]+),? *)+)\)")
>> +                semver_tag = "v" + m_semver.group('major') + "."\
>> +                                +m_semver.group('minor') + "."\
>> +                                +m_semver.group('patch') \
>> +                                +(("-" + m_semver.group('prerelease')) if m_semver.group('prerelease') else "")
>> +                remote_refs = vcs_fetch_remote(semver_tag)
>> +                # probe tag using 'ls-remote', which is faster than fetching complete history
>> +                sha1 = get_sha1_remote(re_tags_remote, 'tag')
>> +                if sha1:
>> +                    hash = sha1
>> +                else:
>> +                    # backup: fetch complete history
>> +                    remote_refs = vcs_fetch_all()
>> +                    hash = get_sha1_remote(re_tags_all, 3)
>> +        return hash
>> +
>> +    def _handle_dependencies(self, d, srctree, go_mod):
>> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
>> +        src_uris = []
>> +        src_revs = []
>> +        for require in go_mod['Require']:
>> +            module_path = require['Path']
>> +            module_version = require['Version']
>> +
>> +            repodata = self._resolve_repository(module_path)
>> +            commit_id = self._resolve_pseudo_semver(d, repodata.repourl, module_version)
>> +            url = urllib.parse.urlparse(repodata.repourl)
>> +            repo_url = url.netloc + url.path
>> +            inline_fcn = "${@go_src_uri("
>> +            inline_fcn += "'{}'".format(repo_url)
>> +            if repo_url != module_path:
>> +                inline_fcn += ",path='{}'".format(module_path)
>> +            if repodata.suffix and not re.match("v[0-9]+", repodata.suffix):
>> +                inline_fcn += ",subdir='{}'".format(repodata.suffix)
>> +            if repodata.vcs != 'git':
>> +                inline_fcn += ",vcs='{}'".format(repodata.vcs)
>> +            inline_fcn += ")}"
>> +
>> +            src_uris.append(inline_fcn)
>> +            flat_module_path = module_path.replace('/', '.')
>> +            src_rev = "# %s@%s => %s\n" % (module_path, module_version, commit_id)
>> +            src_rev += "SRCREV_%s = \"%s\"\n" % (flat_module_path, commit_id)
>> +            src_rev += "GO_MODULE_PATH[%s] = \"%s\"\n" % (flat_module_path, module_path)
>> +            src_rev += "GO_MODULE_VERSION[%s] = \"%s\"" % (flat_module_path, module_version)
>> +            src_revs.append(src_rev)
>> +
>> +        return src_uris, src_revs
>> +
>> +    def _go_mod_patch(self, patchfile, go_import, srctree, localfilesdir, extravalues, d):
>> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
>> +        # first remove go.mod and go.sum, otherwise 'go mod init' will fail
>> +        bb.utils.remove(os.path.join(srctree, "go.mod"))
>> +        bb.utils.remove(os.path.join(srctree, "go.sum"))
>> +        bb.process.run("go mod init %s" % go_import, stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        bb.process.run("go mod tidy", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        bb.process.run("git diff go.mod > %s" % (patchfile), stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        bb.process.run("git checkout HEAD go.mod go.sum;", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        go_mod = json.loads(output)
>> +        tmpfile = os.path.join(localfilesdir, patchfile)
>> +        shutil.move(os.path.join(srctree, patchfile), tmpfile)
>> +        extravalues.setdefault('extrafiles', {})
>> +        extravalues['extrafiles'][patchfile] = tmpfile
>> +
>> +        return go_mod
>> +
>> +    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
>> +
>> +        if 'buildsystem' in handled:
>> +            return False
>> +
>> +        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
>> +        if not files:
>> +            return False
>> +
>> +        go_bindir = ensure_native_cmd(tinfoil, "go")
>> +
>> +        d = bb.data.createCopy(tinfoil.config_data)
>> +        d.prependVar('PATH', '%s:' % go_bindir)
>> +        handled.append('buildsystem')
>> +        classes.append("go-vendor")
>> +
>> +        runenv = dict(os.environ, PATH=d.getVar('PATH'))
>> +        output, _ = bb.process.run("go mod edit -json", stderr=subprocess.STDOUT, env=runenv, shell=True, cwd=srctree)
>> +        go_mod = json.loads(output)
>> +
>> +        go_import = go_mod['Module']['Path']
>> +        go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
>> +        go_version_major = int(go_version_match.group(1))
>> +        go_version_minor = int(go_version_match.group(2))
>> +        src_uris = []
>> +        if go_version_major == 1 and go_version_minor < 17:
>> +            logger.warning("go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
>> +            patchfile = "go.mod.patch"
>> +            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
>> +            go_mod = self._go_mod_patch(patchfile, go_import, srctree, localfilesdir, extravalues, d)
>> +            src_uris.append("file://%s;patchdir=src/${GO_IMPORT}" % (patchfile))
>> +
>> +        if not os.path.exists(os.path.join(srctree, "vendor")):
>> +            dep_src_uris, src_revs = self._handle_dependencies(d, srctree, go_mod)
>> +            src_uris.extend(dep_src_uris)
>> +            lines_after.append("#TODO: Subdirectories are heuristically derived from " \
>> +                              "the import path and might be incorrect.")
>> +            for src_rev in src_revs:
>> +                lines_after.append(src_rev)
>> +
>> +        self._rewrite_src_uri(src_uris, lines_before)
>> +
>> +        handle_license_vars(srctree, lines_before, handled, extravalues, d)
>> +        self._rewrite_lic_uri(lines_before)
>> +
>> +        lines_before.append("GO_IMPORT = \"{}\"".format(go_import))
>> +        lines_before.append("SRCREV_FORMAT = \"${PN}\"")
>> +
>> +    def _update_lines_before(self, updated, newlines, lines_before):
>> +        if updated:
>> +            del lines_before[:]
>> +            for line in newlines:
>> +                # Hack to avoid newlines that edit_metadata inserts
>> +                if line.endswith('\n'):
>> +                    line = line[:-1]
>> +                lines_before.append(line)
>> +        return updated
>> +
>> +    def _rewrite_lic_uri(self, lines_before):
>> +
>> +        def varfunc(varname, origvalue, op, newlines):
>> +            if varname == 'LIC_FILES_CHKSUM':
>> +                new_licenses = []
>> +                licenses = origvalue.split()
>> +
>> +                for license in licenses:
>> +                    uri, chksum = license.split(';', 1)
>> +                    url = urllib.parse.urlparse(uri)
>> +                    new_uri = os.path.join(url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
>> +                    new_licenses.append(new_uri)
>> +
>> +                return new_licenses, None, -1, True
>> +            return origvalue, None, 0, True
>> +
>> +        updated, newlines = bb.utils.edit_metadata(lines_before, ['LIC_FILES_CHKSUM'], varfunc)
>> +        return self._update_lines_before(updated, newlines, lines_before)
>> +
>> +    def _rewrite_src_uri(self, src_uris_deps, lines_before):
>> +
>> +        def varfunc(varname, origvalue, op, newlines):
>> +            if varname == 'SRC_URI':
>> +                src_uri = []
>> +                src_uri.append("git://${GO_IMPORT};nobranch=1;name=${PN};protocol=https")
>> +                src_uri.extend(src_uris_deps)
>> +                return src_uri, None, -1, True
>> +            return origvalue, None, 0, True
>> +
>> +        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
>> +        return self._update_lines_before(updated, newlines, lines_before)
>> +
>> +def register_recipe_handlers(handlers):
>> +    handlers.append((GoRecipeHandler(), 60))
>> --
>> 2.30.2
>>
>>
>>
>>
>>
>>
>> -=-=-=-=-=-=-=-=-=-=-=-
>> Links: You receive all messages sent to this group.
>> View/Reply Online (#165331): https://lists.openembedded.org/g/openembedded-core/message/165331
>> Mute This Topic: https://lists.openembedded.org/mt/90928688/3616946
>> Group Owner: openembedded-core+owner@lists.openembedded.org
>> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mark.asselstine@windriver.com]
>> -=-=-=-=-=-=-=-=-=-=-=-
>>