[PATCH 0/3] Go module dependencies

[PATCH 0/3] Go module dependencies

[Christian Lindeberg]

From: Christian Lindeberg <christian.lindeberg@axis.com>

Here's a stack that together with another stack for meta-oe will try to
close the gap between the go-mod-update-modules class and the two gomod
fetcher based recipes in meta-oe.

Christian Lindeberg (3):
  oe/license_finder: Add find_licenses_up function
  go-mod-update-modules.bbclass: Update license finding
  recipetool/create_go: Tidy up a bit

 .../go-mod-update-modules.bbclass             | 30 ++++----
 meta/lib/oe/license_finder.py                 | 27 ++++++-
 scripts/lib/recipetool/create_go.py           | 70 +++++++++----------
 3 files changed, 72 insertions(+), 55 deletions(-)

-- 
2.39.5

[PATCH 1/3] oe/license_finder: Add find_licenses_up function

[Christian Lindeberg]

From: Christian Lindeberg <christian.lindeberg@axis.com>

Add a function for finding licenses in a directory or upwards but not
above a top directory.

Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com>
---
 meta/lib/oe/license_finder.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oe/license_finder.py b/meta/lib/oe/license_finder.py
index 16f5d7c94c..4f2bb661fd 100644
--- a/meta/lib/oe/license_finder.py
+++ b/meta/lib/oe/license_finder.py
@@ -120,14 +120,34 @@ def _crunch_license(licfile):
     return md5val
 
 
-def find_license_files(srctree, first_only=False):
+def find_license_files(srctree, first_only=False, bottom=""):
     """
     Search srctree for files that look like they could be licenses.
     If first_only is True, only return the first file found.
+    If bottom is not empty, start at bottom and continue upwards to the top.
     """
     licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
     skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go", ".sh")
     licfiles = []
+    if bottom:
+        srcdir = bottom
+        while srcdir.startswith(srctree):
+            files = []
+            with os.scandir(srcdir) as it:
+                for entry in it:
+                    if entry.is_file():
+                        files.append(entry.name)
+            for name in sorted(files):
+                if name.endswith(skip_extensions):
+                    continue
+                for spec in licspecs:
+                    if fnmatch.fnmatch(name, spec):
+                        licfiles.append(os.path.join(srcdir, name))
+                        if first_only:
+                            return licfiles
+            srcdir = os.path.dirname(srcdir)
+        return licfiles
+
     for root, dirs, files in os.walk(srctree):
         # Sort files so that LICENSE is before LICENSE.subcomponent, which is
         # meaningful if first_only is set.
@@ -177,3 +197,8 @@ def find_licenses(srctree, d, first_only=False, extra_hashes={}):
     # FIXME should we grab at least one source file with a license header and add that too?
 
     return licenses
+
+
+def find_licenses_up(srcdir, topdir, d, first_only=False, extra_hashes={}):
+    licfiles = find_license_files(topdir, first_only, srcdir)
+    return match_licenses(licfiles, topdir, d, extra_hashes)
-- 
2.39.5

[PATCH 2/3] go-mod-update-modules.bbclass: Update license finding

[Christian Lindeberg]

From: Christian Lindeberg <christian.lindeberg@axis.com>

Use ${GO_INSTALL} when listing package dependencies.

Look for licenses for each package dependency continuing upwards, but not
above the module root, until some license is found.

Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com>
---
 .../go-mod-update-modules.bbclass             | 30 ++++++++-----------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/meta/classes-recipe/go-mod-update-modules.bbclass b/meta/classes-recipe/go-mod-update-modules.bbclass
index 5fccd0bb0d..0083588a25 100644
--- a/meta/classes-recipe/go-mod-update-modules.bbclass
+++ b/meta/classes-recipe/go-mod-update-modules.bbclass
@@ -15,7 +15,7 @@ do_update_modules[network] = "1"
 python do_update_modules() {
     import subprocess, tempfile, json, re, urllib.parse
     from oe.license import tidy_licenses
-    from oe.license_finder import find_licenses
+    from oe.license_finder import find_licenses_up
 
     def unescape_path(path):
         """Unescape capital letters using exclamation points."""
@@ -47,12 +47,10 @@ python do_update_modules() {
 """
 
         env = dict(os.environ, GOMODCACHE=mod_cache_dir)
-
         source = d.expand("${UNPACKDIR}/${GO_SRCURI_DESTSUFFIX}")
-        output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True)
-        go_mod = json.loads(output)
-
-        output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", f"{go_mod['Module']['Path']}/..."), cwd=source, env=env, text=True)
+        go_install = d.getVar("GO_INSTALL").split()
+        output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", *go_install),
+                                         cwd=source, env=env, text=True)
 
         #
         # Licenses
@@ -66,26 +64,22 @@ python do_update_modules() {
         # Very frustrating that the json parser in python can't repeatedly
         # parse from a stream.
         pkgs = json.loads('[' + output.replace('}\n{', '},\n{') + ']')
+
         # Collect licenses for the dependencies.
-        licenses = set()
-        lic_files_chksum = []
         lic_files = {}
-
         for pkg in pkgs:
-            mod = pkg.get('Module', None)
-            if not mod or mod.get('Main', False):
-                continue
-
-            mod_dir = mod['Dir']
-
-            if not mod_dir.startswith(mod_cache_dir):
+            pkg_dir = pkg['Dir']
+            if not pkg_dir.startswith(mod_cache_dir):
                 continue
 
+            mod_dir = pkg['Module']['Dir']
             path = os.path.relpath(mod_dir, mod_cache_dir)
 
-            for license_name, license_file, license_md5 in find_licenses(mod['Dir'], d, first_only=True, extra_hashes=extra_hashes):
-                lic_files[os.path.join(path, license_file)] = (license_name, license_md5)
+            for name, file, md5 in find_licenses_up(pkg_dir, mod_dir, d, first_only=True, extra_hashes=extra_hashes):
+                lic_files[os.path.join(path, file)] = (name, md5)
 
+        licenses = set()
+        lic_files_chksum = []
         for lic_file in lic_files:
             license_name, license_md5 = lic_files[lic_file]
             if license_name == "Unknown":
-- 
2.39.5

[PATCH 3/3] recipetool/create_go: Tidy up a bit

[Christian Lindeberg]

From: Christian Lindeberg <christian.lindeberg@axis.com>

There is no need for a temporary Go module cache after moving generation
of module dependency include files to go-mod-update-modules.bbclass.

Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com>
---
 scripts/lib/recipetool/create_go.py | 70 ++++++++++++++---------------
 1 file changed, 34 insertions(+), 36 deletions(-)

diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
index 4b1fa39d13..1b2e5a03d5 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -32,7 +32,6 @@ def tinfoil_init(instance):
     tinfoil = instance
 
 
-
 class GoRecipeHandler(RecipeHandler):
     """Class to handle the go recipe creation"""
 
@@ -85,41 +84,40 @@ class GoRecipeHandler(RecipeHandler):
         classes.append("go-mod-update-modules")
         extravalues["run_tasks"] = "update_modules"
 
-        with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir:
-            env = dict(os.environ)
-            env["PATH"] += f":{go_bindir}"
-            env['GOMODCACHE'] = tmp_mod_dir
-
-            stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True)
-            go_mod = json.loads(stdout)
-            go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
-
-            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
-            extravalues.setdefault('extrafiles', {})
-
-            # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
-            basename = "{pn}-licenses.inc"
-            filename = os.path.join(localfilesdir, basename)
-            with open(filename, "w") as f:
-                f.write("# FROM RECIPETOOL\n")
-            extravalues['extrafiles'][f"../{basename}"] = filename
-
-            basename = "{pn}-go-mods.inc"
-            filename = os.path.join(localfilesdir, basename)
-            with open(filename, "w") as f:
-                f.write("# FROM RECIPETOOL\n")
-            extravalues['extrafiles'][f"../{basename}"] = filename
-
-            # Do generic license handling
-            d = bb.data.createCopy(tinfoil.config_data)
-            handle_license_vars(srctree, lines_before, handled, extravalues, d)
-            self.__rewrite_lic_vars(lines_before)
-
-            self.__rewrite_src_uri(lines_before)
-
-            lines_before.append('require ${BPN}-licenses.inc')
-            lines_before.append('require ${BPN}-go-mods.inc')
-            lines_before.append(f'GO_IMPORT = "{go_import}"')
+        env = dict(os.environ)
+        env["PATH"] += f":{go_bindir}"
+
+        stdout = subprocess.check_output(("go", "mod", "edit", "-json"),
+                                         cwd=srctree, env=env, text=True)
+        go_mod = json.loads(stdout)
+        go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
+
+        localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+        extravalues.setdefault('extrafiles', {})
+
+        # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
+        basename = "{pn}-licenses.inc"
+        filename = os.path.join(localfilesdir, basename)
+        with open(filename, "w") as f:
+            f.write("# FROM RECIPETOOL\n")
+        extravalues['extrafiles'][f"../{basename}"] = filename
+
+        basename = "{pn}-go-mods.inc"
+        filename = os.path.join(localfilesdir, basename)
+        with open(filename, "w") as f:
+            f.write("# FROM RECIPETOOL\n")
+        extravalues['extrafiles'][f"../{basename}"] = filename
+
+        # Do generic license handling
+        d = bb.data.createCopy(tinfoil.config_data)
+        handle_license_vars(srctree, lines_before, handled, extravalues, d)
+        self.__rewrite_lic_vars(lines_before)
+
+        self.__rewrite_src_uri(lines_before)
+
+        lines_before.append('require ${BPN}-licenses.inc')
+        lines_before.append('require ${BPN}-go-mods.inc')
+        lines_before.append(f'GO_IMPORT = "{go_import}"')
 
     def __update_lines_before(self, updated, newlines, lines_before):
         if updated:
-- 
2.39.5