Re: [OE-core][PATCH] oeqa/selftest/externalsrc: add test for srctree_hash_files

[Alexandre Belloni <alexandre.belloni@bootlin.com>]

Hello,

The submodule test fails on the autobuilders:

2022-12-06 19:23:44,209 - oe-selftest - INFO - ======================================================================
2022-12-06 19:23:44,209 - oe-selftest - INFO - FAIL: externalsrc.ExternalSrc.test_externalsrc_srctree_hash_files (subunit.RemotedTestCase)
2022-12-06 19:23:44,209 - oe-selftest - INFO - ----------------------------------------------------------------------
2022-12-06 19:23:44,209 - oe-selftest - INFO - testtools.testresult.real._StringException: Traceback (most recent call last):
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/selftest/cases/externalsrc.py", line 34, in test_externalsrc_srctree_hash_files
    self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC")
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/utils/commands.py", line 288, in get_bb_var
    return get_bb_vars([var], target, postconfig)[var]
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/utils/commands.py", line 250, in get_bb_vars
    bbenv = get_bb_env(target, postconfig=postconfig)
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/utils/commands.py", line 244, in get_bb_env
    return bitbake("-e %s" % target, postconfig=postconfig).output
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/utils/commands.py", line 236, in bitbake
    return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options)
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/lib/oeqa/utils/commands.py", line 214, in runCmd
    raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
AssertionError: Command 'bitbake  -e git-submodule-test' returned non-zero exit status 1:
NOTE: Reconnecting to bitbake server...
Loading cache...done.
Loaded 0 entries from dependency cache.
Parsing recipes...ERROR: ExpansionError during parsing /home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040/meta-selftest/recipes-test/git-submodule-test/git-submodule-test.bb
Traceback (most recent call last):
  File "Var <do_compile[file-checksums]>", line 1, in <module>
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/meta/classes/externalsrc.bbclass", line 234, in srctree_hash_files(d=<bb.data_smart.DataSmart object at 0x7f8809b5c310>, srcdir=None):
                 if os.path.exists(".gitmodules"):
    >                submodule_helper = subprocess.check_output(["git", "config", "--file", ".gitmodules", "--get-regexp", "path"], cwd=s_dir, env=env).decode("utf-8")
                     for line in submodule_helper.splitlines():
  File "/usr/lib64/python3.10/subprocess.py", line 421, in check_output(timeout=None, *popenargs=(['git', 'config', '--file', '.gitmodules', '--get-regexp', 'path'],), **kwargs={'cwd': '/tmp/externalsrccxnk_jyq', 'env': {'SHELL': '/bin/bash', 'BB_ENV_PASSTHROUGH_ADDITIONS': 'ALL_PROXY BBPATH_EXTRA BB_LOGCONFIG BB_NO_NETWORK BB_NUMBER_THREADS BB_SETSCENE_ENFORCE BB_SRCREV_POLICY DISTRO FTPS_PROXY FTP_PROXY GIT_PROXY_COMMAND HTTPS_PROXY HTTP_PROXY MACHINE NO_PROXY PARALLEL_MAKE SCREENDIR SDKMACHINE SOCKS5_PASSWD SOCKS5_USER SSH_AGENT_PID SSH_AUTH_SOCK STAMPS_DIR TCLIBC TCMODE all_proxy ftp_proxy ftps_proxy http_proxy https_proxy no_proxy ', 'BB_LOGCONFIG': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040/../bitbake/contrib/autobuilderlog.json', 'PWD': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040', 'LOGNAME': 'pokybuild', 'ftp_proxy': 'http://proxy.yocto.io:5187/', 'HOME': '/home/pokybuild', 'BBPATH': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040', 'USER': 'pokybuild', 'FTP_PROXY': 'http://proxy.yocto.io:5187/', 'PATH': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/scripts:/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/bin:/home/pokybuild/.local/bin:/home/pokybuild/bin:/usr/lib64/ccache:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin', 'LC_ALL': 'en_US.UTF-8', 'GIT_INDEX_FILE': '/tmp/oe-devtool-index18lc6e8_'}}):
     
    >    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
                    **kwargs).stdout
  File "/usr/lib64/python3.10/subprocess.py", line 526, in run(input=None, capture_output=False, timeout=None, check=True, *popenargs=(['git', 'config', '--file', '.gitmodules', '--get-regexp', 'path'],), **kwargs={'stdout': -1, 'cwd': '/tmp/externalsrccxnk_jyq', 'env': {'SHELL': '/bin/bash', 'BB_ENV_PASSTHROUGH_ADDITIONS': 'ALL_PROXY BBPATH_EXTRA BB_LOGCONFIG BB_NO_NETWORK BB_NUMBER_THREADS BB_SETSCENE_ENFORCE BB_SRCREV_POLICY DISTRO FTPS_PROXY FTP_PROXY GIT_PROXY_COMMAND HTTPS_PROXY HTTP_PROXY MACHINE NO_PROXY PARALLEL_MAKE SCREENDIR SDKMACHINE SOCKS5_PASSWD SOCKS5_USER SSH_AGENT_PID SSH_AUTH_SOCK STAMPS_DIR TCLIBC TCMODE all_proxy ftp_proxy ftps_proxy http_proxy https_proxy no_proxy ', 'BB_LOGCONFIG': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040/../bitbake/contrib/autobuilderlog.json', 'PWD': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040', 'LOGNAME': 'pokybuild', 'ftp_proxy': 'http://proxy.yocto.io:5187/', 'HOME': '/home/pokybuild', 'BBPATH': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-3944040', 'USER': 'pokybuild', 'FTP_PROXY': 'http://proxy.yocto.io:5187/', 'PATH': '/home/pokybuild/yocto-worker/oe-selftest-fedora/build/scripts:/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/bin:/home/pokybuild/.local/bin:/home/pokybuild/bin:/usr/lib64/ccache:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin', 'LC_ALL': 'en_US.UTF-8', 'GIT_INDEX_FILE': '/tmp/oe-devtool-index18lc6e8_'}}):
             if check and retcode:
    >            raise CalledProcessError(retcode, process.args,
                                          output=stdout, stderr=stderr)
bb.data_smart.ExpansionError: Failure expanding variable do_compile[file-checksums], expression was ${@srctree_hash_files(d)} which triggered exception CalledProcessError: Command '['git', 'config', '--file', '.gitmodules', '--get-regexp', 'path']' returned non-zero exit status 1.
The variable dependency chain for the failure is: do_compile[file-checksums]
ERROR: Parsing halted due to errors, see error messages above
Summary: There were 2 ERROR messages, returning a non-zero exit code.


https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/4443/steps/14/logs/stdio

On 05/12/2022 22:38:15+0100, Peter Marko wrote:
> Signed-off-by: Peter Marko <peter.marko@siemens.com>
> ---
>  meta/lib/oeqa/selftest/cases/externalsrc.py | 44 +++++++++++++++++++++
>  1 file changed, 44 insertions(+)
>  create mode 100644 meta/lib/oeqa/selftest/cases/externalsrc.py
> 
> diff --git a/meta/lib/oeqa/selftest/cases/externalsrc.py b/meta/lib/oeqa/selftest/cases/externalsrc.py
> new file mode 100644
> index 0000000000..1d800dc82c
> --- /dev/null
> +++ b/meta/lib/oeqa/selftest/cases/externalsrc.py
> @@ -0,0 +1,44 @@
> +#
> +# Copyright OpenEmbedded Contributors
> +#
> +# SPDX-License-Identifier: MIT
> +#
> +
> +import os
> +import shutil
> +import tempfile
> +
> +from oeqa.selftest.case import OESelftestTestCase
> +from oeqa.utils.commands import get_bb_var, runCmd
> +
> +class ExternalSrc(OESelftestTestCase):
> +    # test that srctree_hash_files does not crash
> +    # we should be actually checking do_compile[file-checksums] but oeqa currently does not support it
> +    #     so we check only that a recipe with externalsrc can be parsed
> +    def test_externalsrc_srctree_hash_files(self):
> +        test_recipe = "git-submodule-test"
> +        git_url = "git://git.yoctoproject.org/git-submodule-test"
> +        externalsrc_dir = tempfile.TemporaryDirectory(prefix="externalsrc").name
> +
> +        self.write_config(
> +            """
> +INHERIT += "externalsrc"
> +EXTERNALSRC:pn-%s = "%s"
> +""" % (test_recipe, externalsrc_dir)
> +        )
> +
> +        # test with git without submodules
> +        runCmd('git clone %s %s' % (git_url, externalsrc_dir))
> +        os.unlink(externalsrc_dir + "/.gitmodules")
> +        open(".gitmodules", 'w').close()  # local file .gitmodules in cwd should not affect externalsrc parsing
> +        self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC")
> +        os.unlink(".gitmodules")
> +
> +        # test with git with submodules
> +        runCmd('git checkout .gitmodules', cwd=externalsrc_dir)
> +        runCmd('git submodule update --init --recursive', cwd=externalsrc_dir)
> +        self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC")
> +
> +        # test without git
> +        shutil.rmtree(os.path.join(externalsrc_dir, ".git"))
> +        self.assertEqual(get_bb_var("S", test_recipe), externalsrc_dir, msg = "S does not equal to EXTERNALSRC")
> -- 
> 2.30.2
> 

> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#174304): https://lists.openembedded.org/g/openembedded-core/message/174304
> Mute This Topic: https://lists.openembedded.org/mt/95479820/3617179
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alexandre.belloni@bootlin.com]
> -=-=-=-=-=-=-=-=-=-=-=-
> 


-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com