Re: [PATCH 2/4] tests/functional: Support tests that require a runner

["Alex Bennée" <alex.bennee@linaro.org>]

Gustavo Romero <gustavo.romero@linaro.org> writes:

> Add support for running tests that require a specific runner.
>
> The test is specified via a tuple (name, runner, protocol), where name
> is the test name as found in the tests/functional directory without the
> 'test_' prefix and the .py extension, runner is an array containing the
> runner and any arguments required by the runner, and protocol is
> the test protocol used by Meson to determine whether the test passed or
> failed.
>
> The test tuples are added to arrays that follow the current naming
> logic but with the suffix '_with_runner' appended to their names. In
> Meson it's not easy to select an element in an array at runtime based on
> its type, so it's simpler to have a new array for these new test types
> than use the current ones from the tests that don't require a runner,
> and so avoid mixing strings and tuples in the same array.
>
> Currently there is only one runner, the GDB runner, but more runners can
> be defined and associated to a test via the tuple.
>
> Signed-off-by: Gustavo Romero <gustavo.romero@linaro.org>
> ---
>  meson.build                  |  4 +++
>  tests/functional/meson.build | 62 ++++++++++++++++++++++++++++++++++++
>  2 files changed, 66 insertions(+)
>
> diff --git a/meson.build b/meson.build
> index 50c774a195..8d482f0809 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -75,6 +75,10 @@ have_user = have_linux_user or have_bsd_user
>  
>  sh = find_program('sh')
>  python = import('python').find_installation()
> +# Meson python.get_path() on 'purelib' or 'platlib' doesn't properly return the
> +# site-packages dir in pyvenv, so it is built manually.
> +python_ver = python.language_version()
> +python_site_packages = meson.build_root() / 'pyvenv/lib/python' + python_ver / 'site-packages'
>  
>  cc = meson.get_compiler('c')
>  all_languages = ['c']
> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
> index 311c6f1806..1f70b70fd4 100644
> --- a/tests/functional/meson.build
> +++ b/tests/functional/meson.build
> @@ -349,6 +349,23 @@ tests_xtensa_system_thorough = [
>    'xtensa_replay',
>  ]
>  
> +# Tests that require a specific runner.
> +gdb = find_program('gdb-multiarch', required: false)
> +if gdb.found()
> +    gdb_runner_script =  meson.project_source_root() + '/tests/guest-debug/run-test.py'
> +    gdb_runner = [gdb_runner_script, '--gdb', gdb, '--test']
> +
> +    # A test with a runner is a tuple (name, runner, protocol).
> +    # The tests must be elements of an array named like:
> +    #
> +    # test_<arch>_<mode=[system|linuxuser|bsduser]>_<speed=[quick|thorough]>_with_runner = [
> +    #      ['test0', gdb_runner, 'exitcode'],
> +    #      ...
> +    # ]
> +else
> +    message('GDB multiarch not found, skipping functional tests that rely on it.')
> +endif
> +

We already have a --gdb option in configure which sets GDB= for
check-tcg - we should use the same configuration for these tests. You
may need to convert the configure probe to a meson-option.

>  precache_all = []
>  foreach speed : ['quick', 'thorough']
>    foreach dir : target_dirs
> @@ -372,9 +389,11 @@ foreach speed : ['quick', 'thorough']
>        suites = ['func-quick', 'func-' + target_base]
>        target_tests = get_variable('tests_' + target_base + '_' + sysmode + '_quick', []) \
>                       + get_variable('tests_generic_' + sysmode)
> +      target_tests_r = get_variable('tests_' + target_base + '_' + sysmode + '_quick_with_runner', [])
>      else
>        suites = ['func-' + speed, 'func-' + target_base + '-' + speed, speed]
>        target_tests = get_variable('tests_' + target_base + '_' + sysmode + '_' + speed, [])
> +      target_tests_r = get_variable('tests_' + target_base + '_' + sysmode + '_' + speed + '_with_runner', [])
>      endif
>  
>      test_deps = [roms, keymap_targets]
> @@ -423,6 +442,49 @@ foreach speed : ['quick', 'thorough']
>             priority: test_timeouts.get(test, 90),
>             suite: suites)
>      endforeach
> +
> +    # Prepare tests that require a specific runner.
> +    foreach test : target_tests_r
> +      testname = '@0@-@1@'.format(target_base, test[0])
> +      testfile = 'test_' + test[0] + '.py'
> +      testpath = meson.current_source_dir() / testfile
> +      teststamp = testname + '.tstamp'
> +      testrunner  = test[1]
> +      testproto = test[2]
> +
> +      test_precache_env = environment()
> +      test_precache_env.set('QEMU_TEST_PRECACHE', meson.current_build_dir() / teststamp)
> +      # python_site_packages, i.e., site packages from Python in pyvenv, is
> +      # added to PYTHONPATH because some runners can run a program that has its
> +      # own Python hooks that, by its turn, will search for modules based on
> +      # PYTHONPATH independently of the Python used by the runner, like, for
> +      # example, GDB using libpython.
> +      test_precache_env.set('PYTHONPATH', meson.project_source_root() / 'python' +
> +                                          ':' + meson.current_source_dir() +
> +                                          ':' + python_site_packages)
> +      precache = custom_target('func-precache-' + testname,
> +                               output: teststamp,
> +                               command: [testrunner, testpath],
> +                               depend_files: files(testpath),
> +                               build_by_default: false,
> +                               env: test_precache_env)
> +      precache_all += precache
> +
> +      # See comment above about python_site_packages in test_precache_env.
> +      # Don't append to test_env since it will affect previous uses of it.
> +      test_r_env = test_env
> +      test_r_env.append('PYTHONPATH', python_site_packages)
> +
> +      test('func-' + testname,
> +           python,
> +           depends: [test_deps, test_emulator, emulator_modules, plugin_modules],
> +           env: test_r_env,
> +           args: [testrunner, testpath],
> +           protocol: testproto,
> +           timeout: test_timeouts.get(test[0], 90),
> +           priority: test_timeouts.get(test[0], 90),
> +           suite: suites)
> +    endforeach
>    endforeach
>  endforeach

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro