Re: [PATCH v4 08/10] acpi/tests/avocado/bits: add acpi and smbios avocado tests that uses biosbits

[Ani Sinha <ani@anisinha.ca>]

On Tue, Oct 18, 2022 at 9:10 PM Alex Bennée <alex.bennee@linaro.org> wrote:
>
>
> Ani Sinha <ani@anisinha.ca> writes:
>
> > This introduces QEMU acpi/smbios biosbits avocado test which is run
> > from within the python virtual environment. When the bits tests are run, bits
> > binaries are downloaded from an external repo/location, bios bits iso is
> > regenerated containing the acpi/smbios bits tests that are maintained as a part
> > of the QEMU source under tests/avocado/acpi-bits/bits-test . When the VM is
> > spawned with the iso, it runs the tests in batch mode and the results are pushed
> > out from the VM to the test machine where they are analyzed by this script and
> > pass/fail results are reported.
> >
> > Cc: Daniel P. Berrangé <berrange@redhat.com>
> > Cc: Paolo Bonzini <pbonzini@redhat.com>
> > Cc: Maydell Peter <peter.maydell@linaro.org>
> > Cc: John Snow <jsnow@redhat.com>
> > Cc: Thomas Huth <thuth@redhat.com>
> > Cc: Alex Bennée <alex.bennee@linaro.org>
> > Cc: Igor Mammedov <imammedo@redhat.com>
> > Cc: Michael Tsirkin <mst@redhat.com>
> > Signed-off-by: Ani Sinha <ani@anisinha.ca>
> > ---
> >  tests/avocado/acpi-bits.py | 359 +++++++++++++++++++++++++++++++++++++
> >  1 file changed, 359 insertions(+)
> >  create mode 100644 tests/avocado/acpi-bits.py
> >
> > diff --git a/tests/avocado/acpi-bits.py b/tests/avocado/acpi-bits.py
> > new file mode 100644
> > index 0000000000..009c7f5327
> > --- /dev/null
> > +++ b/tests/avocado/acpi-bits.py
> > @@ -0,0 +1,359 @@
> > +#!/usr/bin/env python3
> > +# group: rw quick
> > +# Exercize QEMU generated ACPI/SMBIOS tables using biosbits,
> > +# https://biosbits.org/
> > +#
> > +# This program is free software; you can redistribute it and/or modify
> > +# it under the terms of the GNU General Public License as published by
> > +# the Free Software Foundation; either version 2 of the License, or
> > +# (at your option) any later version.
> > +#
> > +# This program is distributed in the hope that it will be useful,
> > +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > +# GNU General Public License for more details.
> > +#
> > +# You should have received a copy of the GNU General Public License
> > +# along with this program.  If not, see <http://www.gnu.org/licenses/>.
> > +#
> > +#
> > +# Author:
> > +#  Ani Sinha <ani@anisinha.ca>
> > +
> > +# pylint: disable=invalid-name
> > +# pylint: disable=consider-using-f-string
> > +
> > +"""
> > +This is QEMU ACPI/SMBIOS avocado tests using biosbits.
> > +Biosbits is available originally at https://biosbits.org/.
> > +This test uses a fork of the upstream bits and has numerous fixes
> > +including an upgraded acpica. The fork is located here:
> > +https://gitlab.com/qemu-project/biosbits-bits .
> > +"""
> > +
> > +import logging
> > +import os
> > +import re
> > +import shutil
> > +import subprocess
> > +import tarfile
> > +import tempfile
> > +import time
> > +import zipfile
> > +from typing import (
> > +    List,
> > +    Optional,
> > +    Sequence,
> > +)
> > +from qemu.machine import QEMUMachine
> > +from avocado import skipIf
> > +from avocado_qemu import QemuBaseTest
> > +
> > +def _print_log(log):
> > +    print('\nlogs from biosbits follows:')
> > +    print('==========================================\n')
> > +    print(log)
> > +    print('==========================================\n')
> > +
> > +class QEMUBitsMachine(QEMUMachine): # pylint: disable=too-few-public-methods
> > +    """
> > +    A QEMU VM, with isa-debugcon enabled and bits iso passed
> > +    using -cdrom to QEMU commandline.
> > +
> > +    """
> > +    def __init__(self,
> > +                 binary: str,
> > +                 args: Sequence[str] = (),
> > +                 wrapper: Sequence[str] = (),
> > +                 name: Optional[str] = None,
> > +                 base_temp_dir: str = "/var/tmp",
> > +                 debugcon_log: str = "debugcon-log.txt",
> > +                 debugcon_addr: str = "0x403",
> > +                 sock_dir: Optional[str] = None,
> > +                 qmp_timer: Optional[float] = None):
> > +        # pylint: disable=too-many-arguments
> > +
> > +        if name is None:
> > +            name = "qemu-bits-%d" % os.getpid()
> > +        if sock_dir is None:
> > +            sock_dir = base_temp_dir
> > +        super().__init__(binary, args, wrapper=wrapper, name=name,
> > +                         base_temp_dir=base_temp_dir,
> > +                         sock_dir=sock_dir, qmp_timer=qmp_timer)
> > +        self.debugcon_log = debugcon_log
> > +        self.debugcon_addr = debugcon_addr
> > +        self.base_temp_dir = base_temp_dir
> > +
> > +    @property
> > +    def _base_args(self) -> List[str]:
> > +        args = super()._base_args
> > +        args.extend([
> > +            '-chardev',
> > +            'file,path=%s,id=debugcon' %os.path.join(self.base_temp_dir,
> > +                                                     self.debugcon_log),
> > +            '-device',
> > +            'isa-debugcon,iobase=%s,chardev=debugcon' %self.debugcon_addr,
> > +        ])
> > +        return args
> > +
> > +    def base_args(self):
> > +        """return the base argument to QEMU binary"""
> > +        return self._base_args
> > +
> > +@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
> > +class AcpiBitsTest(QemuBaseTest): #pylint: disable=too-many-instance-attributes
> > +    """
> > +    ACPI and SMBIOS tests using biosbits.
> > +
> > +    :avocado: tags=arch:x86_64
> > +    :avocado: tags=acpi
> > +
> > +    """
> > +    def __init__(self, *args, **kwargs):
> > +        super().__init__(*args, **kwargs)
> > +        self._vm = None
> > +        self._workDir = None
> > +        self._baseDir = None
> > +
> > +        # following are some standard configuration constants
> > +        self._bitsInternalVer = 2020
> > +        self._bitsCommitHash = 'b972c69d' # commit hash must match
> > +                                          # the artifact tag below
> > +        self._bitsTag = "qemu-bits-09272022" # this is the latest bits
> > +                                             # release as of today.
> > +        self._bitsArtSHA1Hash = '6915ad4781de0d80d1a099438a4cb4bd9e12df70'
> > +        self._bitsArtURL = ("https://gitlab.com/qemu-project/"
> > +                            "biosbits-bits/-/jobs/artifacts/%s/"
> > +                            "download?job=qemu-bits-build" %self._bitsTag)
> > +        self._debugcon_addr = '0x403'
> > +        self._debugcon_log = 'debugcon-log.txt'
> > +        logging.basicConfig(level=logging.INFO)
> > +
> > +    def copy_bits_config(self):
> > +        """ copies the bios bits config file into bits.
> > +        """
> > +        config_file = 'bits-cfg.txt'
> > +        bits_config_dir = os.path.join(self._baseDir, 'acpi-bits',
> > +                                       'bits-config')
> > +        target_config_dir = os.path.join(self._workDir,
> > +                                         'bits-%d' %self._bitsInternalVer,
> > +                                         'boot')
> > +        self.assertTrue(os.path.exists(bits_config_dir))
> > +        self.assertTrue(os.path.exists(target_config_dir))
> > +        self.assertTrue(os.access(os.path.join(bits_config_dir,
> > +                                               config_file), os.R_OK))
> > +        shutil.copy2(os.path.join(bits_config_dir, config_file),
> > +                     target_config_dir)
> > +        logging.info('copied config file %s to %s',
> > +                     config_file, target_config_dir)
> > +
> > +    def copy_test_scripts(self):
> > +        """copies the python test scripts into bits. """
> > +
> > +        bits_test_dir = os.path.join(self._baseDir, 'acpi-bits',
> > +                                     'bits-tests')
> > +        target_test_dir = os.path.join(self._workDir,
> > +                                       'bits-%d' %self._bitsInternalVer,
> > +                                       'boot', 'python')
> > +
> > +        self.assertTrue(os.path.exists(bits_test_dir))
> > +        self.assertTrue(os.path.exists(target_test_dir))
> > +
> > +        for filename in os.listdir(bits_test_dir):
> > +            if os.path.isfile(os.path.join(bits_test_dir, filename)) and \
> > +               filename.endswith('.py2'):
> > +                # all test scripts are named with extension .py2 so that
> > +                # avocado does not try to load them. These scripts are
> > +                # written for python 2.7 not python 3 and hence if avocado
> > +                # loaded them, it would complain about python 3 specific
> > +                # syntaxes.
> > +                newfilename = os.path.splitext(filename)[0] + '.py'
> > +                shutil.copy2(os.path.join(bits_test_dir, filename),
> > +                             os.path.join(target_test_dir, newfilename))
> > +                logging.info('copied test file %s to %s',
> > +                             filename, target_test_dir)
> > +
> > +                # now remove the pyc test file if it exists, otherwise the
> > +                # changes in the python test script won't be executed.
> > +                testfile_pyc = os.path.splitext(filename)[0] + '.pyc'
> > +                if os.access(os.path.join(target_test_dir, testfile_pyc),
> > +                             os.F_OK):
> > +                    os.remove(os.path.join(target_test_dir, testfile_pyc))
> > +                    logging.info('removed compiled file %s',
> > +                                 os.path.join(target_test_dir, testfile_pyc))
> > +
> > +    def fix_mkrescue(self, mkrescue):
> > +        """ grub-mkrescue is a bash script with two variables, 'prefix' and
> > +            'libdir'. They must be pointed to the right location so that the
> > +            iso can be generated appropriately. We point the two variables to
> > +            the directory where we have extracted our pre-built bits grub
> > +            tarball.
> > +        """
> > +        grub_x86_64_mods = os.path.join(self._workDir, 'grub-inst-x86_64-efi')
> > +        grub_i386_mods = os.path.join(self._workDir, 'grub-inst')
> > +
> > +        self.assertTrue(os.path.exists(grub_x86_64_mods))
> > +        self.assertTrue(os.path.exists(grub_i386_mods))
> > +
> > +        new_script = ""
> > +        with open(mkrescue, 'r', encoding='utf-8') as filehandle:
> > +            orig_script = filehandle.read()
> > +            new_script = re.sub('(^prefix=)(.*)',
> > +                                r'\1"%s"' %grub_x86_64_mods,
> > +                                orig_script, flags=re.M)
> > +            new_script = re.sub('(^libdir=)(.*)', r'\1"%s/lib"' %grub_i386_mods,
> > +                                new_script, flags=re.M)
> > +
> > +        with open(mkrescue, 'w', encoding='utf-8') as filehandle:
> > +            filehandle.write(new_script)
> > +
> > +    def generate_bits_iso(self):
> > +        """ Uses grub-mkrescue to generate a fresh bits iso with the python
> > +            test scripts
> > +        """
> > +        bits_dir = os.path.join(self._workDir,
> > +                                'bits-%d' %self._bitsInternalVer)
> > +        iso_file = os.path.join(self._workDir,
> > +                                'bits-%d.iso' %self._bitsInternalVer)
> > +        mkrescue_script = os.path.join(self._workDir,
> > +                                       'grub-inst-x86_64-efi', 'bin',
> > +                                       'grub-mkrescue')
>
> This seems wrong because it fails to find grub-mkrescue (which on my
> system is in the PATH):
>
>   ➜  which grub-mkrescue
>   /usr/bin/grub-mkrescue
>
> but because it can't find this it never works.

Unfortunately you cannot use grub-mkrescue from your host. It has to
be from the bits binary artifact that is downloaded. Is the download
failing?

>
> --
> Alex Bennée