[Qemu-devel] [RFC PATCH] Acceptance tests: run generic tests on all built targets

[Cleber Rosa <crosa@redhat.com>]

The intention here is to discuss the validity of running the the
acceptance tests are not depedent on target specific functionality on
all built targets.

Subtle but important questions that this topic brings:

 1) Should the QEMU binary target provide, as much as possible,
    a similar experience across targets, or should upper layer
    code deal with it?

 2) Are those tests exercising the same exact features and
    implementation, which just happen to be linked to various
    different binaries?  Or is the simple fact that they are
    integrated into different code worth testing?

 3) Should the default target match the host target?  Or the
    first binary found in the build tree?  Or something else?

An example of a Travis CI job based on this can be seen here:

 https://travis-ci.org/clebergnu/qemu/jobs/548905146

Signed-off-by: Cleber Rosa <crosa@redhat.com>
---
 .travis.yml                               |  2 +-
 tests/Makefile.include                    | 18 +++++++++++++++++-
 tests/acceptance/avocado_qemu/__init__.py |  9 +++++++++
 tests/acceptance/cpu_queries.py           |  3 +++
 tests/acceptance/migration.py             |  4 +++-
 5 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index aeb9b211cd..310febb866 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -232,7 +232,7 @@ matrix:
     # Acceptance (Functional) tests
     - env:
         - CONFIG="--python=/usr/bin/python3 --target-list=x86_64-softmmu,mips-softmmu,mips64el-softmmu,aarch64-softmmu,arm-softmmu,s390x-softmmu,alpha-softmmu"
-        - TEST_CMD="make check-acceptance"
+        - TEST_CMD="make check-acceptance-all"
       after_failure:
         - cat tests/results/latest/job.log
       addons:
diff --git a/tests/Makefile.include b/tests/Makefile.include
index db750dd6d0..34126167a5 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -1154,7 +1154,23 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR)
             --filter-by-tags-include-empty --filter-by-tags-include-empty-key \
             $(AVOCADO_TAGS) \
             --failfast=on $(SRC_PATH)/tests/acceptance, \
-            "AVOCADO", "tests/acceptance")
+            "AVOCADO", "tests/acceptance (target arch based on test)")
+
+# Allows one to run tests that are generic (independent of target)
+# using a given target
+check-acceptance-generic-on-%: check-venv $(TESTS_RESULTS_DIR)
+	$(call quiet-command, \
+            $(TESTS_VENV_DIR)/bin/python -m avocado \
+            --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
+            --filter-by-tags-include-empty --filter-by-tags-include-empty-key \
+            --filter-by-tags=-arch -p arch=$* \
+            -p add-qtest=yes -p set-arm-aarch64-machine=yes \
+            --failfast=on $(SRC_PATH)/tests/acceptance, \
+            "AVOCADO", "tests/acceptance (target arch set to $*)")
+
+check-acceptance-generic: $(patsubst %-softmmu,check-acceptance-generic-on-%, $(filter %-softmmu,$(TARGET_DIRS)))
+
+check-acceptance-all: check-acceptance check-acceptance-generic
 
 # Consolidated targets
 
diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py
index 2b236a1cf0..7a47f0d514 100644
--- a/tests/acceptance/avocado_qemu/__init__.py
+++ b/tests/acceptance/avocado_qemu/__init__.py
@@ -69,6 +69,15 @@ class Test(avocado.Test):
         vm = QEMUMachine(self.qemu_bin)
         if args:
             vm.add_args(*args)
+        if self.params.get('add-qtest', default=False):
+            # mips and sh4 targets require either a bios or a kernel or
+            # qtest enabled to not abort right at the commad line
+            if self.arch in ('mips', 'mipsel', 'mips64', 'mips64el', 'sh4'):
+                vm.add_args('-accel', 'qtest')
+        if self.params.get('set-arm-aarch64-machine', default=False):
+            # arm and aarch64 require a machine type to be set
+            if self.arch in ('arm', 'aarch64'):
+                vm.set_machine('virt')
         return vm
 
     @property
diff --git a/tests/acceptance/cpu_queries.py b/tests/acceptance/cpu_queries.py
index e71edec39f..af47d2795a 100644
--- a/tests/acceptance/cpu_queries.py
+++ b/tests/acceptance/cpu_queries.py
@@ -18,6 +18,9 @@ class QueryCPUModelExpansion(Test):
     """
 
     def test(self):
+        """
+        :avocado: tags=arch:x86_64
+        """
         self.vm.set_machine('none')
         self.vm.add_args('-S')
         self.vm.launch()
diff --git a/tests/acceptance/migration.py b/tests/acceptance/migration.py
index 6115cf6c24..c4ed87cd98 100644
--- a/tests/acceptance/migration.py
+++ b/tests/acceptance/migration.py
@@ -33,8 +33,10 @@ class Migration(Test):
             self.cancel('Failed to find a free port')
         return port
 
-
     def test_migration_with_tcp_localhost(self):
+        blacklist_targets = ["arm", "ppc64", "sh4", "s390x"]
+        if self.arch in blacklist_targets:
+            self.cancel("Test failing on targets: %s" % ", ".join(blacklist_targets))
         source_vm = self.get_vm()
         dest_uri = 'tcp:localhost:%u' % self._get_free_port()
         dest_vm = self.get_vm('-incoming', dest_uri)
-- 
2.21.0