[PATCH v4 4/4] tests/functional: Use vmstate-static-checker.py to test data from v7.2

[Thomas Huth <thuth@redhat.com>]

From: Thomas Huth <thuth@redhat.com>

We've got this nice vmstate-static-checker.py script that can help to
detect screw-ups in the migration states. Unfortunately, it's currently
only run manually, which can be cumbersome. Let's run it from a functional
test automatically with the reference data from QEMU 7.2, so that we get
at least a basic coverage here. Since the test can fail when the checker
script detects a false positive, mark the test with a skipFlakyTest
decorator for now, so that it is only run when the user also set the
QEMU_TEST_FLAKY_TESTS environment variable.

Signed-off-by: Thomas Huth <thuth@redhat.com>
---
 MAINTAINERS                              |  1 +
 tests/functional/aarch64/meson.build     |  1 +
 tests/functional/generic/test_vmstate.py | 67 ++++++++++++++++++++++++
 tests/functional/m68k/meson.build        |  4 ++
 tests/functional/ppc64/meson.build       |  1 +
 tests/functional/s390x/meson.build       |  4 ++
 tests/functional/x86_64/meson.build      |  3 +-
 7 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100755 tests/functional/generic/test_vmstate.py

diff --git a/MAINTAINERS b/MAINTAINERS
index 9c8bd7b1180..435155d2116 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3604,6 +3604,7 @@ F: migration/
 F: scripts/vmstate-static-checker.py
 F: tests/functional/migration.py
 F: tests/functional/*/*migration.py
+F: tests/functional/generic/test_vmstate.py
 F: tests/functional/x86_64/test_bad_vmstate.py
 F: tests/data/vmstate-static-checker/
 F: tests/qtest/migration/
diff --git a/tests/functional/aarch64/meson.build b/tests/functional/aarch64/meson.build
index 04846c6eb18..5ad52f93e1d 100644
--- a/tests/functional/aarch64/meson.build
+++ b/tests/functional/aarch64/meson.build
@@ -19,6 +19,7 @@ test_aarch64_timeouts = {
 
 tests_aarch64_system_quick = [
   'migration',
+  'vmstate',
 ]
 
 tests_aarch64_system_thorough = [
diff --git a/tests/functional/generic/test_vmstate.py b/tests/functional/generic/test_vmstate.py
new file mode 100755
index 00000000000..387ff542426
--- /dev/null
+++ b/tests/functional/generic/test_vmstate.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+'''This test runs the vmstate-static-checker script with the current QEMU'''
+
+import subprocess
+
+from qemu_test import QemuSystemTest, skipFlakyTest
+
+
+@skipFlakyTest("vmstate-static-checker can produce false positives")
+class VmStateTest(QemuSystemTest):
+    '''
+    This test helps to check whether there are problems between old
+    reference data and the current QEMU
+    '''
+
+    def test_vmstate_7_2(self):
+        '''Check reference data from QEMU v7.2'''
+
+        target_machine = {
+            'aarch64': 'virt-7.2',
+            'm68k': 'virt-7.2',
+            'ppc64': 'pseries-7.2',
+            's390x': 's390-ccw-virtio-7.2',
+            'x86_64': 'pc-q35-7.2',
+        }
+        self.set_machine(target_machine[self.arch])
+
+        # Run QEMU to get the current vmstate json file:
+        dst_json = self.scratch_file('dest.json')
+        self.log.info('Dumping vmstate from %s', self.qemu_bin)
+        cp = subprocess.run([self.qemu_bin, '-nodefaults',
+                             '-M', target_machine[self.arch],
+                             '-dump-vmstate', dst_json],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            text=True, check=True)
+        if cp.stdout:
+            self.log.info('QEMU output: %s', cp.stdout)
+
+        # Check whether the old vmstate json file is still compatible:
+        src_json = self.data_file('..', 'data', 'vmstate-static-checker',
+                                  self.arch,
+                                  target_machine[self.arch] + '.json')
+        self.log.info('Comparing vmstate with %s', src_json)
+        checkerscript = self.data_file('..', '..', 'scripts',
+                                       'vmstate-static-checker.py')
+        cp = subprocess.run([checkerscript, '-s', src_json, '-d', dst_json],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            text=True, check=False)
+        if cp.returncode != 0:
+            self.fail('Running vmstate-static-checker failed:\n' + cp.stdout +
+                      '\nThis either means that there is a migration bug '
+                      'that needs to be fixed, or\nvmstate-static-checker.py '
+                      'needs to be improved (e.g. extend the changed_names\n'
+                      'in case a field has been renamed), or drop the '
+                      'problematic field from\n' + src_json +
+                      '\nin case the script cannot be fixed easily.')
+        if cp.stdout:
+            self.log.warning('vmstate-static-checker output: %s', cp.stdout)
+
+
+if __name__ == '__main__':
+    QemuSystemTest.main()
diff --git a/tests/functional/m68k/meson.build b/tests/functional/m68k/meson.build
index e29044a6d73..679faaf86d6 100644
--- a/tests/functional/m68k/meson.build
+++ b/tests/functional/m68k/meson.build
@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+tests_m68k_system_quick = [
+  'vmstate',
+]
+
 tests_m68k_system_thorough = [
   'mcf5208evb',
   'nextcube',
diff --git a/tests/functional/ppc64/meson.build b/tests/functional/ppc64/meson.build
index 842fe0fc715..1fa0a70f7ed 100644
--- a/tests/functional/ppc64/meson.build
+++ b/tests/functional/ppc64/meson.build
@@ -11,6 +11,7 @@ test_ppc64_timeouts = {
 
 tests_ppc64_system_quick = [
   'migration',
+  'vmstate',
 ]
 
 tests_ppc64_system_thorough = [
diff --git a/tests/functional/s390x/meson.build b/tests/functional/s390x/meson.build
index 030b116039c..70cd36e2913 100644
--- a/tests/functional/s390x/meson.build
+++ b/tests/functional/s390x/meson.build
@@ -4,6 +4,10 @@ test_s390x_timeouts = {
   'ccw_virtio' : 420,
 }
 
+tests_s390x_system_quick = [
+  'vmstate',
+]
+
 tests_s390x_system_thorough = [
   'ccw_virtio',
   'pxelinux',
diff --git a/tests/functional/x86_64/meson.build b/tests/functional/x86_64/meson.build
index ef12ac43b37..967426c30c3 100644
--- a/tests/functional/x86_64/meson.build
+++ b/tests/functional/x86_64/meson.build
@@ -14,10 +14,11 @@ tests_x86_64_system_quick = [
   'cpu_model_versions',
   'cpu_queries',
   'mem_addr_space',
+  'memlock',
   'migration',
   'pc_cpu_hotplug_props',
   'virtio_version',
-  'memlock',
+  'vmstate',
 ]
 
 tests_x86_64_system_thorough = [
-- 
2.51.0