* [PATCH v4 1/5] qemu-iotests: do not buffer the test output
2021-05-03 11:01 [PATCH v4 0/5] qemu-iotests: quality of life improvements Paolo Bonzini
@ 2021-05-03 11:01 ` Paolo Bonzini
2021-05-03 11:01 ` [PATCH v4 2/5] qemu-iotests: allow passing unittest.main arguments to the test scripts Paolo Bonzini
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Paolo Bonzini @ 2021-05-03 11:01 UTC (permalink / raw)
To: qemu-devel; +Cc: eesposit, vsementsov, mreitz
Instead of buffering the test output into a StringIO, patch it on
the fly by wrapping sys.stdout's write method. This can be
done unconditionally, even if using -d, which makes execute_unittest
a bit simpler.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Tested-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
Message-Id: <20210323181928.311862-2-pbonzini@redhat.com>
---
tests/qemu-iotests/240.out | 8 ++--
tests/qemu-iotests/245.out | 8 ++--
tests/qemu-iotests/295.out | 6 +--
tests/qemu-iotests/296.out | 8 ++--
tests/qemu-iotests/iotests.py | 70 ++++++++++++++++++++---------------
5 files changed, 56 insertions(+), 44 deletions(-)
diff --git a/tests/qemu-iotests/240.out b/tests/qemu-iotests/240.out
index e0982831ae..89ed25e506 100644
--- a/tests/qemu-iotests/240.out
+++ b/tests/qemu-iotests/240.out
@@ -15,7 +15,7 @@
{"return": {}}
{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}
{"return": {}}
-==Attach two SCSI disks using the same block device and the same iothread==
+.==Attach two SCSI disks using the same block device and the same iothread==
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}
{"return": {}}
{"execute": "object-add", "arguments": {"id": "iothread0", "qom-type": "iothread"}}
@@ -32,7 +32,7 @@
{"return": {}}
{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}
{"return": {}}
-==Attach two SCSI disks using the same block device but different iothreads==
+.==Attach two SCSI disks using the same block device but different iothreads==
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}
{"return": {}}
{"execute": "object-add", "arguments": {"id": "iothread0", "qom-type": "iothread"}}
@@ -55,7 +55,7 @@
{"return": {}}
{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}
{"return": {}}
-==Attach a SCSI disks using the same block device as a NBD server==
+.==Attach a SCSI disks using the same block device as a NBD server==
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}
{"return": {}}
{"execute": "nbd-server-start", "arguments": {"addr": {"data": {"path": "SOCK_DIR/PID-nbd.sock"}, "type": "unix"}}}
@@ -68,7 +68,7 @@
{"return": {}}
{"execute": "device_add", "arguments": {"drive": "hd0", "driver": "scsi-hd", "id": "scsi-hd0"}}
{"return": {}}
-....
+.
----------------------------------------------------------------------
Ran 4 tests
diff --git a/tests/qemu-iotests/245.out b/tests/qemu-iotests/245.out
index 4b33dcaf5c..99c12f4f98 100644
--- a/tests/qemu-iotests/245.out
+++ b/tests/qemu-iotests/245.out
@@ -1,16 +1,16 @@
-{"execute": "job-finalize", "arguments": {"id": "commit0"}}
+..{"execute": "job-finalize", "arguments": {"id": "commit0"}}
{"return": {}}
{"data": {"id": "commit0", "type": "commit"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "commit0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "commit"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
-{"execute": "job-finalize", "arguments": {"id": "stream0"}}
+...{"execute": "job-finalize", "arguments": {"id": "stream0"}}
{"return": {}}
{"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
-{"execute": "job-finalize", "arguments": {"id": "stream0"}}
+.{"execute": "job-finalize", "arguments": {"id": "stream0"}}
{"return": {}}
{"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
-.....................
+...............
----------------------------------------------------------------------
Ran 21 tests
diff --git a/tests/qemu-iotests/295.out b/tests/qemu-iotests/295.out
index ad34b2ca2c..5ff91f116c 100644
--- a/tests/qemu-iotests/295.out
+++ b/tests/qemu-iotests/295.out
@@ -4,7 +4,7 @@
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}
{"return": {}}
-{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
+.{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}
{"return": {}}
@@ -13,7 +13,7 @@ Job failed: Invalid password, cannot unlock any keyslot
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
-{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
+.{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
@@ -33,7 +33,7 @@ Job failed: All the active keyslots match the (old) password that was given and
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}
{"return": {}}
-...
+.
----------------------------------------------------------------------
Ran 3 tests
diff --git a/tests/qemu-iotests/296.out b/tests/qemu-iotests/296.out
index cb2859a15c..6c69735604 100644
--- a/tests/qemu-iotests/296.out
+++ b/tests/qemu-iotests/296.out
@@ -13,7 +13,7 @@ Job failed: Failed to get shared "consistent read" lock
qemu-img: Failed to get shared "consistent read" lock
Is another process using the image [TEST_DIR/test.img]?
-Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
+.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
Job failed: Block node is read-only
{"execute": "job-dismiss", "arguments": {"id": "job0"}}
@@ -26,15 +26,15 @@ Job failed: Failed to get shared "consistent read" lock
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job0"}}
{"return": {}}
-Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
+.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
{"return": {}}
{"error": {"class": "GenericError", "desc": "Failed to get \"write\" lock"}}
-Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
+.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
{"return": {}}
{"return": {}}
-....
+.
----------------------------------------------------------------------
Ran 4 tests
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 5af0182895..55a017577f 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -20,7 +20,6 @@
import bz2
from collections import OrderedDict
import faulthandler
-import io
import json
import logging
import os
@@ -32,7 +31,7 @@
import sys
import time
from typing import (Any, Callable, Dict, Iterable,
- List, Optional, Sequence, Tuple, TypeVar)
+ List, Optional, Sequence, TextIO, Tuple, Type, TypeVar)
import unittest
from contextlib import contextmanager
@@ -1271,37 +1270,50 @@ def func_wrapper(*args, **kwargs):
return func(*args, **kwargs)
return func_wrapper
+# We need to filter out the time taken from the output so that
+# qemu-iotest can reliably diff the results against master output,
+# and hide skipped tests from the reference output.
+
+class ReproducibleTestResult(unittest.TextTestResult):
+ def addSkip(self, test, reason):
+ # Same as TextTestResult, but print dot instead of "s"
+ unittest.TestResult.addSkip(self, test, reason)
+ if self.showAll:
+ self.stream.writeln("skipped {0!r}".format(reason))
+ elif self.dots:
+ self.stream.write(".")
+ self.stream.flush()
+
+class ReproducibleStreamWrapper:
+ def __init__(self, stream: TextIO):
+ self.stream = stream
+
+ def __getattr__(self, attr):
+ if attr in ('stream', '__getstate__'):
+ raise AttributeError(attr)
+ return getattr(self.stream, attr)
+
+ def write(self, arg=None):
+ arg = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', arg)
+ arg = re.sub(r' \(skipped=\d+\)', r'', arg)
+ self.stream.write(arg)
+
+class ReproducibleTestRunner(unittest.TextTestRunner):
+ def __init__(self, stream: Optional[TextIO] = None,
+ resultclass: Type[unittest.TestResult] = ReproducibleTestResult,
+ **kwargs: Any) -> None:
+ rstream = ReproducibleStreamWrapper(stream or sys.stdout)
+ super().__init__(stream=rstream, # type: ignore
+ descriptions=True,
+ resultclass=resultclass,
+ **kwargs)
+
def execute_unittest(debug=False):
"""Executes unittests within the calling module."""
verbosity = 2 if debug else 1
-
- if debug:
- output = sys.stdout
- else:
- # We need to filter out the time taken from the output so that
- # qemu-iotest can reliably diff the results against master output.
- output = io.StringIO()
-
- runner = unittest.TextTestRunner(stream=output, descriptions=True,
- verbosity=verbosity)
- try:
- # unittest.main() will use sys.exit(); so expect a SystemExit
- # exception
- unittest.main(testRunner=runner)
- finally:
- # We need to filter out the time taken from the output so that
- # qemu-iotest can reliably diff the results against master output.
- if not debug:
- out = output.getvalue()
- out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out)
-
- # Hide skipped tests from the reference output
- out = re.sub(r'OK \(skipped=\d+\)', 'OK', out)
- out_first_line, out_rest = out.split('\n', 1)
- out = out_first_line.replace('s', '.') + '\n' + out_rest
-
- sys.stderr.write(out)
+ runner = ReproducibleTestRunner(verbosity=verbosity)
+ unittest.main(testRunner=runner)
def execute_setup_common(supported_fmts: Sequence[str] = (),
supported_platforms: Sequence[str] = (),
--
2.31.1
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v4 3/5] qemu-iotests: move command line and environment handling from TestRunner to TestEnv
2021-05-03 11:01 [PATCH v4 0/5] qemu-iotests: quality of life improvements Paolo Bonzini
2021-05-03 11:01 ` [PATCH v4 1/5] qemu-iotests: do not buffer the test output Paolo Bonzini
2021-05-03 11:01 ` [PATCH v4 2/5] qemu-iotests: allow passing unittest.main arguments to the test scripts Paolo Bonzini
@ 2021-05-03 11:01 ` Paolo Bonzini
2021-05-03 11:01 ` [PATCH v4 4/5] qemu-iotests: let "check" spawn an arbitrary test command Paolo Bonzini
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Paolo Bonzini @ 2021-05-03 11:01 UTC (permalink / raw)
To: qemu-devel; +Cc: eesposit, vsementsov, mreitz
In the next patch, "check" will learn how to execute a test script without
going through TestRunner. To enable this, keep only the text output
and subprocess handling in the TestRunner; move into TestEnv the logic
to prepare for running a subprocess.
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Tested-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
Message-Id: <20210323181928.311862-4-pbonzini@redhat.com>
---
tests/qemu-iotests/testenv.py | 17 ++++++++++++++++-
tests/qemu-iotests/testrunner.py | 14 +-------------
2 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
index 6d27712617..fca3a609e0 100644
--- a/tests/qemu-iotests/testenv.py
+++ b/tests/qemu-iotests/testenv.py
@@ -25,7 +25,7 @@
import random
import subprocess
import glob
-from typing import Dict, Any, Optional, ContextManager
+from typing import List, Dict, Any, Optional, ContextManager
def isxfile(path: str) -> bool:
@@ -74,6 +74,21 @@ class TestEnv(ContextManager['TestEnv']):
'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',
'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_']
+ def prepare_subprocess(self, args: List[str]) -> Dict[str, str]:
+ if self.debug:
+ args.append('-d')
+
+ with open(args[0], encoding="utf-8") as f:
+ try:
+ if f.readline().rstrip() == '#!/usr/bin/env python3':
+ args.insert(0, self.python)
+ except UnicodeDecodeError: # binary test? for future.
+ pass
+
+ os_env = os.environ.copy()
+ os_env.update(self.get_env())
+ return os_env
+
def get_env(self) -> Dict[str, str]:
env = {}
for v in self.env_variables:
diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py
index 1fc61fcaa3..519924dc81 100644
--- a/tests/qemu-iotests/testrunner.py
+++ b/tests/qemu-iotests/testrunner.py
@@ -129,7 +129,6 @@ class TestRunner(ContextManager['TestRunner']):
def __init__(self, env: TestEnv, makecheck: bool = False,
color: str = 'auto') -> None:
self.env = env
- self.test_run_env = self.env.get_env()
self.makecheck = makecheck
self.last_elapsed = LastElapsedTime('.last-elapsed-cache', env)
@@ -243,18 +242,7 @@ def do_run_test(self, test: str) -> TestResult:
silent_unlink(p)
args = [str(f_test.resolve())]
- if self.env.debug:
- args.append('-d')
-
- with f_test.open(encoding="utf-8") as f:
- try:
- if f.readline().rstrip() == '#!/usr/bin/env python3':
- args.insert(0, self.env.python)
- except UnicodeDecodeError: # binary test? for future.
- pass
-
- env = os.environ.copy()
- env.update(self.test_run_env)
+ env = self.env.prepare_subprocess(args)
t0 = time.time()
with f_bad.open('w', encoding="utf-8") as f:
--
2.31.1
^ permalink raw reply related [flat|nested] 7+ messages in thread