[PATCH 2/7] kunit: tool: Combine reports from isolated kernels

[Mark Brown <broonie@kernel.org>]

With --run_isolated, parse_tests() writes the same JSON or JUnit
destination after every kernel. Only the last kernel survives in a file,
so an earlier failing suite can disappear from the saved results.
Stdout similarly receives several independent report documents.

Separate report writing from parsing. Keep printing each kernel's results
as they are parsed, accumulate the top-level tests and counts, and write
one report after execution finishes. Preserve the parsed root for a
single kernel and continue suppressing reports with --raw_output.

Test JSON and JUnit files with an early failure and suite and test
isolation. Also check equivalence between single-kernel execution and
parsing.

Fixes: ff9e09a3762f ("kunit: tool: support running each suite/test separately")
Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit.py           | 35 +++++++++++++++----
 tools/testing/kunit/kunit_tool_test.py | 61 ++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 7 deletions(-)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 218ae294ad1b..aea67b6a72c4 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -16,7 +16,7 @@ import time
 
 assert sys.version_info >= (3, 7), "Python version is too old"
 
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from enum import Enum, auto
 from typing import Iterable, List, Optional, Sequence, Tuple
 
@@ -191,6 +191,10 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 	metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig')
 
 	test_counts = kunit_parser.TestCounts()
+	report = kunit_parser.Test()
+	report.name = 'main'
+	report.counts = test_counts
+	parse_request = replace(request, json=None, junit=None)
 	test_failed = False
 	exec_time = 0.0
 	for i, filter_glob in enumerate(filter_globs):
@@ -205,7 +209,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 			filter_action=request.filter_action,
 			build_dir=request.build_dir)
 
-		_, test_result = parse_tests(request, metadata, run_result)
+		_, test_result = parse_tests(parse_request, metadata, run_result)
 		# run_kernel() doesn't block on the kernel exiting.
 		# That only happens after we get the last line of output from `run_result`.
 		# So exec_time here actually contains parsing + execution time, which is fine.
@@ -216,6 +220,19 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 		# A kernel with no results has no failed test counts to aggregate.
 		if _map_to_overall_status(test_result.status) != KunitStatus.SUCCESS:
 			test_failed = True
+		if request.json or request.junit:
+			if len(filter_globs) == 1:
+				report = test_result
+			else:
+				report.subtests.extend(test_result.subtests)
+				report.log.extend(test_result.log)
+	if len(filter_globs) != 1:
+		report.expected_count = len(report.subtests)
+		report.status = test_counts.get_status()
+		if test_failed and report.ok_status():
+			report.status = kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS
+	if not request.raw_output:
+		_write_test_results(request, metadata, report)
 
 	if len(filter_globs) == 1 and test_counts.crashed > 0:
 		bd = request.build_dir
@@ -264,6 +281,15 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input
 		kunit_parser.print_test(test, request.failed, stdout)
 	kunit_parser.print_summary_line(test, stdout)
 
+	_write_test_results(request, metadata, test)
+
+	if test.status != kunit_parser.TestStatus.SUCCESS:
+		return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
+
+	return KunitResult(KunitStatus.SUCCESS, parse_time), test
+
+def _write_test_results(request: KunitParseRequest, metadata: kunit_json.Metadata,
+			test: kunit_parser.Test) -> None:
 	if request.json:
 		json_str = kunit_json.get_json_result(
 					test=test,
@@ -283,11 +309,6 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input
 			kunit_junit.write_junit_result(test=test,filename=request.junit)
 			stdout.print_with_timestamp(f"Test results stored in {os.path.abspath(request.junit)}")
 
-	if test.status != kunit_parser.TestStatus.SUCCESS:
-		return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
-
-	return KunitResult(KunitStatus.SUCCESS, parse_time), test
-
 def run_tests(linux: kunit_kernel.LinuxSourceTree,
 	      request: KunitRequest) -> KunitResult:
 	run_start = time.time()
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index ca2dac6651aa..4590108e4306 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -19,6 +19,7 @@ import signal
 import subprocess
 import sys
 from typing import Iterable
+import xml.etree.ElementTree as ET
 
 import kunit_config
 import kunit_parser
@@ -738,6 +739,7 @@ class KUnitMainTest(unittest.TestCase):
 
 		self.mock_linux_init = mock.patch.object(kunit_kernel, 'LinuxSourceTree').start()
 		self.linux_source_mock = self.mock_linux_init.return_value
+		self.linux_source_mock.arch.return_value = 'um'
 		self.linux_source_mock.build_reconfig.return_value = True
 		self.linux_source_mock.build_kernel.return_value = True
 		self.linux_source_mock.run_kernel.return_value = all_passed_log
@@ -1049,6 +1051,65 @@ class KUnitMainTest(unittest.TestCase):
 
 				kunit.main(['exec', '--run_isolated=suite'])
 
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_reports_include_all_kernels(self, mock_tests):
+		def fake_run_kernel(filter_glob, **kwargs):
+			if '.' in filter_glob:
+				suite, case = filter_glob.split('.')
+				failed = case == 'test1'
+			else:
+				suite, case = filter_glob, 'test'
+				failed = suite == 'suite1'
+			status = 'not ok' if failed else 'ok'
+			return ['KTAP version 1', '1..1', '    # Subtest: ' + suite,
+				'    1..1', f'    {status} 1 - {case}', f'{status} 1 - {suite}']
+		self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+		for isolation in ('suite', 'test'):
+			with self.subTest(isolation=isolation), \
+			     tempfile.TemporaryDirectory() as output_dir:
+				if isolation == 'suite':
+					mock_tests.return_value = ['suite1.test', 'suite2.test']
+					expected = [('suite1', 'test', 'FAIL'), ('suite2', 'test', 'PASS')]
+				else:
+					mock_tests.return_value = ['suite.test1', 'suite.test2']
+					expected = [('suite', 'test1', 'FAIL'), ('suite', 'test2', 'PASS')]
+				json_path = os.path.join(output_dir, 'results.json')
+				xml_path = os.path.join(output_dir, 'results.xml')
+				with self.assertRaises(SystemExit) as e:
+					kunit.main(['exec', '--run_isolated=' + isolation,
+						    '--json=' + json_path, '--junit=' + xml_path])
+				self.assertEqual(e.exception.code, 1)
+
+				with open(json_path) as f:
+					report = json.load(f)
+				self.assertEqual(report['misc']['tests'], 2)
+				self.assertEqual(report['misc']['failed'], 1)
+				self.assertEqual(report['arch'], 'um')
+				self.assertEqual([(suite['name'], case['name'], case['status'])
+					for suite in report['sub_groups'] for case in suite['test_cases']], expected)
+				root = ET.parse(xml_path).getroot()
+				self.assertEqual(root.get('tests'), '2')
+				self.assertEqual(root.get('failures'), '1')
+				self.assertEqual([(suite.get('name'), case.get('name'),
+					'FAIL' if case.find('failure') is not None else 'PASS')
+					for suite in root for case in suite], expected)
+
+	def test_single_kernel_reports_match_parse(self):
+		with tempfile.TemporaryDirectory() as output_dir:
+			json_path = os.path.join(output_dir, 'results.json')
+			xml_path = os.path.join(output_dir, 'results.xml')
+			reports = []
+			for args in (['exec'], ['parse', _test_data_path('test_is_test_passed-all_passed.log')]):
+				kunit.main(args + ['--json=' + json_path, '--junit=' + xml_path])
+				with open(json_path) as f:
+					report = json.load(f)
+				# Execution adds build metadata, which parse cannot know.
+				reports.append((report['misc'],
+					[(s['name'], s['test_cases']) for s in report['sub_groups']],
+					ET.tostring(ET.parse(xml_path).getroot())))
+			self.assertEqual(reports[0], reports[1])
+
 	@mock.patch.object(kunit, '_list_tests')
 	@mock.patch.object(sys, 'stdout', new_callable=io.StringIO)
 	def test_list_suites(self, mock_stdout, mock_tests):

-- 
2.47.3