[PATCH 1/7] kunit: tool: Preserve failures from kernels without test results

[Mark Brown <broonie@kernel.org>]

When an isolated kernel produces no test results, the parser
reports failure without adding failed test counts. Aggregating only the
counts therefore lets a passing kernel mask the missing results and makes
the command exit successfully. A kernel reporting an empty test plan has
the same problem.

Preserve each kernel's failure status alongside the aggregate counts.
Keep treating skipped tests as successful and retain the failure when no
kernels run at all. Add regression coverage for missing and empty
results in either execution order and for passing and skipped kernels.

Fixes: d65d07cb5b01 ("kunit: tool: improve compatibility of kunit_parser with KTAP specification")
Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit.py           |  6 ++++++
 tools/testing/kunit/kunit_tool_test.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 91d234ac3b57..218ae294ad1b 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -191,6 +191,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 	metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig')
 
 	test_counts = kunit_parser.TestCounts()
+	test_failed = False
 	exec_time = 0.0
 	for i, filter_glob in enumerate(filter_globs):
 		stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs)))
@@ -212,6 +213,9 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 		exec_time += test_end - test_start
 
 		test_counts.add_subtest_counts(test_result.counts)
+		# A kernel with no results has no failed test counts to aggregate.
+		if _map_to_overall_status(test_result.status) != KunitStatus.SUCCESS:
+			test_failed = True
 
 	if len(filter_globs) == 1 and test_counts.crashed > 0:
 		bd = request.build_dir
@@ -220,6 +224,8 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 				bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0]))
 
 	kunit_status = _map_to_overall_status(test_counts.get_status())
+	if test_failed:
+		kunit_status = KunitStatus.TEST_FAILURE
 	return KunitResult(status=kunit_status, elapsed_time=exec_time)
 
 def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 85ae21754bdf..ca2dac6651aa 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -1018,6 +1018,37 @@ class KUnitMainTest(unittest.TestCase):
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter='', filter_action=None, timeout=300),
 		])
 
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_kernel_without_results_fails(self, mock_tests):
+		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
+		for missing_output in ([], ['KTAP version 1', '1..0']):
+			for missing_suite in ('suite1', 'suite2'):
+				with self.subTest(output=missing_output,
+						 suite=missing_suite):
+					def fake_run_kernel(filter_glob, **kwargs):
+						if filter_glob == missing_suite:
+							return missing_output
+						return ['KTAP version 1', '1..1', 'ok 1 - test1']
+					self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+					with self.assertRaises(SystemExit) as e:
+						kunit.main(['exec', '--run_isolated=suite'])
+					self.assertEqual(e.exception.code, 1)
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_skipped_kernels_succeed(self, mock_tests):
+		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
+		for all_skipped in (False, True):
+			with self.subTest(all_skipped=all_skipped):
+				def fake_run_kernel(filter_glob, **kwargs):
+					result = 'ok 1 - test1'
+					if all_skipped or filter_glob == 'suite1':
+						result += ' # SKIP unavailable'
+					return ['KTAP version 1', '1..1', result]
+				self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+				kunit.main(['exec', '--run_isolated=suite'])
+
 	@mock.patch.object(kunit, '_list_tests')
 	@mock.patch.object(sys, 'stdout', new_callable=io.StringIO)
 	def test_list_suites(self, mock_stdout, mock_tests):

-- 
2.47.3