[PATCH 0/7] kunit: tool: Support parallel test execution

[PATCH 0/7] kunit: tool: Support parallel test execution

[Mark Brown]

Currently the test execution part of KUnit is very single threaded,
meaning that on larger systems with many CPUs the wall clock time for
running the full suite is longer than it could be.  While we don't want
to run multiple tests in parallel on the same kernel for stability and
clarity reasons all the tests are being run in a virtual environment of
some kind so we can achieve better parallelism by splitting the tests
between multiple simultaneously running virtual environments.  The KUnit
runner already has support for splitting tests up and running either
suites or individual tests in isolated guests, build on this to support
parallel execution.

While developing this some existing issues became more visible, some of
these are fixed with some standalone patches at the start of the series 
but I split some issues with test naming out, these were sent as:

 - lib/tests: util_macros: Drop the '.' from the KUnit suite name
   https://lore.kernel.org/r/20261012-kunit-util-macros-name-v1-1-810b401baeb0@kernel.org
 - kunit: Add KUNIT_CASE_PARAM_UNIQUE() and KUNIT_CASE_PARAM_WITH_INIT_UNIQUE()
   https://lore.kernel.org/r/20261013-kunit-param-names-v1-0-4b22ec5b85b9@kernel.org

Performance results with --alltests, using the elapsed time reported by
the KUnit runner with an already built kernel on a Raspberry Pi 5 with 4
CPUs and a Graviton 4 with 192 CPUs:

                     default  parallel  per test   parallel per test
  Raspberry Pi 5     67.7s    49.4s     836.4s     326.5s
  Graviton 4         64.1s    30.8s     445.9s      39.7s

Even with the overhead of having to boot more kernels we are seeing
substantial speedups, even on lower end systems with limited parallelism
available.

Signed-off-by: Mark Brown <broonie@kernel.org>
---
Mark Brown (7):
      kunit: tool: Preserve failures from kernels without test results
      kunit: tool: Combine reports from isolated kernels
      kunit: tool: Boot with a larger log buffer when listing tests
      kunit: tool: Add an outfile argument to run_kernel()
      kunit: tool: Allow several run_kernel() calls to be in flight at once
      kunit: tool: Serialise output written from multiple threads
      kunit: tool: Add --parallel to run suites in concurrent kernels

 Documentation/dev-tools/kunit/run_wrapper.rst |  11 ++
 tools/testing/kunit/kunit.py                  | 151 +++++++++++----
 tools/testing/kunit/kunit_junit.py            |   3 +-
 tools/testing/kunit/kunit_kernel.py           |  78 +++++---
 tools/testing/kunit/kunit_printer.py          |   9 +-
 tools/testing/kunit/kunit_tool_test.py        | 253 +++++++++++++++++++++++++-
 6 files changed, 438 insertions(+), 67 deletions(-)
---
base-commit: 24a19b953e2a3d0ea5a8c4951a0ab8f32720c907
base-tree: git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm master
change-id: 20261012-kunit-parallel-a8a7e3c19a30

Best regards,
--  
Mark Brown <broonie@kernel.org>

[PATCH 1/7] kunit: tool: Preserve failures from kernels without test results

[Mark Brown]

When an isolated kernel produces no test results, the parser
reports failure without adding failed test counts. Aggregating only the
counts therefore lets a passing kernel mask the missing results and makes
the command exit successfully. A kernel reporting an empty test plan has
the same problem.

Preserve each kernel's failure status alongside the aggregate counts.
Keep treating skipped tests as successful and retain the failure when no
kernels run at all. Add regression coverage for missing and empty
results in either execution order and for passing and skipped kernels.

Fixes: d65d07cb5b01 ("kunit: tool: improve compatibility of kunit_parser with KTAP specification")
Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit.py           |  6 ++++++
 tools/testing/kunit/kunit_tool_test.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 91d234ac3b57..218ae294ad1b 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -191,6 +191,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 	metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig')
 
 	test_counts = kunit_parser.TestCounts()
+	test_failed = False
 	exec_time = 0.0
 	for i, filter_glob in enumerate(filter_globs):
 		stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs)))
@@ -212,6 +213,9 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 		exec_time += test_end - test_start
 
 		test_counts.add_subtest_counts(test_result.counts)
+		# A kernel with no results has no failed test counts to aggregate.
+		if _map_to_overall_status(test_result.status) != KunitStatus.SUCCESS:
+			test_failed = True
 
 	if len(filter_globs) == 1 and test_counts.crashed > 0:
 		bd = request.build_dir
@@ -220,6 +224,8 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 				bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0]))
 
 	kunit_status = _map_to_overall_status(test_counts.get_status())
+	if test_failed:
+		kunit_status = KunitStatus.TEST_FAILURE
 	return KunitResult(status=kunit_status, elapsed_time=exec_time)
 
 def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 85ae21754bdf..ca2dac6651aa 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -1018,6 +1018,37 @@ class KUnitMainTest(unittest.TestCase):
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter='', filter_action=None, timeout=300),
 		])
 
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_kernel_without_results_fails(self, mock_tests):
+		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
+		for missing_output in ([], ['KTAP version 1', '1..0']):
+			for missing_suite in ('suite1', 'suite2'):
+				with self.subTest(output=missing_output,
+						 suite=missing_suite):
+					def fake_run_kernel(filter_glob, **kwargs):
+						if filter_glob == missing_suite:
+							return missing_output
+						return ['KTAP version 1', '1..1', 'ok 1 - test1']
+					self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+					with self.assertRaises(SystemExit) as e:
+						kunit.main(['exec', '--run_isolated=suite'])
+					self.assertEqual(e.exception.code, 1)
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_skipped_kernels_succeed(self, mock_tests):
+		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
+		for all_skipped in (False, True):
+			with self.subTest(all_skipped=all_skipped):
+				def fake_run_kernel(filter_glob, **kwargs):
+					result = 'ok 1 - test1'
+					if all_skipped or filter_glob == 'suite1':
+						result += ' # SKIP unavailable'
+					return ['KTAP version 1', '1..1', result]
+				self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+				kunit.main(['exec', '--run_isolated=suite'])
+
 	@mock.patch.object(kunit, '_list_tests')
 	@mock.patch.object(sys, 'stdout', new_callable=io.StringIO)
 	def test_list_suites(self, mock_stdout, mock_tests):

-- 
2.47.3

[PATCH 2/7] kunit: tool: Combine reports from isolated kernels

[Mark Brown]

With --run_isolated, parse_tests() writes the same JSON or JUnit
destination after every kernel. Only the last kernel survives in a file,
so an earlier failing suite can disappear from the saved results.
Stdout similarly receives several independent report documents.

Separate report writing from parsing. Keep printing each kernel's results
as they are parsed, accumulate the top-level tests and counts, and write
one report after execution finishes. Preserve the parsed root for a
single kernel and continue suppressing reports with --raw_output.

Test JSON and JUnit files with an early failure and suite and test
isolation. Also check equivalence between single-kernel execution and
parsing.

Fixes: ff9e09a3762f ("kunit: tool: support running each suite/test separately")
Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit.py           | 35 +++++++++++++++----
 tools/testing/kunit/kunit_tool_test.py | 61 ++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 7 deletions(-)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 218ae294ad1b..aea67b6a72c4 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -16,7 +16,7 @@ import time
 
 assert sys.version_info >= (3, 7), "Python version is too old"
 
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from enum import Enum, auto
 from typing import Iterable, List, Optional, Sequence, Tuple
 
@@ -191,6 +191,10 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 	metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig')
 
 	test_counts = kunit_parser.TestCounts()
+	report = kunit_parser.Test()
+	report.name = 'main'
+	report.counts = test_counts
+	parse_request = replace(request, json=None, junit=None)
 	test_failed = False
 	exec_time = 0.0
 	for i, filter_glob in enumerate(filter_globs):
@@ -205,7 +209,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 			filter_action=request.filter_action,
 			build_dir=request.build_dir)
 
-		_, test_result = parse_tests(request, metadata, run_result)
+		_, test_result = parse_tests(parse_request, metadata, run_result)
 		# run_kernel() doesn't block on the kernel exiting.
 		# That only happens after we get the last line of output from `run_result`.
 		# So exec_time here actually contains parsing + execution time, which is fine.
@@ -216,6 +220,19 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 		# A kernel with no results has no failed test counts to aggregate.
 		if _map_to_overall_status(test_result.status) != KunitStatus.SUCCESS:
 			test_failed = True
+		if request.json or request.junit:
+			if len(filter_globs) == 1:
+				report = test_result
+			else:
+				report.subtests.extend(test_result.subtests)
+				report.log.extend(test_result.log)
+	if len(filter_globs) != 1:
+		report.expected_count = len(report.subtests)
+		report.status = test_counts.get_status()
+		if test_failed and report.ok_status():
+			report.status = kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS
+	if not request.raw_output:
+		_write_test_results(request, metadata, report)
 
 	if len(filter_globs) == 1 and test_counts.crashed > 0:
 		bd = request.build_dir
@@ -264,6 +281,15 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input
 		kunit_parser.print_test(test, request.failed, stdout)
 	kunit_parser.print_summary_line(test, stdout)
 
+	_write_test_results(request, metadata, test)
+
+	if test.status != kunit_parser.TestStatus.SUCCESS:
+		return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
+
+	return KunitResult(KunitStatus.SUCCESS, parse_time), test
+
+def _write_test_results(request: KunitParseRequest, metadata: kunit_json.Metadata,
+			test: kunit_parser.Test) -> None:
 	if request.json:
 		json_str = kunit_json.get_json_result(
 					test=test,
@@ -283,11 +309,6 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input
 			kunit_junit.write_junit_result(test=test,filename=request.junit)
 			stdout.print_with_timestamp(f"Test results stored in {os.path.abspath(request.junit)}")
 
-	if test.status != kunit_parser.TestStatus.SUCCESS:
-		return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
-
-	return KunitResult(KunitStatus.SUCCESS, parse_time), test
-
 def run_tests(linux: kunit_kernel.LinuxSourceTree,
 	      request: KunitRequest) -> KunitResult:
 	run_start = time.time()
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index ca2dac6651aa..4590108e4306 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -19,6 +19,7 @@ import signal
 import subprocess
 import sys
 from typing import Iterable
+import xml.etree.ElementTree as ET
 
 import kunit_config
 import kunit_parser
@@ -738,6 +739,7 @@ class KUnitMainTest(unittest.TestCase):
 
 		self.mock_linux_init = mock.patch.object(kunit_kernel, 'LinuxSourceTree').start()
 		self.linux_source_mock = self.mock_linux_init.return_value
+		self.linux_source_mock.arch.return_value = 'um'
 		self.linux_source_mock.build_reconfig.return_value = True
 		self.linux_source_mock.build_kernel.return_value = True
 		self.linux_source_mock.run_kernel.return_value = all_passed_log
@@ -1049,6 +1051,65 @@ class KUnitMainTest(unittest.TestCase):
 
 				kunit.main(['exec', '--run_isolated=suite'])
 
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_reports_include_all_kernels(self, mock_tests):
+		def fake_run_kernel(filter_glob, **kwargs):
+			if '.' in filter_glob:
+				suite, case = filter_glob.split('.')
+				failed = case == 'test1'
+			else:
+				suite, case = filter_glob, 'test'
+				failed = suite == 'suite1'
+			status = 'not ok' if failed else 'ok'
+			return ['KTAP version 1', '1..1', '    # Subtest: ' + suite,
+				'    1..1', f'    {status} 1 - {case}', f'{status} 1 - {suite}']
+		self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+		for isolation in ('suite', 'test'):
+			with self.subTest(isolation=isolation), \
+			     tempfile.TemporaryDirectory() as output_dir:
+				if isolation == 'suite':
+					mock_tests.return_value = ['suite1.test', 'suite2.test']
+					expected = [('suite1', 'test', 'FAIL'), ('suite2', 'test', 'PASS')]
+				else:
+					mock_tests.return_value = ['suite.test1', 'suite.test2']
+					expected = [('suite', 'test1', 'FAIL'), ('suite', 'test2', 'PASS')]
+				json_path = os.path.join(output_dir, 'results.json')
+				xml_path = os.path.join(output_dir, 'results.xml')
+				with self.assertRaises(SystemExit) as e:
+					kunit.main(['exec', '--run_isolated=' + isolation,
+						    '--json=' + json_path, '--junit=' + xml_path])
+				self.assertEqual(e.exception.code, 1)
+
+				with open(json_path) as f:
+					report = json.load(f)
+				self.assertEqual(report['misc']['tests'], 2)
+				self.assertEqual(report['misc']['failed'], 1)
+				self.assertEqual(report['arch'], 'um')
+				self.assertEqual([(suite['name'], case['name'], case['status'])
+					for suite in report['sub_groups'] for case in suite['test_cases']], expected)
+				root = ET.parse(xml_path).getroot()
+				self.assertEqual(root.get('tests'), '2')
+				self.assertEqual(root.get('failures'), '1')
+				self.assertEqual([(suite.get('name'), case.get('name'),
+					'FAIL' if case.find('failure') is not None else 'PASS')
+					for suite in root for case in suite], expected)
+
+	def test_single_kernel_reports_match_parse(self):
+		with tempfile.TemporaryDirectory() as output_dir:
+			json_path = os.path.join(output_dir, 'results.json')
+			xml_path = os.path.join(output_dir, 'results.xml')
+			reports = []
+			for args in (['exec'], ['parse', _test_data_path('test_is_test_passed-all_passed.log')]):
+				kunit.main(args + ['--json=' + json_path, '--junit=' + xml_path])
+				with open(json_path) as f:
+					report = json.load(f)
+				# Execution adds build metadata, which parse cannot know.
+				reports.append((report['misc'],
+					[(s['name'], s['test_cases']) for s in report['sub_groups']],
+					ET.tostring(ET.parse(xml_path).getroot())))
+			self.assertEqual(reports[0], reports[1])
+
 	@mock.patch.object(kunit, '_list_tests')
 	@mock.patch.object(sys, 'stdout', new_callable=io.StringIO)
 	def test_list_suites(self, mock_stdout, mock_tests):

-- 
2.47.3

[PATCH 3/7] kunit: tool: Boot with a larger log buffer when listing tests

[Mark Brown]

kunit.action=list prints the name of every suite and test in one burst
during boot, which on a large configuration such as --alltests comes to
several thousand lines. That is more than the default 128KiB printk
ring buffer (CONFIG_LOG_BUF_SHIFT=17, overridden by many defconfigs) can
hold, and since the console drains more slowly than the executor prints
the buffer wraps and lines can easily be dropped.

If the KTAP header is dropped _list_tests() fails with:

  ValueError: LineStream: going past EOF, last line was

but otherwise the loss is silent, --list_tests prints an incomplete list
and --run_isolated=test will execute a timing dependent sent of tests.

Pass log_buf_len=4M on the kernel command line when booting to list
tests so that the whole list fits in the buffer, at least until we get
substantially more tests than we currently have.

Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit.py           | 11 +++++++++--
 tools/testing/kunit/kunit_tool_test.py |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index aea67b6a72c4..971e70b0fb1a 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -109,8 +109,15 @@ def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
 
 	return build_tests(linux, request)
 
+# kunit.action=list prints every suite and test name in one burst at boot,
+# which can be thousands of lines on a large config. That easily overflows
+# the default 128KiB printk buffer before the console has drained it, so
+# lines (sometimes including the KTAP header) are dropped and the list
+# comes out incomplete. Give the kernel a large enough buffer to hold it.
+_LIST_KERNEL_ARGS = ['log_buf_len=4M']
+
 def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
-	args = ['kunit.action=list']
+	args = ['kunit.action=list'] + _LIST_KERNEL_ARGS
 
 	if request.kernel_args:
 		args.extend(request.kernel_args)
@@ -129,7 +136,7 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest)
 	return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
 
 def _list_tests_attr(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]:
-	args = ['kunit.action=list_attr']
+	args = ['kunit.action=list_attr'] + _LIST_KERNEL_ARGS
 
 	if request.kernel_args:
 		args.extend(request.kernel_args)
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 4590108e4306..17f926ef2820 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -979,7 +979,7 @@ class KUnitMainTest(unittest.TestCase):
 		self.assertEqual(got, want)
 		# Should respect the user's filter glob when listing tests.
 		self.linux_source_mock.run_kernel.assert_called_once_with(
-			args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter='', filter_action=None, timeout=300)
+			args=['kunit.action=list', 'log_buf_len=4M'], build_dir='.kunit', filter_glob='suite*', filter='', filter_action=None, timeout=300)
 
 	def test_list_tests_with_prefix(self):
 		want = ['suite.test1', 'suite.test2', 'suite2.test1']

-- 
2.47.3

[PATCH 4/7] kunit: tool: Add an outfile argument to run_kernel()

[Mark Brown]

run_kernel() always tees the kernel's console output to test.log in the
build directory. If more than one kernel were to be run at a time they
would all write to the same file, so allow the caller to choose where
the log goes with a new optional outfile argument.

Also teach get_outfile_path() to generate per-kernel log names of the
form test.<name>.log when given a name, for callers that need to keep
the logs of several kernels apart. Characters that are awkward in file
names, such as the '*' in a filter glob, are replaced with '_'.

Add a unit test covering the new argument.

Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit_kernel.py    | 28 ++++++++++++++++++++++++----
 tools/testing/kunit/kunit_tool_test.py | 17 +++++++++++++++++
 2 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index 58557c47d85f..2641db9eddba 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -11,6 +11,7 @@ import importlib.util
 import logging
 import subprocess
 import os
+import re
 import shlex
 import shutil
 import signal
@@ -193,8 +194,19 @@ def get_parsed_kunitconfig(build_dir: str,
 		merged.merge_in_entries(partial)
 	return merged
 
-def get_outfile_path(build_dir: str) -> str:
-	return os.path.join(build_dir, OUTFILE_PATH)
+def get_outfile_path(build_dir: str, name: Optional[str]=None) -> str:
+	"""Returns the path of the kernel console log.
+
+	When several kernels are run each needs its own log, so `name` is
+	used to tell them apart, giving test.<name>.log. Characters that are
+	awkward in file names (anything other than letters, digits, '_', '-'
+	and '.') are replaced with '_'.
+	"""
+	if name is None:
+		return os.path.join(build_dir, OUTFILE_PATH)
+	base, ext = os.path.splitext(OUTFILE_PATH)
+	name = re.sub(r'[^\w.-]', '_', name)
+	return os.path.join(build_dir, f'{base}.{name}{ext}')
 
 def _default_qemu_config_path(arch: str) -> str:
 	config_path = os.path.join(QEMU_CONFIGS_DIR, arch + '.py')
@@ -352,7 +364,12 @@ class LinuxSourceTree:
 			return
 		subprocess.call(['stty', 'sane'])
 
-	def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None) -> Iterator[str]:
+	def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None, outfile: Optional[str]=None) -> Iterator[str]:
+		"""Boots the kernel and yields its console output line by line.
+
+		The output is also written to `outfile`, which defaults to
+		test.log in the build directory.
+		"""
 		# Copy to avoid mutating the caller-supplied list. exec_tests() reuses
 		# the same args across repeated run_kernel() calls (e.g. --run_isolated),
 		# so appending to the original would accumulate stale flags on each call.
@@ -365,6 +382,9 @@ class LinuxSourceTree:
 			args.append('kunit.filter_action=' + filter_action)
 		args.append('kunit.enable=1')
 
+		if outfile is None:
+			outfile = get_outfile_path(build_dir)
+
 		self._process = self._ops.start(args, build_dir)
 		assert self._process is not None # tell mypy it's set
 		assert self._process.stdout is not None  # tell mypy it's set
@@ -382,7 +402,7 @@ class LinuxSourceTree:
 		waiter = threading.Thread(target=_wait_proc)
 		waiter.start()
 
-		output = open(get_outfile_path(build_dir), 'w')
+		output = open(outfile, 'w')
 		try:
 			# Tee the output to the file and to our caller in real time.
 			for line in self._process.stdout:
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 17f926ef2820..4e8068dce918 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -522,6 +522,23 @@ class LinuxSourceTreeTest(unittest.TestCase):
 			with open(kunit_kernel.get_outfile_path(build_dir), 'rt') as outfile:
 				self.assertEqual(outfile.read(), 'hi\nbye\n', msg='Missing some output')
 
+	def test_run_kernel_outfile(self):
+		"""Verify run_kernel() logs to the given outfile instead of test.log."""
+		def fake_start(unused_args, unused_build_dir):
+			return subprocess.Popen(['printf', 'hi\n'], text=True, stdout=subprocess.PIPE)
+
+		with tempfile.TemporaryDirectory('') as build_dir:
+			tree = kunit_kernel.LinuxSourceTree(build_dir, kunitconfig_paths=[os.devnull])
+			mock.patch.object(tree._ops, 'start', side_effect=fake_start).start()
+
+			# Awkward characters in the name are sanitised.
+			outfile = kunit_kernel.get_outfile_path(build_dir, 'my suite.test*')
+			self.assertEqual(outfile, os.path.join(build_dir, 'test.my_suite.test_.log'))
+			self.assertEqual(list(tree.run_kernel(build_dir=build_dir, outfile=outfile)), ['hi\n'])
+			with open(outfile, 'rt') as f:
+				self.assertEqual(f.read(), 'hi\n')
+			self.assertFalse(os.path.exists(kunit_kernel.get_outfile_path(build_dir)))
+
 	def test_run_kernel_args_not_mutated(self):
 		"""Verify run_kernel() copies args so callers can reuse them."""
 		start_calls = []

-- 
2.47.3

[PATCH 5/7] kunit: tool: Allow several run_kernel() calls to be in flight at once

[Mark Brown]

run_kernel() records the kernel it starts in a single self._process
attribute, which the timeout thread, the cleanup at the end of the
output and the SIGINT handler all read back. That only works while one
kernel is running at a time, which rules out running suites in
parallel guests from multiple threads.

Track running kernels in a list protected by a lock instead, with each
run_kernel() invocation only adding and removing its own entry. The
timeout thread and the cleanup code work on a local reference to their
own process rather than going via self, so one kernel timing out can no
longer terminate a different one. The SIGINT handler terminates every
kernel that is still running.

No functional change when a single kernel is run at a time.

Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit_kernel.py    | 45 ++++++++++++++++++++--------------
 tools/testing/kunit/kunit_tool_test.py | 27 ++++++++++++++++++++
 2 files changed, 53 insertions(+), 19 deletions(-)

diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index 2641db9eddba..1d3226f5a136 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -277,7 +277,11 @@ class LinuxSourceTree:
 		if kconfig_add:
 			kconfig = kunit_config.parse_from_string('\n'.join(kconfig_add))
 			self._kconfig.merge_in_entries(kconfig)
-		self._process : Optional[subprocess.Popen[Any]] = None
+		# Kernels currently running. There can be several when
+		# run_kernel() is called concurrently from multiple threads.
+		self._processes : List[subprocess.Popen[Any]] = []
+		# SIGINT can interrupt the main thread while it holds this lock.
+		self._processes_lock = threading.RLock()
 
 	def arch(self) -> str:
 		return self._arch
@@ -368,7 +372,9 @@ class LinuxSourceTree:
 		"""Boots the kernel and yields its console output line by line.
 
 		The output is also written to `outfile`, which defaults to
-		test.log in the build directory.
+		test.log in the build directory. This may be called from
+		several threads at once to run kernels concurrently, in which
+		case each caller must supply a distinct `outfile`.
 		"""
 		# Copy to avoid mutating the caller-supplied list. exec_tests() reuses
 		# the same args across repeated run_kernel() calls (e.g. --run_isolated),
@@ -385,37 +391,36 @@ class LinuxSourceTree:
 		if outfile is None:
 			outfile = get_outfile_path(build_dir)
 
-		self._process = self._ops.start(args, build_dir)
-		assert self._process is not None # tell mypy it's set
-		assert self._process.stdout is not None  # tell mypy it's set
+		process = self._ops.start(args, build_dir)
+		assert process is not None # tell mypy it's set
+		assert process.stdout is not None  # tell mypy it's set
+		with self._processes_lock:
+			self._processes.append(process)
 
 		# Enforce the timeout in a background thread.
 		def _wait_proc() -> None:
 			try:
-				if self._process:
-					self._process.wait(timeout=timeout)
+				process.wait(timeout=timeout)
 			except Exception as e:
 				print(e)
-				if self._process:
-					self._process.terminate()
-					self._process.wait()
+				process.terminate()
+				process.wait()
 		waiter = threading.Thread(target=_wait_proc)
 		waiter.start()
 
 		output = open(outfile, 'w')
 		try:
 			# Tee the output to the file and to our caller in real time.
-			for line in self._process.stdout:
+			for line in process.stdout:
 				output.write(line)
 				yield line
 		# This runs even if our caller doesn't consume every line.
 		finally:
 			# Flush any leftover output to the file
-			if self._process:
-				if self._process.stdout:
-					output.write(self._process.stdout.read())
-					self._process.stdout.close()
-				self._process = None
+			output.write(process.stdout.read())
+			process.stdout.close()
+			with self._processes_lock:
+				self._processes.remove(process)
 			output.close()
 
 			waiter.join()
@@ -423,7 +428,9 @@ class LinuxSourceTree:
 
 	def signal_handler(self, unused_sig: int, unused_frame: Optional[FrameType]) -> None:
 		logging.error('Build interruption occurred. Cleaning console.')
-		if self._process:
-				self._process.terminate()
-				self._process.wait()
+		with self._processes_lock:
+			processes = list(self._processes)
+		for process in processes:
+			process.terminate()
+			process.wait()
 		self._restore_terminal_if_tty()
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 4e8068dce918..c4999273e491 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -18,6 +18,7 @@ import os
 import signal
 import subprocess
 import sys
+import textwrap
 from typing import Iterable
 import xml.etree.ElementTree as ET
 
@@ -538,6 +539,8 @@ class LinuxSourceTreeTest(unittest.TestCase):
 			with open(outfile, 'rt') as f:
 				self.assertEqual(f.read(), 'hi\n')
 			self.assertFalse(os.path.exists(kunit_kernel.get_outfile_path(build_dir)))
+			# The kernel has exited so should no longer be tracked.
+			self.assertEqual(tree._processes, [])
 
 	def test_run_kernel_args_not_mutated(self):
 		"""Verify run_kernel() copies args so callers can reuse them."""
@@ -607,6 +610,30 @@ class LinuxSourceTreeTest(unittest.TestCase):
 			mock_error.assert_called_once()
 			mock_call.assert_called_once_with(['stty', 'sane'])
 
+	def test_signal_handler_during_process_update(self):
+		# Run in a subprocess so a deadlock fails with a timeout rather
+		# than hanging the test runner. Deliver a real signal while the
+		# main thread holds the lock used to update the process list.
+		code = textwrap.dedent('''\
+			import os
+			import signal
+			from unittest import mock
+			import kunit_kernel
+
+			tree = kunit_kernel.LinuxSourceTree('', kunitconfig_paths=[os.devnull])
+			process = mock.Mock()
+			with tree._processes_lock:
+				tree._processes.append(process)
+				os.kill(os.getpid(), signal.SIGINT)
+			process.terminate.assert_called_once_with()
+			process.wait.assert_called_once_with()
+		''')
+		result = subprocess.run([sys.executable, '-c', code],
+			cwd=os.path.dirname(os.path.abspath(__file__)),
+			stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
+			stderr=subprocess.STDOUT, text=True, timeout=5)
+		self.assertEqual(result.returncode, 0, msg=result.stdout)
+
 	def test_build_reconfig_no_config(self):
 		with tempfile.TemporaryDirectory('') as build_dir:
 			with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f:

-- 
2.47.3

[PATCH 6/7] kunit: tool: Serialise output written from multiple threads

[Mark Brown]

Python's text streams are not thread safe. If two threads write to
sys.stdout at the same time the stream's internal buffer can be
corrupted, resulting in lost output and runs of NUL bytes or heap
garbage being written in its place. This isn't an issue currently, but
will be when we implement support for parallel test execution.

Add a lock to kunit_printer which Printer.print() takes around each
write, and route the other prints that can run concurrently with the
test results through the printer: the kernel start banners and timeout
message in kunit_kernel, and the raw, JSON and JUnit output to stdout.
Prints that only happen before any kernel is started, such as during
configuration and build, are left alone.

No change in output when a single kernel is run at a time.

Signed-off-by: Mark Brown <broonie@kernel.org>
---
 tools/testing/kunit/kunit.py         | 4 ++--
 tools/testing/kunit/kunit_junit.py   | 3 ++-
 tools/testing/kunit/kunit_kernel.py  | 7 ++++---
 tools/testing/kunit/kunit_printer.py | 9 ++++++++-
 4 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 971e70b0fb1a..0132687cdd43 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -272,7 +272,7 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input
 		elif request.raw_output == 'kunit':
 			output = kunit_parser.extract_tap_lines(output)
 		for line in output:
-			print(line.rstrip())
+			stdout.print(line.rstrip())
 		parse_time = time.time() - parse_start
 		return KunitResult(KunitStatus.SUCCESS, parse_time), fake_test
 
@@ -302,7 +302,7 @@ def _write_test_results(request: KunitParseRequest, metadata: kunit_json.Metadat
 					test=test,
 					metadata=metadata)
 		if request.json == 'stdout':
-			print(json_str)
+			stdout.print(json_str)
 		else:
 			with open(request.json, 'w') as f:
 				f.write(json_str)
diff --git a/tools/testing/kunit/kunit_junit.py b/tools/testing/kunit/kunit_junit.py
index 3622070358e7..a15c6061003f 100644
--- a/tools/testing/kunit/kunit_junit.py
+++ b/tools/testing/kunit/kunit_junit.py
@@ -6,6 +6,7 @@
 
 from xml.sax.saxutils import quoteattr, XMLGenerator
 import xml.etree.ElementTree as ET
+import kunit_printer
 from kunit_parser import Test, TestStatus
 from typing import Optional
 
@@ -51,7 +52,7 @@ def get_junit_result(test: Test) -> str:
 def print_junit_result(test: Test) -> None:
 	root_element = get_test_suite(test, None)
 	ET.indent(root_element)
-	ET.dump(root_element)
+	kunit_printer.stdout.print(ET.tostring(root_element, encoding='unicode'))
 
 # Write an entire XML file for the test structure starting at test
 def write_junit_result(test: Test, filename: str) -> None:
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index 1d3226f5a136..508bd62eaa40 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -21,6 +21,7 @@ from typing import Iterator, List, Optional, Tuple, Any
 from types import FrameType
 
 import kunit_config
+import kunit_printer
 import qemu_config
 
 KCONFIG_PATH = '.config'
@@ -132,7 +133,7 @@ class LinuxSourceTreeOperationsQemu(LinuxSourceTreeOperations):
 				'-accel', 'tcg',
 				'-serial', self._serial] + self._extra_qemu_params
 		# Note: shlex.join() does what we want, but requires python 3.8+.
-		print('Running tests with:\n$', ' '.join(shlex.quote(arg) for arg in qemu_command))
+		kunit_printer.stdout.print('Running tests with:\n$ ' + ' '.join(shlex.quote(arg) for arg in qemu_command))
 		return subprocess.Popen(qemu_command,
 					stdin=subprocess.PIPE,
 					stdout=subprocess.PIPE,
@@ -154,7 +155,7 @@ class LinuxSourceTreeOperationsUml(LinuxSourceTreeOperations):
 		"""Runs the Linux UML binary. Must be named 'linux'."""
 		linux_bin = os.path.join(build_dir, 'linux')
 		params.extend(['mem=1G', 'console=tty', 'kunit_shutdown=halt'])
-		print('Running tests with:\n$', linux_bin, ' '.join(shlex.quote(arg) for arg in params))
+		kunit_printer.stdout.print('Running tests with:\n$ ' + linux_bin + ' ' + ' '.join(shlex.quote(arg) for arg in params))
 		return subprocess.Popen([linux_bin] + params,
 					   stdin=subprocess.PIPE,
 					   stdout=subprocess.PIPE,
@@ -402,7 +403,7 @@ class LinuxSourceTree:
 			try:
 				process.wait(timeout=timeout)
 			except Exception as e:
-				print(e)
+				kunit_printer.stdout.print(str(e))
 				process.terminate()
 				process.wait()
 		waiter = threading.Thread(target=_wait_proc)
diff --git a/tools/testing/kunit/kunit_printer.py b/tools/testing/kunit/kunit_printer.py
index ca119f61fe79..440aa7667ece 100644
--- a/tools/testing/kunit/kunit_printer.py
+++ b/tools/testing/kunit/kunit_printer.py
@@ -8,10 +8,16 @@
 
 import datetime
 import sys
+import threading
 import typing
 
 _RESET = '\033[0;0m'
 
+# Python's text streams aren't thread safe: concurrent writes to
+# sys.stdout from several threads can lose output or write garbage. All
+# output from the tool should go through this lock.
+print_lock = threading.Lock()
+
 class Printer:
 	"""Wraps a file object, providing utilities for coloring output, etc."""
 
@@ -25,7 +31,8 @@ class Printer:
 
 	def print(self, message: str) -> None:
 		if self._print:
-			print(message, file=self._output)
+			with print_lock:
+				print(message, file=self._output)
 
 	def print_with_timestamp(self, message: str) -> None:
 		ts = datetime.datetime.now().strftime('%H:%M:%S')

-- 
2.47.3

[PATCH 7/7] kunit: tool: Add --parallel to run suites in concurrent kernels

[Mark Brown]

kunit.py boots a single kernel to run all the selected tests, one after
another on a single CPU. This is good for robustness but means that the
overall time taken to complete testing could be improved. Since we are
already able to split up the set of tests run into multiple kernel boots
with the --run-isolated option let's take advantage of that and allow
execution in parallel.

Add a --parallel N option to the run and exec subcommands. When N is
greater than one the tests are enumerated with kunit.action=list and
split up by top level suite, as --run_isolated=suite does, or by
individual test if --run_isolated=test is also given. The resulting
kernels are run from a thread pool N wide, each logging its console to
test.<suite/test>.log in the build directory.

Each kernel's output is buffered by its worker and handed to the parser
in the order the suites were enumerated in, which is the order the
kernel would have run them, so the results are reported identically
regardless of which kernel happens to finish first. While developing
this I noticed some issues with the existing --run_isolated support due
to test naming, with those fixes in place results were identical with
and without parallelism.

Performance results with --alltests, using the elapsed time reported by
the KUnit runner with an already built kernel on a Raspberry Pi 5 with 4
CPUs and a Graviton 4 with 192 CPUs:

		     default  parallel  per test   parallel per test
  Raspberry Pi 5     67.7s    49.4s     836.4s     326.5s
  Graviton 4         64.1s    30.8s     445.9s      39.7s

Even with the overhead of having to boot more kernels we are seeing
substantial speedups, for obvious reasons these are much better with
more CPUs available.

Signed-off-by: Mark Brown <broonie@kernel.org>
---
 Documentation/dev-tools/kunit/run_wrapper.rst |  11 ++
 tools/testing/kunit/kunit.py                  |  95 +++++++++++----
 tools/testing/kunit/kunit_tool_test.py        | 165 +++++++++++++++++++++-----
 3 files changed, 218 insertions(+), 53 deletions(-)

diff --git a/Documentation/dev-tools/kunit/run_wrapper.rst b/Documentation/dev-tools/kunit/run_wrapper.rst
index cecc110a3399..af5176463966 100644
--- a/Documentation/dev-tools/kunit/run_wrapper.rst
+++ b/Documentation/dev-tools/kunit/run_wrapper.rst
@@ -315,6 +315,17 @@ command line arguments:
   This is useful for debugging a non-hermetic test, one that
   might pass/fail based on what ran before it.
 
+- ``--parallel``: Specifies the number of kernels to run at the same time.
+  The tests are split up by suite (or by test, with ``--run_isolated=test``)
+  and each is run in its own kernel, up to ``N`` at a time. The results are
+  reported in the same order as a sequential run, so the output is stable
+  regardless of which kernel finishes first. Each kernel's console output is
+  saved to its own log in the build directory, named after the suite or test
+  it ran, e.g. ``test.kunit_executor_test.log``. Note that each
+  kernel needs its own memory (1GiB for QEMU), so this is best combined with
+  ``--filter`` or a filter glob to avoid running more kernels than the host
+  can comfortably accommodate.
+
 - ``--raw_output``: If set, generates unformatted output from kernel. Possible options are:
 
    - ``all``: To view the full kernel output, use ``--raw_output=all``.
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 0132687cdd43..4c320ef207df 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -8,6 +8,7 @@
 # Author: Brendan Higgins <brendanhiggins@google.com>
 
 import argparse
+import concurrent.futures
 import os
 import re
 import shlex
@@ -18,7 +19,7 @@ assert sys.version_info >= (3, 7), "Python version is too old"
 
 from dataclasses import dataclass, replace
 from enum import Enum, auto
-from typing import Iterable, List, Optional, Sequence, Tuple
+from typing import Iterable, Iterator, List, Optional, Sequence, Tuple
 
 import kunit_json
 import kunit_junit
@@ -66,6 +67,7 @@ class KunitExecRequest(KunitParseRequest):
 	list_tests: bool
 	list_tests_attr: bool
 	list_suites: bool
+	parallel: int
 
 @dataclass
 class KunitRequest(KunitExecRequest, KunitBuildRequest):
@@ -166,6 +168,52 @@ def _suites_from_test_list(tests: List[str]) -> List[str]:
 			suites.append(suite)
 	return suites
 
+def _run_kernels(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest,
+		 filter_globs: List[str]) -> Iterator[Iterable[str]]:
+	"""Boots a kernel for each filter glob and yields each kernel's output.
+
+	Outputs are always yielded in the order of filter_globs, so the
+	results are reported in the same order whether the kernels are run
+	one after another or (with --parallel) several at a time.
+	"""
+	num_kernels = len(filter_globs)
+	workers = min(request.parallel, num_kernels)
+	if workers <= 1:
+		for i, filter_glob in enumerate(filter_globs):
+			stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, num_kernels))
+			yield linux.run_kernel(
+				args=request.kernel_args,
+				timeout=request.timeout,
+				filter_glob=filter_glob,
+				filter=request.filter,
+				filter_action=request.filter_action,
+				build_dir=request.build_dir)
+		return
+
+	# Each kernel's output is buffered by a worker thread and only
+	# handed back once all the kernels before it in filter_globs have
+	# been reported, so which kernel finishes first doesn't matter.
+	# The kernels write to separate log files named after the suite or
+	# test they ran, e.g. test.<suite>.log.
+	def run(filter_glob: str) -> List[str]:
+		return list(linux.run_kernel(
+			args=request.kernel_args,
+			timeout=request.timeout,
+			filter_glob=filter_glob,
+			filter=request.filter,
+			filter_action=request.filter_action,
+			build_dir=request.build_dir,
+			outfile=kunit_kernel.get_outfile_path(request.build_dir, filter_glob)))
+
+	stdout.print_with_timestamp('Starting {} KUnit Kernels, {} at a time...'.format(num_kernels, workers))
+	with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
+		futures = [pool.submit(run, filter_glob) for filter_glob in filter_globs]
+		for i, (filter_glob, future) in enumerate(zip(filter_globs, futures)):
+			output = future.result()
+			stdout.print_with_timestamp('Results from KUnit Kernel ({}/{}) for {}, console log in {}:'.format(
+				i+1, num_kernels, filter_glob, kunit_kernel.get_outfile_path(request.build_dir, filter_glob)))
+			yield output
+
 def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
 	filter_globs = [request.filter_glob]
 	if request.list_tests:
@@ -184,11 +232,14 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 		for line in output:
 			print(line.rstrip())
 		return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0)
-	if request.run_isolated:
+	if request.run_isolated or request.parallel > 1:
 		tests = _list_tests(linux, request)
 		if request.run_isolated == 'test':
 			filter_globs = tests
-		elif request.run_isolated == 'suite':
+		else:
+			# One kernel per suite: either --run_isolated=suite, or
+			# --parallel, which splits the tests up by suite so that
+			# each suite can be run in a separate kernel.
 			filter_globs = _suites_from_test_list(tests)
 			# Apply the test-part of the user's glob, if present.
 			if '.' in request.filter_glob:
@@ -203,26 +254,9 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 	report.counts = test_counts
 	parse_request = replace(request, json=None, junit=None)
 	test_failed = False
-	exec_time = 0.0
-	for i, filter_glob in enumerate(filter_globs):
-		stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs)))
-
-		test_start = time.time()
-		run_result = linux.run_kernel(
-			args=request.kernel_args,
-			timeout=request.timeout,
-			filter_glob=filter_glob,
-			filter=request.filter,
-			filter_action=request.filter_action,
-			build_dir=request.build_dir)
-
+	exec_start = time.time()
+	for run_result in _run_kernels(linux, request, filter_globs):
 		_, test_result = parse_tests(parse_request, metadata, run_result)
-		# run_kernel() doesn't block on the kernel exiting.
-		# That only happens after we get the last line of output from `run_result`.
-		# So exec_time here actually contains parsing + execution time, which is fine.
-		test_end = time.time()
-		exec_time += test_end - test_start
-
 		test_counts.add_subtest_counts(test_result.counts)
 		# A kernel with no results has no failed test counts to aggregate.
 		if _map_to_overall_status(test_result.status) != KunitStatus.SUCCESS:
@@ -240,6 +274,10 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 			report.status = kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS
 	if not request.raw_output:
 		_write_test_results(request, metadata, report)
+	# run_kernel() doesn't block on the kernel exiting.
+	# That only happens after we get the last line of output from `run_result`.
+	# So exec_time here actually contains parsing + execution time, which is fine.
+	exec_time = time.time() - exec_start
 
 	if len(filter_globs) == 1 and test_counts.crashed > 0:
 		bd = request.build_dir
@@ -492,6 +530,13 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
 	parser.add_argument('--list_suites', help='If set, list all suites that will be '
 			    'run.',
 			    action='store_true')
+	parser.add_argument('--parallel',
+			    help='Number of kernels to run at the same time. The tests '
+			    'are split up by suite (or by test with --run_isolated=test) '
+			    'and each suite is run in its own kernel, with the results '
+			    'reported in the same order as a sequential run. Defaults '
+			    'to 1, i.e. a single kernel running all the tests.',
+			    type=int, default=1, metavar='N')
 
 def add_parse_opts(parser: argparse.ArgumentParser) -> None:
 	parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. '
@@ -562,7 +607,8 @@ def run_handler(cli_args: argparse.Namespace) -> None:
 					run_isolated=cli_args.run_isolated,
 					list_tests=cli_args.list_tests,
 					list_tests_attr=cli_args.list_tests_attr,
-					list_suites=cli_args.list_suites)
+					list_suites=cli_args.list_suites,
+					parallel=cli_args.parallel)
 	result = run_tests(linux, request)
 	if result.status != KunitStatus.SUCCESS:
 		sys.exit(1)
@@ -613,7 +659,8 @@ def exec_handler(cli_args: argparse.Namespace) -> None:
 					run_isolated=cli_args.run_isolated,
 					list_tests=cli_args.list_tests,
 					list_tests_attr=cli_args.list_tests_attr,
-					list_suites=cli_args.list_suites)
+					list_suites=cli_args.list_suites,
+					parallel=cli_args.parallel)
 	result = exec_tests(linux, exec_request)
 	stdout.print_with_timestamp((
 		'Elapsed time: %.3fs\n') % (result.elapsed_time))
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index c4999273e491..beb80c728004 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -19,6 +19,7 @@ import signal
 import subprocess
 import sys
 import textwrap
+import time
 from typing import Iterable
 import xml.etree.ElementTree as ET
 
@@ -1019,7 +1020,7 @@ class KUnitMainTest(unittest.TestCase):
 		self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
 
 		got = kunit._list_tests(self.linux_source_mock,
-				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False))
+				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False, 1))
 		self.assertEqual(got, want)
 		# Should respect the user's filter glob when listing tests.
 		self.linux_source_mock.run_kernel.assert_called_once_with(
@@ -1034,7 +1035,7 @@ class KUnitMainTest(unittest.TestCase):
 			'[    0.300000] suite2.test1']
 
 		got = kunit._list_tests(self.linux_source_mock,
-				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False))
+				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False, 1))
 		self.assertEqual(got, want)
 
 	@mock.patch.object(kunit, '_list_tests')
@@ -1044,7 +1045,7 @@ class KUnitMainTest(unittest.TestCase):
 
 		# Should respect the user's filter glob when listing tests.
 		mock_tests.assert_called_once_with(mock.ANY,
-				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, 'suite', False, False, False))
+				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, 'suite', False, False, False, 1))
 		self.linux_source_mock.run_kernel.assert_has_calls([
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter='', filter_action=None, timeout=300),
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter='', filter_action=None, timeout=300),
@@ -1057,7 +1058,7 @@ class KUnitMainTest(unittest.TestCase):
 
 		# Should respect the user's filter glob when listing tests.
 		mock_tests.assert_called_once_with(mock.ANY,
-				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'test', False, False, False))
+				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'test', False, False, False, 1))
 		self.linux_source_mock.run_kernel.assert_has_calls([
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter='', filter_action=None, timeout=300),
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter='', filter_action=None, timeout=300),
@@ -1065,35 +1066,115 @@ class KUnitMainTest(unittest.TestCase):
 		])
 
 	@mock.patch.object(kunit, '_list_tests')
-	def test_isolated_kernel_without_results_fails(self, mock_tests):
+	def test_parallel_by_suite(self, mock_tests):
+		mock_tests.return_value = ['suite.test1', 'suite.test2', 'suite2.test1']
+		kunit.main(['exec', '--parallel=2', 'suite*.test*'])
+
+		# Should split by suite, respecting the user's filter glob.
+		mock_tests.assert_called_once_with(mock.ANY,
+				     kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, None, False, False, False, 2))
+		# Each kernel gets its own log file, named after its suite.
+		# The kernels run concurrently, so don't depend on the order
+		# they start in.
+		self.assertEqual(self.linux_source_mock.run_kernel.call_count, 2)
+		self.linux_source_mock.run_kernel.assert_has_calls([
+			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter='', filter_action=None, timeout=300, outfile='.kunit/test.suite.test_.log'),
+			mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter='', filter_action=None, timeout=300, outfile='.kunit/test.suite2.test_.log'),
+		], any_order=True)
+		self.print_mock.assert_any_call(StrContains('Starting 2 KUnit Kernels, 2 at a time...'))
+		self.print_mock.assert_any_call(StrContains('Results from KUnit Kernel (1/2) for suite.test*, console log in .kunit/test.suite.test_.log:'))
+		self.print_mock.assert_any_call(StrContains('Results from KUnit Kernel (2/2) for suite2.test*, console log in .kunit/test.suite2.test_.log:'))
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_parallel_by_test(self, mock_tests):
+		mock_tests.return_value = ['suite.test1', 'suite.test2', 'suite2.test1']
+		kunit.main(['exec', '--parallel=2', '--run_isolated=test', 'suite*'])
+
+		self.assertEqual(self.linux_source_mock.run_kernel.call_count, 3)
+		self.linux_source_mock.run_kernel.assert_has_calls([
+			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter='', filter_action=None, timeout=300, outfile='.kunit/test.suite.test1.log'),
+			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter='', filter_action=None, timeout=300, outfile='.kunit/test.suite.test2.log'),
+			mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter='', filter_action=None, timeout=300, outfile='.kunit/test.suite2.test1.log'),
+		], any_order=True)
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_parallel_single_suite_runs_one_kernel(self, mock_tests):
+		# With only one suite there's nothing to parallelise, so this
+		# should behave exactly like a sequential run.
+		mock_tests.return_value = ['suite.test1', 'suite.test2']
+		kunit.main(['exec', '--parallel=4', 'suite'])
+
+		self.linux_source_mock.run_kernel.assert_called_once_with(
+			args=None, build_dir='.kunit', filter_glob='suite', filter='', filter_action=None, timeout=300)
+		self.print_mock.assert_any_call(StrContains('Starting KUnit Kernel (1/1)...'))
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_parallel_stable_order(self, mock_tests):
+		mock_tests.return_value = ['suite1.test1', 'suite2.test1', 'suite3.test1']
+
+		# Make the kernels finish in the reverse of the order they're
+		# listed in: results must still come out in the listed order.
+		def fake_run_kernel(filter_glob, **kwargs):
+			delay = {'suite1': 0.2, 'suite2': 0.1, 'suite3': 0.0}[filter_glob]
+			time.sleep(delay)
+			return ['TAP version 14', '1..1', 'ok 1 - ' + filter_glob]
+		self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+		kunit.main(['exec', '--parallel=3', '--raw_output=kunit'])
+
+		output = [call.args[0] for call in self.print_mock.call_args_list]
+		want = ['ok 1 - suite1', 'ok 1 - suite2', 'ok 1 - suite3']
+		self.assertEqual([l for l in output if l.startswith('ok')], want)
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_parallel_counts_all_kernels(self, mock_tests):
 		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
-		for missing_output in ([], ['KTAP version 1', '1..0']):
-			for missing_suite in ('suite1', 'suite2'):
-				with self.subTest(output=missing_output,
-						 suite=missing_suite):
-					def fake_run_kernel(filter_glob, **kwargs):
-						if filter_glob == missing_suite:
-							return missing_output
-						return ['KTAP version 1', '1..1', 'ok 1 - test1']
-					self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
 
-					with self.assertRaises(SystemExit) as e:
-						kunit.main(['exec', '--run_isolated=suite'])
-					self.assertEqual(e.exception.code, 1)
+		def fake_run_kernel(filter_glob, **kwargs):
+			if filter_glob == 'suite2':
+				return ['TAP version 14', '1..1', 'not ok 1 - suite2']
+			return ['TAP version 14', '1..1', 'ok 1 - suite1']
+		self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+		# A failure in any of the kernels must fail the overall run.
+		with self.assertRaises(SystemExit) as e:
+			kunit.main(['exec', '--parallel=2'])
+		self.assertEqual(e.exception.code, 1)
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_isolated_kernel_without_results_fails(self, mock_tests):
+		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
+		for parallel in (1, 2):
+			for missing_output in ([], ['KTAP version 1', '1..0']):
+				for missing_suite in ('suite1', 'suite2'):
+					with self.subTest(parallel=parallel, output=missing_output,
+							 suite=missing_suite):
+						def fake_run_kernel(filter_glob, **kwargs):
+							if filter_glob == missing_suite:
+								return missing_output
+							return ['KTAP version 1', '1..1', 'ok 1 - test1']
+						self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+
+						with self.assertRaises(SystemExit) as e:
+							kunit.main(['exec', '--run_isolated=suite',
+								    '--parallel=' + str(parallel)])
+						self.assertEqual(e.exception.code, 1)
 
 	@mock.patch.object(kunit, '_list_tests')
 	def test_isolated_skipped_kernels_succeed(self, mock_tests):
 		mock_tests.return_value = ['suite1.test1', 'suite2.test1']
-		for all_skipped in (False, True):
-			with self.subTest(all_skipped=all_skipped):
-				def fake_run_kernel(filter_glob, **kwargs):
-					result = 'ok 1 - test1'
-					if all_skipped or filter_glob == 'suite1':
-						result += ' # SKIP unavailable'
-					return ['KTAP version 1', '1..1', result]
-				self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
+		for parallel in (1, 2):
+			for all_skipped in (False, True):
+				with self.subTest(parallel=parallel, all_skipped=all_skipped):
+					def fake_run_kernel(filter_glob, **kwargs):
+						result = 'ok 1 - test1'
+						if all_skipped or filter_glob == 'suite1':
+							result += ' # SKIP unavailable'
+						return ['KTAP version 1', '1..1', result]
+					self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
 
-				kunit.main(['exec', '--run_isolated=suite'])
+					kunit.main(['exec', '--run_isolated=suite',
+						    '--parallel=' + str(parallel)])
 
 	@mock.patch.object(kunit, '_list_tests')
 	def test_isolated_reports_include_all_kernels(self, mock_tests):
@@ -1109,8 +1190,8 @@ class KUnitMainTest(unittest.TestCase):
 				'    1..1', f'    {status} 1 - {case}', f'{status} 1 - {suite}']
 		self.linux_source_mock.run_kernel.side_effect = fake_run_kernel
 
-		for isolation in ('suite', 'test'):
-			with self.subTest(isolation=isolation), \
+		for parallel, isolation in itertools.product((1, 2), ('suite', 'test')):
+			with self.subTest(parallel=parallel, isolation=isolation), \
 			     tempfile.TemporaryDirectory() as output_dir:
 				if isolation == 'suite':
 					mock_tests.return_value = ['suite1.test', 'suite2.test']
@@ -1121,7 +1202,8 @@ class KUnitMainTest(unittest.TestCase):
 				json_path = os.path.join(output_dir, 'results.json')
 				xml_path = os.path.join(output_dir, 'results.xml')
 				with self.assertRaises(SystemExit) as e:
-					kunit.main(['exec', '--run_isolated=' + isolation,
+					kunit.main(['exec', '--parallel=' + str(parallel),
+						    '--run_isolated=' + isolation,
 						    '--json=' + json_path, '--junit=' + xml_path])
 				self.assertEqual(e.exception.code, 1)
 
@@ -1139,6 +1221,31 @@ class KUnitMainTest(unittest.TestCase):
 					'FAIL' if case.find('failure') is not None else 'PASS')
 					for suite in root for case in suite], expected)
 
+	@mock.patch.object(kunit, '_list_tests')
+	def test_parallel_reports_to_stdout(self, mock_tests):
+		mock_tests.return_value = ['suite1.test', 'suite2.test']
+		self.linux_source_mock.run_kernel.side_effect = lambda filter_glob, **kwargs: [
+			'KTAP version 1', '1..1', 'ok 1 - ' + filter_glob]
+		kunit.main(['exec', '--parallel=2', '--json', '--junit'])
+		output = [call.args[0] for call in self.print_mock.call_args_list]
+		json_reports = [json.loads(line) for line in output if line.startswith('{')]
+		xml_reports = [ET.fromstring(line) for line in output if line.startswith('<testsuite')]
+		self.assertEqual(len(json_reports), 1)
+		self.assertEqual(json_reports[0]['misc']['tests'], 2)
+		self.assertEqual(len(xml_reports), 1)
+		self.assertEqual(xml_reports[0].get('tests'), '2')
+
+	@mock.patch.object(kunit, '_list_tests')
+	def test_parallel_raw_output_does_not_write_reports(self, mock_tests):
+		mock_tests.return_value = ['suite1.test', 'suite2.test']
+		with tempfile.TemporaryDirectory() as output_dir:
+			json_path = os.path.join(output_dir, 'results.json')
+			xml_path = os.path.join(output_dir, 'results.xml')
+			kunit.main(['exec', '--parallel=2', '--raw_output=all',
+				    '--json=' + json_path, '--junit=' + xml_path])
+			self.assertFalse(os.path.exists(json_path))
+			self.assertFalse(os.path.exists(xml_path))
+
 	def test_single_kernel_reports_match_parse(self):
 		with tempfile.TemporaryDirectory() as output_dir:
 			json_path = os.path.join(output_dir, 'results.json')

-- 
2.47.3