[igt-dev] [PATCH 1/3] runner: Normalize testlist entries that don't list subtests

[igt-dev] [PATCH 1/3] runner: Normalize testlist entries that don't list subtests

[Petri Latvala]

As the syntax for "all subtests" and "test that doesn't have subtests"
is the same, check the subtest listing when building the execution
plan. Doing that makes it possible to still have "all subtests" in
testlists (albeit not originally designed to support that) and enables
blocklisting particular subtests.

Signed-off-by: Petri Latvala <adrinael@adrinael.net>
Cc: Mauro Carvalho Chehab <mchehab@kernel.org>
Cc: Arkadiusz Hiler <arek@hiler.eu>
Cc: Kamil Konieczny <kamil.konieczny@linux.intel.com>
Cc: Juha-Pekka Heikkila <juhapekka.heikkila@gmail.com>
Cc: Bhanuprakash Modem <bhanuprakash.modem@intel.com>
Cc: Ashutosh Dixit <ashutosh.dixit@intel.com>
Closes: https://gitlab.freedesktop.org/drm/igt-gpu-tools/-/issues/149
---
 runner/job_list.c | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/runner/job_list.c b/runner/job_list.c
index e6ea83631..27cbb10bc 100644
--- a/runner/job_list.c
+++ b/runner/job_list.c
@@ -230,8 +230,28 @@ static bool job_list_from_test_list(struct job_list *job_list,
 			continue;
 
 		if (sscanf(line, "igt@%ms", &binary) == 1) {
-			if ((delim = strchr(binary, '@')) != NULL)
+			if ((delim = strchr(binary, '@')) != NULL) {
 				*delim++ = '\0';
+			} else {
+				/*
+				 * No subtests specified. Check
+				 * whether the user means "all
+				 * subtests" or if the test doesn't
+				 * have any.
+				 */
+				if (entry.binary) {
+					/* First flush the entry we're building for multiple-mode */
+					add_job_list_entry(job_list, entry.binary, entry.subtests, entry.subtest_count);
+					memset(&entry, 0, sizeof(entry));
+					any = true;
+				}
+
+				add_subtests(job_list, settings, binary,
+					     &settings->include_regexes,
+					     &settings->exclude_regexes);
+				any = true;
+				continue;
+			}
 
 			if (!settings->multiple_mode) {
 				char **subtests = NULL;
-- 
2.39.2

[igt-dev] [PATCH 2/3] runner/runner_tests: Unit tests for binary-name-only testlist entries

[Petri Latvala]

Signed-off-by: Petri Latvala <adrinael@adrinael.net>
---
 runner/runner_tests.c | 70 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index a7e968f85..e6024de11 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -1810,6 +1810,76 @@ igt_main
 		}
 	}
 
+	igt_subtest_group {
+		const char testlisttext[] = "igt@successtest";
+		const char blocktext[] = "igt@successtest@first";
+		struct job_list *list = malloc(sizeof(*list));
+		volatile int dirfd = -1;
+		char dirname[] = "tmpdirXXXXXX";
+		volatile int listfd;
+		volatile int blockfd;
+		char listfilename[] = "tmplistXXXXXX";
+		char blockfilename[] = "tmpblockXXXXXX";
+
+		igt_fixture {
+			igt_require(mkdtemp(dirname) != NULL);
+			rmdir(dirname);
+
+			igt_require((listfd = mkstemp(listfilename)) >= 0);
+			igt_require(write(listfd, testlisttext, strlen(testlisttext)) == strlen(testlisttext));
+			igt_require((blockfd = mkstemp(blockfilename)) >= 0);
+			igt_require(write(blockfd, blocktext, strlen(blocktext)) == strlen(blocktext));
+			close(listfd);
+			close(blockfd);
+
+			init_job_list(list);
+		}
+
+		igt_subtest("only-binary-name-in-testlist") {
+			const char *argv[] = { "runner",
+				"--allow-non-root",
+				"--test-list", listfilename,
+				testdatadir,
+				dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+
+			igt_assert(create_job_list(list, settings));
+			/* No multi-mode, binary has two subtests, should be normalized to have individual subtests */
+			igt_assert_eq(list->size, 2);
+			igt_assert_eq(list->entries[0].subtest_count, 1);
+			igt_assert_eq(list->entries[1].subtest_count, 1);
+		}
+
+		igt_subtest("only-binary-name-in-testlist-with-blocks") {
+			const char *argv[] = { "runner",
+				"--allow-non-root",
+				"--test-list", listfilename,
+				"-b", blockfilename,
+				testdatadir,
+				dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+
+			igt_assert(create_job_list(list, settings));
+			/* No multi-mode, binary has two subtests, one of them blocked */
+			igt_assert_eq(list->size, 1);
+			igt_assert_eq(list->entries[0].subtest_count, 1);
+			igt_assert_eqstr(list->entries[0].subtests[0], "second-subtest");
+		}
+
+		igt_fixture {
+			unlink(listfilename);
+			unlink(blockfilename);
+			close(dirfd);
+			clear_directory(dirname);
+			free_job_list(list);
+			free(list);
+		}
+	}
+
 	igt_subtest_group {
 		struct job_list *list = malloc(sizeof(*list));
 		volatile int dirfd = -1;
-- 
2.39.2

Re: [igt-dev] [PATCH 2/3] runner/runner_tests: Unit tests for binary-name-only testlist entries

[Mauro Carvalho Chehab]

On Fri, 20 Oct 2023 16:30:40 +0300
Petri Latvala <adrinael@adrinael.net> wrote:

> Signed-off-by: Petri Latvala <adrinael@adrinael.net>

Acked-by: Mauro Carvalho Chehab <mchehab@kernel.org>

> ---
>  runner/runner_tests.c | 70 +++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 70 insertions(+)
> 
> diff --git a/runner/runner_tests.c b/runner/runner_tests.c
> index a7e968f85..e6024de11 100644
> --- a/runner/runner_tests.c
> +++ b/runner/runner_tests.c
> @@ -1810,6 +1810,76 @@ igt_main
>  		}
>  	}
>  
> +	igt_subtest_group {
> +		const char testlisttext[] = "igt@successtest";
> +		const char blocktext[] = "igt@successtest@first";
> +		struct job_list *list = malloc(sizeof(*list));
> +		volatile int dirfd = -1;
> +		char dirname[] = "tmpdirXXXXXX";
> +		volatile int listfd;
> +		volatile int blockfd;
> +		char listfilename[] = "tmplistXXXXXX";
> +		char blockfilename[] = "tmpblockXXXXXX";
> +
> +		igt_fixture {
> +			igt_require(mkdtemp(dirname) != NULL);
> +			rmdir(dirname);
> +
> +			igt_require((listfd = mkstemp(listfilename)) >= 0);
> +			igt_require(write(listfd, testlisttext, strlen(testlisttext)) == strlen(testlisttext));
> +			igt_require((blockfd = mkstemp(blockfilename)) >= 0);
> +			igt_require(write(blockfd, blocktext, strlen(blocktext)) == strlen(blocktext));
> +			close(listfd);
> +			close(blockfd);
> +
> +			init_job_list(list);
> +		}
> +
> +		igt_subtest("only-binary-name-in-testlist") {
> +			const char *argv[] = { "runner",
> +				"--allow-non-root",
> +				"--test-list", listfilename,
> +				testdatadir,
> +				dirname,
> +			};
> +
> +			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
> +
> +			igt_assert(create_job_list(list, settings));
> +			/* No multi-mode, binary has two subtests, should be normalized to have individual subtests */
> +			igt_assert_eq(list->size, 2);
> +			igt_assert_eq(list->entries[0].subtest_count, 1);
> +			igt_assert_eq(list->entries[1].subtest_count, 1);
> +		}
> +
> +		igt_subtest("only-binary-name-in-testlist-with-blocks") {
> +			const char *argv[] = { "runner",
> +				"--allow-non-root",
> +				"--test-list", listfilename,
> +				"-b", blockfilename,
> +				testdatadir,
> +				dirname,
> +			};
> +
> +			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
> +
> +			igt_assert(create_job_list(list, settings));
> +			/* No multi-mode, binary has two subtests, one of them blocked */
> +			igt_assert_eq(list->size, 1);
> +			igt_assert_eq(list->entries[0].subtest_count, 1);
> +			igt_assert_eqstr(list->entries[0].subtests[0], "second-subtest");
> +		}
> +
> +		igt_fixture {
> +			unlink(listfilename);
> +			unlink(blockfilename);
> +			close(dirfd);
> +			clear_directory(dirname);
> +			free_job_list(list);
> +			free(list);
> +		}
> +	}
> +
>  	igt_subtest_group {
>  		struct job_list *list = malloc(sizeof(*list));
>  		volatile int dirfd = -1;

Re: [igt-dev] [PATCH 2/3] runner/runner_tests: Unit tests for binary-name-only testlist entries

[Kamil Konieczny]

Hi Petri,

Please add a description here, see checkpatch.pl:

WARNING: Missing commit description - Add an appropriate one

On 2023-10-20 at 16:30:40 +0300, Petri Latvala wrote:
> Signed-off-by: Petri Latvala <adrinael@adrinael.net>
> ---
>  runner/runner_tests.c | 70 +++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 70 insertions(+)
> 
> diff --git a/runner/runner_tests.c b/runner/runner_tests.c
> index a7e968f85..e6024de11 100644
> --- a/runner/runner_tests.c
> +++ b/runner/runner_tests.c
> @@ -1810,6 +1810,76 @@ igt_main
>  		}
>  	}
>  
> +	igt_subtest_group {
> +		const char testlisttext[] = "igt@successtest";

WARNING: const array should probably be static const

> +		const char blocktext[] = "igt@successtest@first";
Same here.

> +		struct job_list *list = malloc(sizeof(*list));
> +		volatile int dirfd = -1;

WARNING: Use of volatile is usually wrong: see Documentation/process/volatile-considered-harmful.rst
Maybe static?

> +		char dirname[] = "tmpdirXXXXXX";
> +		volatile int listfd;
--------^
Same here.

> +		volatile int blockfd;
--------^
Same here.

> +		char listfilename[] = "tmplistXXXXXX";
> +		char blockfilename[] = "tmpblockXXXXXX";
> +
> +		igt_fixture {
> +			igt_require(mkdtemp(dirname) != NULL);
> +			rmdir(dirname);
> +
> +			igt_require((listfd = mkstemp(listfilename)) >= 0);
> +			igt_require(write(listfd, testlisttext, strlen(testlisttext)) == strlen(testlisttext));
> +			igt_require((blockfd = mkstemp(blockfilename)) >= 0);
> +			igt_require(write(blockfd, blocktext, strlen(blocktext)) == strlen(blocktext));
> +			close(listfd);
> +			close(blockfd);
> +
> +			init_job_list(list);
> +		}
> +
> +		igt_subtest("only-binary-name-in-testlist") {
> +			const char *argv[] = { "runner",

WARNING: char * array declaration might be better as static const

> +				"--allow-non-root",
> +				"--test-list", listfilename,
> +				testdatadir,
> +				dirname,
> +			};
> +
> +			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));

ERROR: "(foo**)" should be "(foo **)"

> +
> +			igt_assert(create_job_list(list, settings));
> +			/* No multi-mode, binary has two subtests, should be normalized to have individual subtests */
> +			igt_assert_eq(list->size, 2);
> +			igt_assert_eq(list->entries[0].subtest_count, 1);
> +			igt_assert_eq(list->entries[1].subtest_count, 1);
> +		}
> +
> +		igt_subtest("only-binary-name-in-testlist-with-blocks") {
> +			const char *argv[] = { "runner",

WARNING: char * array declaration might be better as static const

> +				"--allow-non-root",
> +				"--test-list", listfilename,
> +				"-b", blockfilename,
> +				testdatadir,
> +				dirname,
> +			};
> +
> +			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));

ERROR: "(foo**)" should be "(foo **)"

Looks ok, tests passes.

Regards,
Kamil

> +
> +			igt_assert(create_job_list(list, settings));
> +			/* No multi-mode, binary has two subtests, one of them blocked */
> +			igt_assert_eq(list->size, 1);
> +			igt_assert_eq(list->entries[0].subtest_count, 1);
> +			igt_assert_eqstr(list->entries[0].subtests[0], "second-subtest");
> +		}
> +
> +		igt_fixture {
> +			unlink(listfilename);
> +			unlink(blockfilename);
> +			close(dirfd);
> +			clear_directory(dirname);
> +			free_job_list(list);
> +			free(list);
> +		}
> +	}
> +
>  	igt_subtest_group {
>  		struct job_list *list = malloc(sizeof(*list));
>  		volatile int dirfd = -1;
> -- 
> 2.39.2
> 

[igt-dev] [PATCH 3/3] runner/runner_tests: Fix name of no-subtests in unit tests

[Petri Latvala]

Now that runner normalizes no-subtest entries in testlists, make sure
the name matches what's available.

Signed-off-by: Petri Latvala <adrinael@adrinael.net>
---
 runner/runner_tests.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index e6024de11..11ff05bc0 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -802,7 +802,7 @@ igt_main
 		char filename[] = "tmplistXXXXXX";
 		const char testlisttext[] = "igt@successtest@first-subtest\n"
 			"igt@successtest@second-subtest\n"
-			"igt@nosubtests\n";
+			"igt@no-subtests\n";
 		int multiple;
 		struct job_list *list = malloc(sizeof(*list));
 
@@ -830,7 +830,7 @@ igt_main
 
 				igt_assert_eqstr(list->entries[0].binary, "successtest");
 				if (!multiple) igt_assert_eqstr(list->entries[1].binary, "successtest");
-				igt_assert_eqstr(list->entries[multiple ? 1 : 2].binary, "nosubtests");
+				igt_assert_eqstr(list->entries[multiple ? 1 : 2].binary, "no-subtests");
 
 				igt_assert_eq(list->entries[0].subtest_count, multiple ? 2 : 1);
 				igt_assert_eq(list->entries[1].subtest_count, multiple ? 0 : 1);
-- 
2.39.2

Re: [igt-dev] [PATCH 3/3] runner/runner_tests: Fix name of no-subtests in unit tests

[Mauro Carvalho Chehab]

On Fri, 20 Oct 2023 16:30:41 +0300
Petri Latvala <adrinael@adrinael.net> wrote:

> Now that runner normalizes no-subtest entries in testlists, make sure
> the name matches what's available.
> 
> Signed-off-by: Petri Latvala <adrinael@adrinael.net>

Acked-by: Mauro Carvalho Chehab <mchehab@kernel.org>

> ---
>  runner/runner_tests.c | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/runner/runner_tests.c b/runner/runner_tests.c
> index e6024de11..11ff05bc0 100644
> --- a/runner/runner_tests.c
> +++ b/runner/runner_tests.c
> @@ -802,7 +802,7 @@ igt_main
>  		char filename[] = "tmplistXXXXXX";
>  		const char testlisttext[] = "igt@successtest@first-subtest\n"
>  			"igt@successtest@second-subtest\n"
> -			"igt@nosubtests\n";
> +			"igt@no-subtests\n";
>  		int multiple;
>  		struct job_list *list = malloc(sizeof(*list));
>  
> @@ -830,7 +830,7 @@ igt_main
>  
>  				igt_assert_eqstr(list->entries[0].binary, "successtest");
>  				if (!multiple) igt_assert_eqstr(list->entries[1].binary, "successtest");
> -				igt_assert_eqstr(list->entries[multiple ? 1 : 2].binary, "nosubtests");
> +				igt_assert_eqstr(list->entries[multiple ? 1 : 2].binary, "no-subtests");
>  
>  				igt_assert_eq(list->entries[0].subtest_count, multiple ? 2 : 1);
>  				igt_assert_eq(list->entries[1].subtest_count, multiple ? 0 : 1);

[igt-dev] ✓ Fi.CI.BAT: success for series starting with [1/3] runner: Normalize testlist entries that don't list subtests

[Patchwork]

[-- Attachment #1: Type: text/plain, Size: 1912 bytes --]

== Series Details ==

Series: series starting with [1/3] runner: Normalize testlist entries that don't list subtests
URL   : https://patchwork.freedesktop.org/series/125401/
State : success

== Summary ==

CI Bug Log - changes from IGT_7551 -> IGTPW_10041
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_10041/index.html

Participating hosts (36 -> 34)
------------------------------

  Missing    (2): fi-hsw-4770 fi-bsw-n3050 

Known issues
------------

  Here are the changes found in IGTPW_10041 that come from known issues:

### IGT changes ###

#### Possible fixes ####

  * igt@kms_pipe_crc_basic@read-crc-frame-sequence@pipe-d-edp-1:
    - bat-rplp-1:         [ABORT][1] ([i915#8668]) -> [PASS][2]
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/IGT_7551/bat-rplp-1/igt@kms_pipe_crc_basic@read-crc-frame-sequence@pipe-d-edp-1.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_10041/bat-rplp-1/igt@kms_pipe_crc_basic@read-crc-frame-sequence@pipe-d-edp-1.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [i915#8668]: https://gitlab.freedesktop.org/drm/intel/issues/8668


Build changes
-------------

  * CI: CI-20190529 -> None
  * IGT: IGT_7551 -> IGTPW_10041

  CI-20190529: 20190529
  CI_DRM_13777: 1b4fd688d213556268c50f853746c94c9a0cfee7 @ git://anongit.freedesktop.org/gfx-ci/linux
  IGTPW_10041: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_10041/index.html
  IGT_7551: 15e7d92ca5f98d10feffa27a76724c33cbd68da5 @ https://gitlab.freedesktop.org/drm/igt-gpu-tools.git


Testlist changes
----------------

-igt@xe_create@multigpu-create-massive-size

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_10041/index.html

[-- Attachment #2: Type: text/html, Size: 2509 bytes --]

[igt-dev] ✓ CI.xeBAT: success for series starting with [1/3] runner: Normalize testlist entries that don't list subtests

[Patchwork]

[-- Attachment #1: Type: text/plain, Size: 2650 bytes --]

== Series Details ==

Series: series starting with [1/3] runner: Normalize testlist entries that don't list subtests
URL   : https://patchwork.freedesktop.org/series/125401/
State : success

== Summary ==

CI Bug Log - changes from XEIGT_7551_BAT -> XEIGTPW_10041_BAT
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  

Participating hosts (4 -> 4)
------------------------------

  No changes in participating hosts

Known issues
------------

  Here are the changes found in XEIGTPW_10041_BAT that come from known issues:

### IGT changes ###

#### Possible fixes ####

  * igt@kms_flip@basic-flip-vs-wf_vblank@c-edp1:
    - bat-adlp-7:         [FAIL][1] ([Intel XE#480]) -> [PASS][2]
   [1]: https://intel-gfx-ci.01.org/tree/intel-xe/IGT_7551/bat-adlp-7/igt@kms_flip@basic-flip-vs-wf_vblank@c-edp1.html
   [2]: https://intel-gfx-ci.01.org/tree/intel-xe/IGTPW_10041/bat-adlp-7/igt@kms_flip@basic-flip-vs-wf_vblank@c-edp1.html

  * {igt@xe_create@create-execqueues-noleak}:
    - bat-adlp-7:         [FAIL][3] ([Intel XE#524]) -> [PASS][4]
   [3]: https://intel-gfx-ci.01.org/tree/intel-xe/IGT_7551/bat-adlp-7/igt@xe_create@create-execqueues-noleak.html
   [4]: https://intel-gfx-ci.01.org/tree/intel-xe/IGTPW_10041/bat-adlp-7/igt@xe_create@create-execqueues-noleak.html

  
#### Warnings ####

  * igt@xe_exec_fault_mode@twice-invalid-fault:
    - bat-pvc-2:          [INCOMPLETE][5] ([Intel XE#613] / [Intel XE#814]) -> [INCOMPLETE][6] ([Intel XE#613])
   [5]: https://intel-gfx-ci.01.org/tree/intel-xe/IGT_7551/bat-pvc-2/igt@xe_exec_fault_mode@twice-invalid-fault.html
   [6]: https://intel-gfx-ci.01.org/tree/intel-xe/IGTPW_10041/bat-pvc-2/igt@xe_exec_fault_mode@twice-invalid-fault.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [Intel XE#480]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/480
  [Intel XE#524]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/524
  [Intel XE#613]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/613
  [Intel XE#814]: https://gitlab.freedesktop.org/drm/xe/kernel/issues/814


Build changes
-------------

  * IGT: IGT_7551 -> IGTPW_10041

  IGTPW_10041: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_10041/index.html
  IGT_7551: 15e7d92ca5f98d10feffa27a76724c33cbd68da5 @ https://gitlab.freedesktop.org/drm/igt-gpu-tools.git
  xe-442-a0a80950176b39dbc76f8faa92fddf6caaa06191: a0a80950176b39dbc76f8faa92fddf6caaa06191

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/intel-xe/IGTPW_10041/index.html

[-- Attachment #2: Type: text/html, Size: 3347 bytes --]

Re: [igt-dev] [PATCH 1/3] runner: Normalize testlist entries that don't list subtests

[Mauro Carvalho Chehab]

On Fri, 20 Oct 2023 16:30:39 +0300
Petri Latvala <adrinael@adrinael.net> wrote:

> As the syntax for "all subtests" and "test that doesn't have subtests"
> is the same, check the subtest listing when building the execution
> plan. Doing that makes it possible to still have "all subtests" in
> testlists (albeit not originally designed to support that) and enables
> blocklisting particular subtests.
> 
> Signed-off-by: Petri Latvala <adrinael@adrinael.net>
> Cc: Mauro Carvalho Chehab <mchehab@kernel.org>
> Cc: Arkadiusz Hiler <arek@hiler.eu>
> Cc: Kamil Konieczny <kamil.konieczny@linux.intel.com>
> Cc: Juha-Pekka Heikkila <juhapekka.heikkila@gmail.com>
> Cc: Bhanuprakash Modem <bhanuprakash.modem@intel.com>
> Cc: Ashutosh Dixit <ashutosh.dixit@intel.com>
> Closes: https://gitlab.freedesktop.org/drm/igt-gpu-tools/-/issues/149

That does the job.

Reviewed-by: Mauro Carvalho Chehab <mchehab@kernel.org>

> ---
>  runner/job_list.c | 22 +++++++++++++++++++++-
>  1 file changed, 21 insertions(+), 1 deletion(-)
> 
> diff --git a/runner/job_list.c b/runner/job_list.c
> index e6ea83631..27cbb10bc 100644
> --- a/runner/job_list.c
> +++ b/runner/job_list.c
> @@ -230,8 +230,28 @@ static bool job_list_from_test_list(struct job_list *job_list,
>  			continue;
>  
>  		if (sscanf(line, "igt@%ms", &binary) == 1) {
> -			if ((delim = strchr(binary, '@')) != NULL)
> +			if ((delim = strchr(binary, '@')) != NULL) {
>  				*delim++ = '\0';
> +			} else {
> +				/*
> +				 * No subtests specified. Check
> +				 * whether the user means "all
> +				 * subtests" or if the test doesn't
> +				 * have any.
> +				 */
> +				if (entry.binary) {
> +					/* First flush the entry we're building for multiple-mode */
> +					add_job_list_entry(job_list, entry.binary, entry.subtests, entry.subtest_count);
> +					memset(&entry, 0, sizeof(entry));
> +					any = true;
> +				}
> +
> +				add_subtests(job_list, settings, binary,
> +					     &settings->include_regexes,
> +					     &settings->exclude_regexes);
> +				any = true;
> +				continue;
> +			}
>  
>  			if (!settings->multiple_mode) {
>  				char **subtests = NULL;