* [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-28 10:16 ` Arkadiusz Hiler
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 02/10] lib/tests: Unit tests for " Petri Latvala
` (10 subsequent siblings)
11 siblings, 1 reply; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Dynamic subtests, or subtests of subtests, are individual pieces of
tests that are not statically available all the time.
A good example of a need for a dynamic subtest is i915 engine listing:
A normal subtest for each engine class ("bsd"), and a dynamic subtest
for each instance ("bsd0", "bsd2", etc). Or a normal subtest for an
operation with a dynamic subtest for every engine there is.
Another example is a dynamic subtest for pipes: Instead of using
foreach_pipe_static, make one subtest and use foreach_pipe with
dynamic subtests for each pipe.
v2: Rebase and adapt to igt_describe changes
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
lib/igt_core.c | 119 ++++++++++++++++++++++++++++++++++++++++++++-----
lib/igt_core.h | 89 ++++++++++++++++++++++++++++++++++++
2 files changed, 197 insertions(+), 11 deletions(-)
diff --git a/lib/igt_core.c b/lib/igt_core.c
index 1cbb09f9..8b754b9f 100644
--- a/lib/igt_core.c
+++ b/lib/igt_core.c
@@ -263,9 +263,12 @@ bool igt_skip_crc_compare;
static bool list_subtests = false;
static bool describe_subtests = false;
static char *run_single_subtest = NULL;
+static char *run_single_dynamic_subtest = NULL;
static bool run_single_subtest_found = false;
static const char *in_subtest = NULL;
+static const char *in_dynamic_subtest = NULL;
static struct timespec subtest_time;
+static struct timespec dynamic_subtest_time;
static clockid_t igt_clock = (clockid_t)-1;
static bool in_fixture = false;
static bool test_with_subtests = false;
@@ -300,6 +303,7 @@ enum {
OPT_LIST_SUBTESTS = 500,
OPT_DESCRIBE_SUBTESTS,
OPT_RUN_SUBTEST,
+ OPT_RUN_DYNAMIC_SUBTEST,
OPT_DESCRIPTION,
OPT_DEBUG,
OPT_INTERACTIVE_DEBUG,
@@ -323,6 +327,8 @@ char *igt_frame_dump_path;
static bool stderr_needs_sentinel = false;
+static int _igt_dynamic_tests_executed = -1;
+
const char *igt_test_name(void)
{
return command_str;
@@ -354,7 +360,9 @@ static void _igt_log_buffer_dump(void)
{
uint8_t i;
- if (in_subtest)
+ if (in_dynamic_subtest)
+ fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
+ else if (in_subtest)
fprintf(stderr, "Subtest %s failed.\n", in_subtest);
else
fprintf(stderr, "Test %s failed.\n", command_str);
@@ -619,6 +627,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
fprintf(f, " --list-subtests\n"
" --run-subtest <pattern>\n"
+ " --dynamic-subtest <pattern>\n"
" --debug[=log-domain]\n"
" --interactive-debug[=domain]\n"
" --skip-crc-compare\n"
@@ -739,6 +748,7 @@ static int common_init(int *argc, char **argv,
{"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS},
{"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
{"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST},
+ {"dynamic-subtest", required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
{"help-description", no_argument, NULL, OPT_DESCRIPTION},
{"debug", optional_argument, NULL, OPT_DEBUG},
{"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
@@ -858,6 +868,11 @@ static int common_init(int *argc, char **argv,
if (!list_subtests)
run_single_subtest = strdup(optarg);
break;
+ case OPT_RUN_DYNAMIC_SUBTEST:
+ assert(optarg);
+ if (!list_subtests)
+ run_single_dynamic_subtest = strdup(optarg);
+ break;
case OPT_DESCRIPTION:
print_test_description();
ret = -1;
@@ -1107,6 +1122,41 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
return (in_subtest = subtest_name);
}
+bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
+{
+ int i;
+
+ assert(in_subtest);
+ assert(_igt_dynamic_tests_executed >= 0);
+
+ /* check the dynamic_subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
+ for (i = 0; dynamic_subtest_name[i] != '\0'; i++)
+ if (dynamic_subtest_name[i] != '_' && dynamic_subtest_name[i] != '-'
+ && !isalnum(dynamic_subtest_name[i])) {
+ igt_critical("Invalid dynamic subtest name \"%s\".\n",
+ dynamic_subtest_name);
+ igt_exit();
+ }
+
+ if (run_single_dynamic_subtest &&
+ uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
+ return false;
+
+ igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
+ command_str, dynamic_subtest_name);
+ igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
+ fflush(stdout);
+ if (stderr_needs_sentinel)
+ fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
+
+ _igt_log_buffer_reset();
+
+ _igt_dynamic_tests_executed++;
+
+ igt_gettime(&dynamic_subtest_time);
+ return (in_dynamic_subtest = dynamic_subtest_name);
+}
+
/**
* igt_subtest_name:
*
@@ -1161,26 +1211,50 @@ void __igt_subtest_group_restore(int save, int desc)
static bool skipped_one = false;
static bool succeeded_one = false;
static bool failed_one = false;
+static bool dynamic_failed_one = false;
+
+bool __igt_enter_dynamic_container(void)
+{
+ _igt_dynamic_tests_executed = 0;
+ dynamic_failed_one = false;
+
+ return true;
+}
static void exit_subtest(const char *) __attribute__((noreturn));
static void exit_subtest(const char *result)
{
struct timespec now;
+ const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
+ const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
+ struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
+ jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subtest_jmpbuf : &igt_subtest_jmpbuf;
igt_gettime(&now);
- igt_info("%sSubtest %s: %s (%.3fs)%s\n",
+
+ igt_info("%s%s %s: %s (%.3fs)%s\n",
(!__igt_plain_output) ? "\x1b[1m" : "",
- in_subtest, result, igt_time_elapsed(&subtest_time, &now),
+ subtest_text, *subtest_name, result,
+ igt_time_elapsed(thentime, &now),
(!__igt_plain_output) ? "\x1b[0m" : "");
fflush(stdout);
if (stderr_needs_sentinel)
- fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
- in_subtest, result, igt_time_elapsed(&subtest_time, &now));
+ fprintf(stderr, "%s %s: %s (%.3fs)\n",
+ subtest_text, *subtest_name,
+ result, igt_time_elapsed(thentime, &now));
igt_terminate_spins();
- in_subtest = NULL;
- siglongjmp(igt_subtest_jmpbuf, 1);
+ if (!in_dynamic_subtest)
+ _igt_dynamic_tests_executed = -1;
+
+ /* Don't keep the above text in the log, the container would print it again otherwise */
+ if (in_dynamic_subtest)
+ _igt_log_buffer_reset();
+
+ *subtest_name = NULL;
+
+ siglongjmp(*jmptarget, 1);
}
/**
@@ -1211,6 +1285,7 @@ void igt_skip(const char *f, ...)
}
if (in_subtest) {
+ /* Doing the same even if inside a dynamic subtest */
exit_subtest("SKIP");
} else if (test_with_subtests) {
skip_subtests_henceforth = SKIP;
@@ -1267,7 +1342,22 @@ void __igt_skip_check(const char *file, const int line,
*/
void igt_success(void)
{
- succeeded_one = true;
+ if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
+ /*
+ * We're exiting a dynamic container, yield a result
+ * according to the dynamic tests that got
+ * executed.
+ */
+ if (dynamic_failed_one)
+ igt_fail(IGT_EXIT_FAILURE);
+
+ if (_igt_dynamic_tests_executed == 0)
+ igt_skip("No dynamic tests executed.\n");
+ }
+
+ if (!in_dynamic_subtest)
+ succeeded_one = true;
+
if (in_subtest)
exit_subtest("SUCCESS");
}
@@ -1298,10 +1388,17 @@ void igt_fail(int exitcode)
if (in_atexit_handler)
_exit(IGT_EXIT_FAILURE);
- if (!failed_one)
- igt_exitcode = exitcode;
+ if (in_dynamic_subtest) {
+ dynamic_failed_one = true;
+ } else {
+ /* Dynamic subtest containers must not fail explicitly */
+ assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
+
+ if (!failed_one)
+ igt_exitcode = exitcode;
- failed_one = true;
+ failed_one = true;
+ }
/* Silent exit, parent will do the yelling. */
if (test_child)
diff --git a/lib/igt_core.h b/lib/igt_core.h
index 177d2431..21289c8e 100644
--- a/lib/igt_core.h
+++ b/lib/igt_core.h
@@ -144,6 +144,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
/* subtest infrastructure */
jmp_buf igt_subtest_jmpbuf;
+jmp_buf igt_dynamic_subtest_jmpbuf;
typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
#define IGT_OPT_HANDLER_SUCCESS 0
#define IGT_OPT_HANDLER_ERROR -2
@@ -175,6 +176,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
+bool __igt_enter_dynamic_container(void);
+bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
#define __igt_tokencat2(x, y) x ## y
/**
@@ -224,6 +227,92 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
#define igt_subtest_f(f...) \
__igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
+/**
+ * igt_dynamic_subtest_container:
+ * @name: name of the subtest
+ *
+ * This is a magic control flow block which denotes a subtest code
+ * block that contains dynamic subtests. The _f variant accepts a
+ * printf format string, which is useful for constructing
+ * combinatorial tests.
+ *
+ * Within a dynamic subtest container, explicit failure
+ * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
+ * will produce test results. igt_skip()/igt_require() is allowed.
+ *
+ * This is a simpler version of igt_dynamic_subtest_container_f()
+ */
+#define igt_dynamic_subtest_container(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
+ __igt_enter_dynamic_container() && \
+ (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
+ igt_success())
+#define __igt_dynamic_subtest_container_f(tmp, format...) \
+ for (char tmp [256]; \
+ snprintf( tmp , sizeof( tmp ), \
+ format), \
+ __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
+ __igt_enter_dynamic_container() && \
+ (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
+ igt_success())
+
+/**
+ * igt_dynamic_subtest_container_f:
+ * @...: format string and optional arguments
+ *
+ * This is a magic control flow block which denotes a subtest code
+ * block that contains dynamic subtests. The _f variant accepts a
+ * printf format string, which is useful for constructing
+ * combinatorial tests.
+ *
+ * Within a dynamic subtest container, explicit failure
+ * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
+ * will produce test results. igt_skip()/igt_require() is allowed.
+ *
+ * Like igt_dynamic_subtest_container(), but also accepts a printf
+ * format string instead of a static string.
+ */
+#define igt_dynamic_subtest_container_f(f...) \
+ __igt_dynamic_subtest_container_f(igt_tokencat(__tmpchar, __LINE__), f)
+
+/**
+ * igt_dynamic_subtest:
+ * @name: name of the dynamic subtest
+ *
+ * This is a magic control flow block which denotes a dynamic
+ * subtest-of-a-subtest code block. Within that code block
+ * igt_skip|success will only bail out of the dynamic subtest. The _f
+ * variant accepts a printf format string, which is useful for
+ * constructing combinatorial tests.
+ *
+ * This is a simpler version of igt_dynamic_subtest_f()
+ */
+#define igt_dynamic_subtest(name) for (; __igt_run_dynamic_subtest((name)) && \
+ (sigsetjmp(igt_dynamic_subtest_jmpbuf, 1) == 0); \
+ igt_success())
+#define __igt_dynamic_subtest_f(tmp, format...) \
+ for (char tmp [256]; \
+ snprintf( tmp , sizeof( tmp ), \
+ format), \
+ __igt_run_dynamic_subtest( tmp ) && \
+ (sigsetjmp(igt_dynamic_subtest_jmpbuf, 1) == 0); \
+ igt_success())
+
+/**
+ * igt_dynamic_subtest_f:
+ * @...: format string and optional arguments
+ *
+ * This is a magic control flow block which denotes a dynamic
+ * subtest-of-a-subtest code block. Within that code block
+ * igt_skip|success will only bail out of the dynamic subtest. The _f
+ * variant accepts a printf format string, which is useful for
+ * constructing combinatorial tests.
+ *
+ * Like igt_dynamic_subtest(), but also accepts a printf format string
+ * instead of a static string.
+ */
+#define igt_dynamic_subtest_f(f...) \
+ __igt_dynamic_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
+
const char *igt_subtest_name(void);
bool igt_only_list_subtests(void);
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* Re: [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests Petri Latvala
@ 2019-08-28 10:16 ` Arkadiusz Hiler
2019-08-28 10:30 ` Petri Latvala
0 siblings, 1 reply; 20+ messages in thread
From: Arkadiusz Hiler @ 2019-08-28 10:16 UTC (permalink / raw)
To: Petri Latvala; +Cc: igt-dev
On Fri, Aug 16, 2019 at 12:34:17PM +0300, Petri Latvala wrote:
> Dynamic subtests, or subtests of subtests, are individual pieces of
> tests that are not statically available all the time.
>
> A good example of a need for a dynamic subtest is i915 engine listing:
> A normal subtest for each engine class ("bsd"), and a dynamic subtest
> for each instance ("bsd0", "bsd2", etc). Or a normal subtest for an
> operation with a dynamic subtest for every engine there is.
>
> Another example is a dynamic subtest for pipes: Instead of using
> foreach_pipe_static, make one subtest and use foreach_pipe with
> dynamic subtests for each pipe.
>
> v2: Rebase and adapt to igt_describe changes
>
> Signed-off-by: Petri Latvala <petri.latvala@intel.com>
> ---
> lib/igt_core.c | 119 ++++++++++++++++++++++++++++++++++++++++++++-----
> lib/igt_core.h | 89 ++++++++++++++++++++++++++++++++++++
> 2 files changed, 197 insertions(+), 11 deletions(-)
>
> diff --git a/lib/igt_core.c b/lib/igt_core.c
> index 1cbb09f9..8b754b9f 100644
> --- a/lib/igt_core.c
> +++ b/lib/igt_core.c
> @@ -263,9 +263,12 @@ bool igt_skip_crc_compare;
> static bool list_subtests = false;
> static bool describe_subtests = false;
> static char *run_single_subtest = NULL;
> +static char *run_single_dynamic_subtest = NULL;
> static bool run_single_subtest_found = false;
> static const char *in_subtest = NULL;
> +static const char *in_dynamic_subtest = NULL;
> static struct timespec subtest_time;
> +static struct timespec dynamic_subtest_time;
> static clockid_t igt_clock = (clockid_t)-1;
> static bool in_fixture = false;
> static bool test_with_subtests = false;
> @@ -300,6 +303,7 @@ enum {
> OPT_LIST_SUBTESTS = 500,
> OPT_DESCRIBE_SUBTESTS,
> OPT_RUN_SUBTEST,
> + OPT_RUN_DYNAMIC_SUBTEST,
> OPT_DESCRIPTION,
> OPT_DEBUG,
> OPT_INTERACTIVE_DEBUG,
> @@ -323,6 +327,8 @@ char *igt_frame_dump_path;
>
> static bool stderr_needs_sentinel = false;
>
> +static int _igt_dynamic_tests_executed = -1;
> +
> const char *igt_test_name(void)
> {
> return command_str;
> @@ -354,7 +360,9 @@ static void _igt_log_buffer_dump(void)
> {
> uint8_t i;
>
> - if (in_subtest)
> + if (in_dynamic_subtest)
> + fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
> + else if (in_subtest)
> fprintf(stderr, "Subtest %s failed.\n", in_subtest);
> else
> fprintf(stderr, "Test %s failed.\n", command_str);
> @@ -619,6 +627,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
> fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
> fprintf(f, " --list-subtests\n"
> " --run-subtest <pattern>\n"
> + " --dynamic-subtest <pattern>\n"
> " --debug[=log-domain]\n"
> " --interactive-debug[=domain]\n"
> " --skip-crc-compare\n"
> @@ -739,6 +748,7 @@ static int common_init(int *argc, char **argv,
> {"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS},
> {"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
> {"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST},
> + {"dynamic-subtest", required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
This deviates from --run-subtests but even OPT_RUN_DYNAMIC_SUBTEST has
"run" in it. I guess you did it to preserve '--r' shorthand?
Also interaction betwen --rub-subtest and igt_dynamic_subtest_container
is not very obvious.
Same for using both --run-subtest and --dynamic-subtest.
Maybe it's time to introduce short options?
> {"help-description", no_argument, NULL, OPT_DESCRIPTION},
> {"debug", optional_argument, NULL, OPT_DEBUG},
> {"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
> @@ -858,6 +868,11 @@ static int common_init(int *argc, char **argv,
> if (!list_subtests)
> run_single_subtest = strdup(optarg);
> break;
> + case OPT_RUN_DYNAMIC_SUBTEST:
> + assert(optarg);
> + if (!list_subtests)
> + run_single_dynamic_subtest = strdup(optarg);
> + break;
> case OPT_DESCRIPTION:
> print_test_description();
> ret = -1;
> @@ -1107,6 +1122,41 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> return (in_subtest = subtest_name);
> }
>
> +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
> +{
> + int i;
> +
> + assert(in_subtest);
> + assert(_igt_dynamic_tests_executed >= 0);
> +
> + /* check the dynamic_subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
> + for (i = 0; dynamic_subtest_name[i] != '\0'; i++)
> + if (dynamic_subtest_name[i] != '_' && dynamic_subtest_name[i] != '-'
> + && !isalnum(dynamic_subtest_name[i])) {
> + igt_critical("Invalid dynamic subtest name \"%s\".\n",
> + dynamic_subtest_name);
> + igt_exit();
> + }
Same logic as for normal subtests, could use a helper?
> +
> + if (run_single_dynamic_subtest &&
> + uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
> + return false;
> +
> + igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
> + command_str, dynamic_subtest_name);
> + igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
> + fflush(stdout);
> + if (stderr_needs_sentinel)
> + fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
> +
> + _igt_log_buffer_reset();
> +
> + _igt_dynamic_tests_executed++;
> +
> + igt_gettime(&dynamic_subtest_time);
> + return (in_dynamic_subtest = dynamic_subtest_name);
> +}
> +
> /**
> * igt_subtest_name:
> *
> @@ -1161,26 +1211,50 @@ void __igt_subtest_group_restore(int save, int desc)
> static bool skipped_one = false;
> static bool succeeded_one = false;
> static bool failed_one = false;
> +static bool dynamic_failed_one = false;
> +
> +bool __igt_enter_dynamic_container(void)
> +{
> + _igt_dynamic_tests_executed = 0;
> + dynamic_failed_one = false;
> +
> + return true;
> +}
>
> static void exit_subtest(const char *) __attribute__((noreturn));
> static void exit_subtest(const char *result)
> {
> struct timespec now;
> + const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
> + const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
> + struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
> + jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subtest_jmpbuf : &igt_subtest_jmpbuf;
>
> igt_gettime(&now);
> - igt_info("%sSubtest %s: %s (%.3fs)%s\n",
> +
> + igt_info("%s%s %s: %s (%.3fs)%s\n",
> (!__igt_plain_output) ? "\x1b[1m" : "",
> - in_subtest, result, igt_time_elapsed(&subtest_time, &now),
> + subtest_text, *subtest_name, result,
> + igt_time_elapsed(thentime, &now),
> (!__igt_plain_output) ? "\x1b[0m" : "");
> fflush(stdout);
> if (stderr_needs_sentinel)
> - fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
> - in_subtest, result, igt_time_elapsed(&subtest_time, &now));
> + fprintf(stderr, "%s %s: %s (%.3fs)\n",
> + subtest_text, *subtest_name,
> + result, igt_time_elapsed(thentime, &now));
>
> igt_terminate_spins();
>
> - in_subtest = NULL;
> - siglongjmp(igt_subtest_jmpbuf, 1);
> + if (!in_dynamic_subtest)
> + _igt_dynamic_tests_executed = -1;
> +
> + /* Don't keep the above text in the log, the container would print it again otherwise */
> + if (in_dynamic_subtest)
> + _igt_log_buffer_reset();
> +
> + *subtest_name = NULL;
> +
> + siglongjmp(*jmptarget, 1);
> }
>
> /**
> @@ -1211,6 +1285,7 @@ void igt_skip(const char *f, ...)
> }
>
> if (in_subtest) {
> + /* Doing the same even if inside a dynamic subtest */
> exit_subtest("SKIP");
> } else if (test_with_subtests) {
> skip_subtests_henceforth = SKIP;
> @@ -1267,7 +1342,22 @@ void __igt_skip_check(const char *file, const int line,
> */
> void igt_success(void)
> {
> - succeeded_one = true;
> + if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
> + /*
> + * We're exiting a dynamic container, yield a result
> + * according to the dynamic tests that got
> + * executed.
> + */
> + if (dynamic_failed_one)
> + igt_fail(IGT_EXIT_FAILURE);
> +
> + if (_igt_dynamic_tests_executed == 0)
> + igt_skip("No dynamic tests executed.\n");
> + }
> +
> + if (!in_dynamic_subtest)
> + succeeded_one = true;
> +
> if (in_subtest)
> exit_subtest("SUCCESS");
> }
> @@ -1298,10 +1388,17 @@ void igt_fail(int exitcode)
> if (in_atexit_handler)
> _exit(IGT_EXIT_FAILURE);
>
> - if (!failed_one)
> - igt_exitcode = exitcode;
> + if (in_dynamic_subtest) {
> + dynamic_failed_one = true;
> + } else {
> + /* Dynamic subtest containers must not fail explicitly */
> + assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
> +
> + if (!failed_one)
> + igt_exitcode = exitcode;
>
> - failed_one = true;
> + failed_one = true;
> + }
>
> /* Silent exit, parent will do the yelling. */
> if (test_child)
> diff --git a/lib/igt_core.h b/lib/igt_core.h
> index 177d2431..21289c8e 100644
> --- a/lib/igt_core.h
> +++ b/lib/igt_core.h
> @@ -144,6 +144,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
>
> /* subtest infrastructure */
> jmp_buf igt_subtest_jmpbuf;
> +jmp_buf igt_dynamic_subtest_jmpbuf;
> typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
> #define IGT_OPT_HANDLER_SUCCESS 0
> #define IGT_OPT_HANDLER_ERROR -2
> @@ -175,6 +176,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
> igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
>
> bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
> +bool __igt_enter_dynamic_container(void);
> +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
> #define __igt_tokencat2(x, y) x ## y
>
> /**
> @@ -224,6 +227,92 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> #define igt_subtest_f(f...) \
> __igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
>
> +/**
> + * igt_dynamic_subtest_container:
> + * @name: name of the subtest
> + *
> + * This is a magic control flow block which denotes a subtest code
> + * block that contains dynamic subtests. The _f variant accepts a
> + * printf format string, which is useful for constructing
> + * combinatorial tests.
> + *
> + * Within a dynamic subtest container, explicit failure
> + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> + * will produce test results. igt_skip()/igt_require() is allowed.
> + *
> + * This is a simpler version of igt_dynamic_subtest_container_f()
> + */
> +#define igt_dynamic_subtest_container(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
> + __igt_enter_dynamic_container() && \
> + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> + igt_success())
> +#define __igt_dynamic_subtest_container_f(tmp, format...) \
> + for (char tmp [256]; \
> + snprintf( tmp , sizeof( tmp ), \
> + format), \
> + __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
> + __igt_enter_dynamic_container() && \
> + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> + igt_success())
> +
> +/**
> + * igt_dynamic_subtest_container_f:
> + * @...: format string and optional arguments
> + *
> + * This is a magic control flow block which denotes a subtest code
> + * block that contains dynamic subtests. The _f variant accepts a
> + * printf format string, which is useful for constructing
> + * combinatorial tests.
> + *
> + * Within a dynamic subtest container, explicit failure
> + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> + * will produce test results. igt_skip()/igt_require() is allowed.
> + *
> + * Like igt_dynamic_subtest_container(), but also accepts a printf
> + * format string instead of a static string.
> + */
I think we need a bit more of explanation here. It's hard to understand
where the asserts go and that this is more like a subtest group and you
have to nest igt_dynamic_subtest under it.
I think some pseudocode could be usefule, e.g.:
igt_main {
igt_dynamic_subtest_container("engine-tests") {
/* requires ok, no asserts */
igt_require(is_awesome(fd));
for_each_engine(e) {
igt_dynamic_subtest_f("%s", e->name) {
/* asserts ok! */
igt_fail();
}
}
}
}
It's also not very clear how this will be reported out.
Or maybe even use "group" instead of "container" here to build the
parallel with normal subtest group, the obvious difference being that it
takes name.
> +#define igt_dynamic_subtest_container_f(f...) \
> + __igt_dynamic_subtest_container_f(igt_tokencat(__tmpchar, __LINE__), f)
> +
> +/**
> + * igt_dynamic_subtest:
> + * @name: name of the dynamic subtest
> + *
> + * This is a magic control flow block which denotes a dynamic
> + * subtest-of-a-subtest code block. Within that code block
> + * igt_skip|success will only bail out of the dynamic subtest. The _f
> + * variant accepts a printf format string, which is useful for
> + * constructing combinatorial tests.
> + *
> + * This is a simpler version of igt_dynamic_subtest_f()
> + */
> +#define igt_dynamic_subtest(name) for (; __igt_run_dynamic_subtest((name)) && \
> + (sigsetjmp(igt_dynamic_subtest_jmpbuf, 1) == 0); \
> + igt_success())
> +#define __igt_dynamic_subtest_f(tmp, format...) \
> + for (char tmp [256]; \
> + snprintf( tmp , sizeof( tmp ), \
> + format), \
> + __igt_run_dynamic_subtest( tmp ) && \
> + (sigsetjmp(igt_dynamic_subtest_jmpbuf, 1) == 0); \
> + igt_success())
> +
> +/**
> + * igt_dynamic_subtest_f:
> + * @...: format string and optional arguments
> + *
> + * This is a magic control flow block which denotes a dynamic
> + * subtest-of-a-subtest code block. Within that code block
> + * igt_skip|success will only bail out of the dynamic subtest. The _f
> + * variant accepts a printf format string, which is useful for
> + * constructing combinatorial tests.
> + *
> + * Like igt_dynamic_subtest(), but also accepts a printf format string
> + * instead of a static string.
> + */
> +#define igt_dynamic_subtest_f(f...) \
> + __igt_dynamic_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
> +
> const char *igt_subtest_name(void);
> bool igt_only_list_subtests(void);
>
> --
> 2.19.1
>
> _______________________________________________
> igt-dev mailing list
> igt-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/igt-dev
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests
2019-08-28 10:16 ` Arkadiusz Hiler
@ 2019-08-28 10:30 ` Petri Latvala
2019-08-28 10:38 ` Arkadiusz Hiler
0 siblings, 1 reply; 20+ messages in thread
From: Petri Latvala @ 2019-08-28 10:30 UTC (permalink / raw)
To: Arkadiusz Hiler; +Cc: igt-dev
On Wed, Aug 28, 2019 at 01:16:10PM +0300, Arkadiusz Hiler wrote:
> On Fri, Aug 16, 2019 at 12:34:17PM +0300, Petri Latvala wrote:
> > Dynamic subtests, or subtests of subtests, are individual pieces of
> > tests that are not statically available all the time.
> >
> > A good example of a need for a dynamic subtest is i915 engine listing:
> > A normal subtest for each engine class ("bsd"), and a dynamic subtest
> > for each instance ("bsd0", "bsd2", etc). Or a normal subtest for an
> > operation with a dynamic subtest for every engine there is.
> >
> > Another example is a dynamic subtest for pipes: Instead of using
> > foreach_pipe_static, make one subtest and use foreach_pipe with
> > dynamic subtests for each pipe.
> >
> > v2: Rebase and adapt to igt_describe changes
> >
> > Signed-off-by: Petri Latvala <petri.latvala@intel.com>
> > ---
> > lib/igt_core.c | 119 ++++++++++++++++++++++++++++++++++++++++++++-----
> > lib/igt_core.h | 89 ++++++++++++++++++++++++++++++++++++
> > 2 files changed, 197 insertions(+), 11 deletions(-)
> >
> > diff --git a/lib/igt_core.c b/lib/igt_core.c
> > index 1cbb09f9..8b754b9f 100644
> > --- a/lib/igt_core.c
> > +++ b/lib/igt_core.c
> > @@ -263,9 +263,12 @@ bool igt_skip_crc_compare;
> > static bool list_subtests = false;
> > static bool describe_subtests = false;
> > static char *run_single_subtest = NULL;
> > +static char *run_single_dynamic_subtest = NULL;
> > static bool run_single_subtest_found = false;
> > static const char *in_subtest = NULL;
> > +static const char *in_dynamic_subtest = NULL;
> > static struct timespec subtest_time;
> > +static struct timespec dynamic_subtest_time;
> > static clockid_t igt_clock = (clockid_t)-1;
> > static bool in_fixture = false;
> > static bool test_with_subtests = false;
> > @@ -300,6 +303,7 @@ enum {
> > OPT_LIST_SUBTESTS = 500,
> > OPT_DESCRIBE_SUBTESTS,
> > OPT_RUN_SUBTEST,
> > + OPT_RUN_DYNAMIC_SUBTEST,
> > OPT_DESCRIPTION,
> > OPT_DEBUG,
> > OPT_INTERACTIVE_DEBUG,
> > @@ -323,6 +327,8 @@ char *igt_frame_dump_path;
> >
> > static bool stderr_needs_sentinel = false;
> >
> > +static int _igt_dynamic_tests_executed = -1;
> > +
> > const char *igt_test_name(void)
> > {
> > return command_str;
> > @@ -354,7 +360,9 @@ static void _igt_log_buffer_dump(void)
> > {
> > uint8_t i;
> >
> > - if (in_subtest)
> > + if (in_dynamic_subtest)
> > + fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
> > + else if (in_subtest)
> > fprintf(stderr, "Subtest %s failed.\n", in_subtest);
> > else
> > fprintf(stderr, "Test %s failed.\n", command_str);
> > @@ -619,6 +627,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
> > fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
> > fprintf(f, " --list-subtests\n"
> > " --run-subtest <pattern>\n"
> > + " --dynamic-subtest <pattern>\n"
> > " --debug[=log-domain]\n"
> > " --interactive-debug[=domain]\n"
> > " --skip-crc-compare\n"
> > @@ -739,6 +748,7 @@ static int common_init(int *argc, char **argv,
> > {"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS},
> > {"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
> > {"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST},
> > + {"dynamic-subtest", required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
>
> This deviates from --run-subtests but even OPT_RUN_DYNAMIC_SUBTEST has
> "run" in it. I guess you did it to preserve '--r' shorthand?
Yes :P
> Also interaction betwen --rub-subtest and igt_dynamic_subtest_container
> is not very obvious.
>
> Same for using both --run-subtest and --dynamic-subtest.
>
> Maybe it's time to introduce short options?
>
> > {"help-description", no_argument, NULL, OPT_DESCRIPTION},
> > {"debug", optional_argument, NULL, OPT_DEBUG},
> > {"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
> > @@ -858,6 +868,11 @@ static int common_init(int *argc, char **argv,
> > if (!list_subtests)
> > run_single_subtest = strdup(optarg);
> > break;
> > + case OPT_RUN_DYNAMIC_SUBTEST:
> > + assert(optarg);
> > + if (!list_subtests)
> > + run_single_dynamic_subtest = strdup(optarg);
> > + break;
> > case OPT_DESCRIPTION:
> > print_test_description();
> > ret = -1;
> > @@ -1107,6 +1122,41 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> > return (in_subtest = subtest_name);
> > }
> >
> > +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
> > +{
> > + int i;
> > +
> > + assert(in_subtest);
> > + assert(_igt_dynamic_tests_executed >= 0);
> > +
> > + /* check the dynamic_subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
> > + for (i = 0; dynamic_subtest_name[i] != '\0'; i++)
> > + if (dynamic_subtest_name[i] != '_' && dynamic_subtest_name[i] != '-'
> > + && !isalnum(dynamic_subtest_name[i])) {
> > + igt_critical("Invalid dynamic subtest name \"%s\".\n",
> > + dynamic_subtest_name);
> > + igt_exit();
> > + }
>
> Same logic as for normal subtests, could use a helper?
Yes! I'll do that for next revision.
>
> > +
> > + if (run_single_dynamic_subtest &&
> > + uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
> > + return false;
> > +
> > + igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
> > + command_str, dynamic_subtest_name);
> > + igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
> > + fflush(stdout);
> > + if (stderr_needs_sentinel)
> > + fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
> > +
> > + _igt_log_buffer_reset();
> > +
> > + _igt_dynamic_tests_executed++;
> > +
> > + igt_gettime(&dynamic_subtest_time);
> > + return (in_dynamic_subtest = dynamic_subtest_name);
> > +}
> > +
> > /**
> > * igt_subtest_name:
> > *
> > @@ -1161,26 +1211,50 @@ void __igt_subtest_group_restore(int save, int desc)
> > static bool skipped_one = false;
> > static bool succeeded_one = false;
> > static bool failed_one = false;
> > +static bool dynamic_failed_one = false;
> > +
> > +bool __igt_enter_dynamic_container(void)
> > +{
> > + _igt_dynamic_tests_executed = 0;
> > + dynamic_failed_one = false;
> > +
> > + return true;
> > +}
> >
> > static void exit_subtest(const char *) __attribute__((noreturn));
> > static void exit_subtest(const char *result)
> > {
> > struct timespec now;
> > + const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
> > + const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
> > + struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
> > + jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subtest_jmpbuf : &igt_subtest_jmpbuf;
> >
> > igt_gettime(&now);
> > - igt_info("%sSubtest %s: %s (%.3fs)%s\n",
> > +
> > + igt_info("%s%s %s: %s (%.3fs)%s\n",
> > (!__igt_plain_output) ? "\x1b[1m" : "",
> > - in_subtest, result, igt_time_elapsed(&subtest_time, &now),
> > + subtest_text, *subtest_name, result,
> > + igt_time_elapsed(thentime, &now),
> > (!__igt_plain_output) ? "\x1b[0m" : "");
> > fflush(stdout);
> > if (stderr_needs_sentinel)
> > - fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
> > - in_subtest, result, igt_time_elapsed(&subtest_time, &now));
> > + fprintf(stderr, "%s %s: %s (%.3fs)\n",
> > + subtest_text, *subtest_name,
> > + result, igt_time_elapsed(thentime, &now));
> >
> > igt_terminate_spins();
> >
> > - in_subtest = NULL;
> > - siglongjmp(igt_subtest_jmpbuf, 1);
> > + if (!in_dynamic_subtest)
> > + _igt_dynamic_tests_executed = -1;
> > +
> > + /* Don't keep the above text in the log, the container would print it again otherwise */
> > + if (in_dynamic_subtest)
> > + _igt_log_buffer_reset();
> > +
> > + *subtest_name = NULL;
> > +
> > + siglongjmp(*jmptarget, 1);
> > }
> >
> > /**
> > @@ -1211,6 +1285,7 @@ void igt_skip(const char *f, ...)
> > }
> >
> > if (in_subtest) {
> > + /* Doing the same even if inside a dynamic subtest */
> > exit_subtest("SKIP");
> > } else if (test_with_subtests) {
> > skip_subtests_henceforth = SKIP;
> > @@ -1267,7 +1342,22 @@ void __igt_skip_check(const char *file, const int line,
> > */
> > void igt_success(void)
> > {
> > - succeeded_one = true;
> > + if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
> > + /*
> > + * We're exiting a dynamic container, yield a result
> > + * according to the dynamic tests that got
> > + * executed.
> > + */
> > + if (dynamic_failed_one)
> > + igt_fail(IGT_EXIT_FAILURE);
> > +
> > + if (_igt_dynamic_tests_executed == 0)
> > + igt_skip("No dynamic tests executed.\n");
> > + }
> > +
> > + if (!in_dynamic_subtest)
> > + succeeded_one = true;
> > +
> > if (in_subtest)
> > exit_subtest("SUCCESS");
> > }
> > @@ -1298,10 +1388,17 @@ void igt_fail(int exitcode)
> > if (in_atexit_handler)
> > _exit(IGT_EXIT_FAILURE);
> >
> > - if (!failed_one)
> > - igt_exitcode = exitcode;
> > + if (in_dynamic_subtest) {
> > + dynamic_failed_one = true;
> > + } else {
> > + /* Dynamic subtest containers must not fail explicitly */
> > + assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
> > +
> > + if (!failed_one)
> > + igt_exitcode = exitcode;
> >
> > - failed_one = true;
> > + failed_one = true;
> > + }
> >
> > /* Silent exit, parent will do the yelling. */
> > if (test_child)
> > diff --git a/lib/igt_core.h b/lib/igt_core.h
> > index 177d2431..21289c8e 100644
> > --- a/lib/igt_core.h
> > +++ b/lib/igt_core.h
> > @@ -144,6 +144,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
> >
> > /* subtest infrastructure */
> > jmp_buf igt_subtest_jmpbuf;
> > +jmp_buf igt_dynamic_subtest_jmpbuf;
> > typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
> > #define IGT_OPT_HANDLER_SUCCESS 0
> > #define IGT_OPT_HANDLER_ERROR -2
> > @@ -175,6 +176,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
> > igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
> >
> > bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
> > +bool __igt_enter_dynamic_container(void);
> > +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
> > #define __igt_tokencat2(x, y) x ## y
> >
> > /**
> > @@ -224,6 +227,92 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> > #define igt_subtest_f(f...) \
> > __igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
> >
> > +/**
> > + * igt_dynamic_subtest_container:
> > + * @name: name of the subtest
> > + *
> > + * This is a magic control flow block which denotes a subtest code
> > + * block that contains dynamic subtests. The _f variant accepts a
> > + * printf format string, which is useful for constructing
> > + * combinatorial tests.
> > + *
> > + * Within a dynamic subtest container, explicit failure
> > + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> > + * will produce test results. igt_skip()/igt_require() is allowed.
> > + *
> > + * This is a simpler version of igt_dynamic_subtest_container_f()
> > + */
> > +#define igt_dynamic_subtest_container(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
> > + __igt_enter_dynamic_container() && \
> > + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> > + igt_success())
> > +#define __igt_dynamic_subtest_container_f(tmp, format...) \
> > + for (char tmp [256]; \
> > + snprintf( tmp , sizeof( tmp ), \
> > + format), \
> > + __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
> > + __igt_enter_dynamic_container() && \
> > + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> > + igt_success())
> > +
> > +/**
> > + * igt_dynamic_subtest_container_f:
> > + * @...: format string and optional arguments
> > + *
> > + * This is a magic control flow block which denotes a subtest code
> > + * block that contains dynamic subtests. The _f variant accepts a
> > + * printf format string, which is useful for constructing
> > + * combinatorial tests.
> > + *
> > + * Within a dynamic subtest container, explicit failure
> > + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> > + * will produce test results. igt_skip()/igt_require() is allowed.
> > + *
> > + * Like igt_dynamic_subtest_container(), but also accepts a printf
> > + * format string instead of a static string.
> > + */
>
> I think we need a bit more of explanation here. It's hard to understand
> where the asserts go and that this is more like a subtest group and you
> have to nest igt_dynamic_subtest under it.
>
> I think some pseudocode could be usefule, e.g.:
>
> igt_main {
> igt_dynamic_subtest_container("engine-tests") {
> /* requires ok, no asserts */
> igt_require(is_awesome(fd));
>
> for_each_engine(e) {
> igt_dynamic_subtest_f("%s", e->name) {
> /* asserts ok! */
> igt_fail();
> }
> }
> }
> }
>
> It's also not very clear how this will be reported out.
Yeah, your recent examples of embedding code into documentation with
the igt_describe code now makes it possible for me to write better
docs. Noted.
> Or maybe even use "group" instead of "container" here to build the
> parallel with normal subtest group, the obvious difference being that it
> takes name.
The pertinent issue is hiding right here in plain view:
Naming.
The main relative for the "container" is not subtest group, but
subtest. After all, it is an execution point, listed with
--list-subtests and executed with --run-subtest.
igt_dynamic_subtest_container is the best name I could come up with,
and it absolutely sucks at conveying the purpose. Deciding on the name
should be considered to be the most important part of reviewing this
series, suggestions for that are very welcome!
--
Petri Latvala
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests
2019-08-28 10:30 ` Petri Latvala
@ 2019-08-28 10:38 ` Arkadiusz Hiler
2019-08-28 12:58 ` Petri Latvala
0 siblings, 1 reply; 20+ messages in thread
From: Arkadiusz Hiler @ 2019-08-28 10:38 UTC (permalink / raw)
To: igt-dev
On Wed, Aug 28, 2019 at 01:30:23PM +0300, Petri Latvala wrote:
> On Wed, Aug 28, 2019 at 01:16:10PM +0300, Arkadiusz Hiler wrote:
> > On Fri, Aug 16, 2019 at 12:34:17PM +0300, Petri Latvala wrote:
> > > Dynamic subtests, or subtests of subtests, are individual pieces of
> > > tests that are not statically available all the time.
> > >
> > > A good example of a need for a dynamic subtest is i915 engine listing:
> > > A normal subtest for each engine class ("bsd"), and a dynamic subtest
> > > for each instance ("bsd0", "bsd2", etc). Or a normal subtest for an
> > > operation with a dynamic subtest for every engine there is.
> > >
> > > Another example is a dynamic subtest for pipes: Instead of using
> > > foreach_pipe_static, make one subtest and use foreach_pipe with
> > > dynamic subtests for each pipe.
> > >
> > > v2: Rebase and adapt to igt_describe changes
> > >
> > > Signed-off-by: Petri Latvala <petri.latvala@intel.com>
> > > ---
> > > lib/igt_core.c | 119 ++++++++++++++++++++++++++++++++++++++++++++-----
> > > lib/igt_core.h | 89 ++++++++++++++++++++++++++++++++++++
> > > 2 files changed, 197 insertions(+), 11 deletions(-)
> > >
> > > diff --git a/lib/igt_core.c b/lib/igt_core.c
> > > index 1cbb09f9..8b754b9f 100644
> > > --- a/lib/igt_core.c
> > > +++ b/lib/igt_core.c
> > > @@ -263,9 +263,12 @@ bool igt_skip_crc_compare;
> > > static bool list_subtests = false;
> > > static bool describe_subtests = false;
> > > static char *run_single_subtest = NULL;
> > > +static char *run_single_dynamic_subtest = NULL;
> > > static bool run_single_subtest_found = false;
> > > static const char *in_subtest = NULL;
> > > +static const char *in_dynamic_subtest = NULL;
> > > static struct timespec subtest_time;
> > > +static struct timespec dynamic_subtest_time;
> > > static clockid_t igt_clock = (clockid_t)-1;
> > > static bool in_fixture = false;
> > > static bool test_with_subtests = false;
> > > @@ -300,6 +303,7 @@ enum {
> > > OPT_LIST_SUBTESTS = 500,
> > > OPT_DESCRIBE_SUBTESTS,
> > > OPT_RUN_SUBTEST,
> > > + OPT_RUN_DYNAMIC_SUBTEST,
> > > OPT_DESCRIPTION,
> > > OPT_DEBUG,
> > > OPT_INTERACTIVE_DEBUG,
> > > @@ -323,6 +327,8 @@ char *igt_frame_dump_path;
> > >
> > > static bool stderr_needs_sentinel = false;
> > >
> > > +static int _igt_dynamic_tests_executed = -1;
> > > +
> > > const char *igt_test_name(void)
> > > {
> > > return command_str;
> > > @@ -354,7 +360,9 @@ static void _igt_log_buffer_dump(void)
> > > {
> > > uint8_t i;
> > >
> > > - if (in_subtest)
> > > + if (in_dynamic_subtest)
> > > + fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
> > > + else if (in_subtest)
> > > fprintf(stderr, "Subtest %s failed.\n", in_subtest);
> > > else
> > > fprintf(stderr, "Test %s failed.\n", command_str);
> > > @@ -619,6 +627,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
> > > fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
> > > fprintf(f, " --list-subtests\n"
> > > " --run-subtest <pattern>\n"
> > > + " --dynamic-subtest <pattern>\n"
> > > " --debug[=log-domain]\n"
> > > " --interactive-debug[=domain]\n"
> > > " --skip-crc-compare\n"
> > > @@ -739,6 +748,7 @@ static int common_init(int *argc, char **argv,
> > > {"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS},
> > > {"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
> > > {"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST},
> > > + {"dynamic-subtest", required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
> >
> > This deviates from --run-subtests but even OPT_RUN_DYNAMIC_SUBTEST has
> > "run" in it. I guess you did it to preserve '--r' shorthand?
>
> Yes :P
>
> > Also interaction betwen --rub-subtest and igt_dynamic_subtest_container
> > is not very obvious.
> >
> > Same for using both --run-subtest and --dynamic-subtest.
> >
> > Maybe it's time to introduce short options?
> >
> > > {"help-description", no_argument, NULL, OPT_DESCRIPTION},
> > > {"debug", optional_argument, NULL, OPT_DEBUG},
> > > {"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
> > > @@ -858,6 +868,11 @@ static int common_init(int *argc, char **argv,
> > > if (!list_subtests)
> > > run_single_subtest = strdup(optarg);
> > > break;
> > > + case OPT_RUN_DYNAMIC_SUBTEST:
> > > + assert(optarg);
> > > + if (!list_subtests)
> > > + run_single_dynamic_subtest = strdup(optarg);
> > > + break;
> > > case OPT_DESCRIPTION:
> > > print_test_description();
> > > ret = -1;
> > > @@ -1107,6 +1122,41 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> > > return (in_subtest = subtest_name);
> > > }
> > >
> > > +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
> > > +{
> > > + int i;
> > > +
> > > + assert(in_subtest);
> > > + assert(_igt_dynamic_tests_executed >= 0);
> > > +
> > > + /* check the dynamic_subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
> > > + for (i = 0; dynamic_subtest_name[i] != '\0'; i++)
> > > + if (dynamic_subtest_name[i] != '_' && dynamic_subtest_name[i] != '-'
> > > + && !isalnum(dynamic_subtest_name[i])) {
> > > + igt_critical("Invalid dynamic subtest name \"%s\".\n",
> > > + dynamic_subtest_name);
> > > + igt_exit();
> > > + }
> >
> > Same logic as for normal subtests, could use a helper?
>
> Yes! I'll do that for next revision.
>
>
> >
> > > +
> > > + if (run_single_dynamic_subtest &&
> > > + uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
> > > + return false;
> > > +
> > > + igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
> > > + command_str, dynamic_subtest_name);
> > > + igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
> > > + fflush(stdout);
> > > + if (stderr_needs_sentinel)
> > > + fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
> > > +
> > > + _igt_log_buffer_reset();
> > > +
> > > + _igt_dynamic_tests_executed++;
> > > +
> > > + igt_gettime(&dynamic_subtest_time);
> > > + return (in_dynamic_subtest = dynamic_subtest_name);
> > > +}
> > > +
> > > /**
> > > * igt_subtest_name:
> > > *
> > > @@ -1161,26 +1211,50 @@ void __igt_subtest_group_restore(int save, int desc)
> > > static bool skipped_one = false;
> > > static bool succeeded_one = false;
> > > static bool failed_one = false;
> > > +static bool dynamic_failed_one = false;
> > > +
> > > +bool __igt_enter_dynamic_container(void)
> > > +{
> > > + _igt_dynamic_tests_executed = 0;
> > > + dynamic_failed_one = false;
> > > +
> > > + return true;
> > > +}
> > >
> > > static void exit_subtest(const char *) __attribute__((noreturn));
> > > static void exit_subtest(const char *result)
> > > {
> > > struct timespec now;
> > > + const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
> > > + const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
> > > + struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
> > > + jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subtest_jmpbuf : &igt_subtest_jmpbuf;
> > >
> > > igt_gettime(&now);
> > > - igt_info("%sSubtest %s: %s (%.3fs)%s\n",
> > > +
> > > + igt_info("%s%s %s: %s (%.3fs)%s\n",
> > > (!__igt_plain_output) ? "\x1b[1m" : "",
> > > - in_subtest, result, igt_time_elapsed(&subtest_time, &now),
> > > + subtest_text, *subtest_name, result,
> > > + igt_time_elapsed(thentime, &now),
> > > (!__igt_plain_output) ? "\x1b[0m" : "");
> > > fflush(stdout);
> > > if (stderr_needs_sentinel)
> > > - fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
> > > - in_subtest, result, igt_time_elapsed(&subtest_time, &now));
> > > + fprintf(stderr, "%s %s: %s (%.3fs)\n",
> > > + subtest_text, *subtest_name,
> > > + result, igt_time_elapsed(thentime, &now));
> > >
> > > igt_terminate_spins();
> > >
> > > - in_subtest = NULL;
> > > - siglongjmp(igt_subtest_jmpbuf, 1);
> > > + if (!in_dynamic_subtest)
> > > + _igt_dynamic_tests_executed = -1;
> > > +
> > > + /* Don't keep the above text in the log, the container would print it again otherwise */
> > > + if (in_dynamic_subtest)
> > > + _igt_log_buffer_reset();
> > > +
> > > + *subtest_name = NULL;
> > > +
> > > + siglongjmp(*jmptarget, 1);
> > > }
> > >
> > > /**
> > > @@ -1211,6 +1285,7 @@ void igt_skip(const char *f, ...)
> > > }
> > >
> > > if (in_subtest) {
> > > + /* Doing the same even if inside a dynamic subtest */
> > > exit_subtest("SKIP");
> > > } else if (test_with_subtests) {
> > > skip_subtests_henceforth = SKIP;
> > > @@ -1267,7 +1342,22 @@ void __igt_skip_check(const char *file, const int line,
> > > */
> > > void igt_success(void)
> > > {
> > > - succeeded_one = true;
> > > + if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
> > > + /*
> > > + * We're exiting a dynamic container, yield a result
> > > + * according to the dynamic tests that got
> > > + * executed.
> > > + */
> > > + if (dynamic_failed_one)
> > > + igt_fail(IGT_EXIT_FAILURE);
> > > +
> > > + if (_igt_dynamic_tests_executed == 0)
> > > + igt_skip("No dynamic tests executed.\n");
> > > + }
> > > +
> > > + if (!in_dynamic_subtest)
> > > + succeeded_one = true;
> > > +
> > > if (in_subtest)
> > > exit_subtest("SUCCESS");
> > > }
> > > @@ -1298,10 +1388,17 @@ void igt_fail(int exitcode)
> > > if (in_atexit_handler)
> > > _exit(IGT_EXIT_FAILURE);
> > >
> > > - if (!failed_one)
> > > - igt_exitcode = exitcode;
> > > + if (in_dynamic_subtest) {
> > > + dynamic_failed_one = true;
> > > + } else {
> > > + /* Dynamic subtest containers must not fail explicitly */
> > > + assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
> > > +
> > > + if (!failed_one)
> > > + igt_exitcode = exitcode;
> > >
> > > - failed_one = true;
> > > + failed_one = true;
> > > + }
> > >
> > > /* Silent exit, parent will do the yelling. */
> > > if (test_child)
> > > diff --git a/lib/igt_core.h b/lib/igt_core.h
> > > index 177d2431..21289c8e 100644
> > > --- a/lib/igt_core.h
> > > +++ b/lib/igt_core.h
> > > @@ -144,6 +144,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
> > >
> > > /* subtest infrastructure */
> > > jmp_buf igt_subtest_jmpbuf;
> > > +jmp_buf igt_dynamic_subtest_jmpbuf;
> > > typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
> > > #define IGT_OPT_HANDLER_SUCCESS 0
> > > #define IGT_OPT_HANDLER_ERROR -2
> > > @@ -175,6 +176,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
> > > igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
> > >
> > > bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
> > > +bool __igt_enter_dynamic_container(void);
> > > +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
> > > #define __igt_tokencat2(x, y) x ## y
> > >
> > > /**
> > > @@ -224,6 +227,92 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> > > #define igt_subtest_f(f...) \
> > > __igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
> > >
> > > +/**
> > > + * igt_dynamic_subtest_container:
> > > + * @name: name of the subtest
> > > + *
> > > + * This is a magic control flow block which denotes a subtest code
> > > + * block that contains dynamic subtests. The _f variant accepts a
> > > + * printf format string, which is useful for constructing
> > > + * combinatorial tests.
> > > + *
> > > + * Within a dynamic subtest container, explicit failure
> > > + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> > > + * will produce test results. igt_skip()/igt_require() is allowed.
> > > + *
> > > + * This is a simpler version of igt_dynamic_subtest_container_f()
> > > + */
> > > +#define igt_dynamic_subtest_container(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
> > > + __igt_enter_dynamic_container() && \
> > > + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> > > + igt_success())
> > > +#define __igt_dynamic_subtest_container_f(tmp, format...) \
> > > + for (char tmp [256]; \
> > > + snprintf( tmp , sizeof( tmp ), \
> > > + format), \
> > > + __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
> > > + __igt_enter_dynamic_container() && \
> > > + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> > > + igt_success())
> > > +
> > > +/**
> > > + * igt_dynamic_subtest_container_f:
> > > + * @...: format string and optional arguments
> > > + *
> > > + * This is a magic control flow block which denotes a subtest code
> > > + * block that contains dynamic subtests. The _f variant accepts a
> > > + * printf format string, which is useful for constructing
> > > + * combinatorial tests.
> > > + *
> > > + * Within a dynamic subtest container, explicit failure
> > > + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> > > + * will produce test results. igt_skip()/igt_require() is allowed.
> > > + *
> > > + * Like igt_dynamic_subtest_container(), but also accepts a printf
> > > + * format string instead of a static string.
> > > + */
> >
> > I think we need a bit more of explanation here. It's hard to understand
> > where the asserts go and that this is more like a subtest group and you
> > have to nest igt_dynamic_subtest under it.
> >
> > I think some pseudocode could be usefule, e.g.:
> >
> > igt_main {
> > igt_dynamic_subtest_container("engine-tests") {
> > /* requires ok, no asserts */
> > igt_require(is_awesome(fd));
> >
> > for_each_engine(e) {
> > igt_dynamic_subtest_f("%s", e->name) {
> > /* asserts ok! */
> > igt_fail();
> > }
> > }
> > }
> > }
> >
> > It's also not very clear how this will be reported out.
>
> Yeah, your recent examples of embedding code into documentation with
> the igt_describe code now makes it possible for me to write better
> docs. Noted.
>
> > Or maybe even use "group" instead of "container" here to build the
> > parallel with normal subtest group, the obvious difference being that it
> > takes name.
>
> The pertinent issue is hiding right here in plain view:
>
> Naming.
>
> The main relative for the "container" is not subtest group, but
> subtest. After all, it is an execution point, listed with
> --list-subtests and executed with --run-subtest.
>
> igt_dynamic_subtest_container is the best name I could come up with,
> and it absolutely sucks at conveying the purpose. Deciding on the name
> should be considered to be the most important part of reviewing this
> series, suggestions for that are very welcome!
Ok, this is going to look a bit ridiculous, but read this out:
igt_subtest_with_dynamic_subsubtests("xyz");
- conveys that it's a subtest, so will work with --run-subtest
- makes obvious that it's special because it *contains* something "dynamic"
- makes it clear that subsubtests nest under subtests
igt_dynamic_subsubtest - double sub-, but clarifies nesting
I am not able to come up with anything shorter that this, that would not
be just plain confusing.
--
Cheers,
Arek
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests
2019-08-28 10:38 ` Arkadiusz Hiler
@ 2019-08-28 12:58 ` Petri Latvala
0 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-28 12:58 UTC (permalink / raw)
To: Arkadiusz Hiler; +Cc: igt-dev
On Wed, Aug 28, 2019 at 01:38:44PM +0300, Arkadiusz Hiler wrote:
> On Wed, Aug 28, 2019 at 01:30:23PM +0300, Petri Latvala wrote:
> > On Wed, Aug 28, 2019 at 01:16:10PM +0300, Arkadiusz Hiler wrote:
> > > On Fri, Aug 16, 2019 at 12:34:17PM +0300, Petri Latvala wrote:
> > > > Dynamic subtests, or subtests of subtests, are individual pieces of
> > > > tests that are not statically available all the time.
> > > >
> > > > A good example of a need for a dynamic subtest is i915 engine listing:
> > > > A normal subtest for each engine class ("bsd"), and a dynamic subtest
> > > > for each instance ("bsd0", "bsd2", etc). Or a normal subtest for an
> > > > operation with a dynamic subtest for every engine there is.
> > > >
> > > > Another example is a dynamic subtest for pipes: Instead of using
> > > > foreach_pipe_static, make one subtest and use foreach_pipe with
> > > > dynamic subtests for each pipe.
> > > >
> > > > v2: Rebase and adapt to igt_describe changes
> > > >
> > > > Signed-off-by: Petri Latvala <petri.latvala@intel.com>
> > > > ---
> > > > lib/igt_core.c | 119 ++++++++++++++++++++++++++++++++++++++++++++-----
> > > > lib/igt_core.h | 89 ++++++++++++++++++++++++++++++++++++
> > > > 2 files changed, 197 insertions(+), 11 deletions(-)
> > > >
> > > > diff --git a/lib/igt_core.c b/lib/igt_core.c
> > > > index 1cbb09f9..8b754b9f 100644
> > > > --- a/lib/igt_core.c
> > > > +++ b/lib/igt_core.c
> > > > @@ -263,9 +263,12 @@ bool igt_skip_crc_compare;
> > > > static bool list_subtests = false;
> > > > static bool describe_subtests = false;
> > > > static char *run_single_subtest = NULL;
> > > > +static char *run_single_dynamic_subtest = NULL;
> > > > static bool run_single_subtest_found = false;
> > > > static const char *in_subtest = NULL;
> > > > +static const char *in_dynamic_subtest = NULL;
> > > > static struct timespec subtest_time;
> > > > +static struct timespec dynamic_subtest_time;
> > > > static clockid_t igt_clock = (clockid_t)-1;
> > > > static bool in_fixture = false;
> > > > static bool test_with_subtests = false;
> > > > @@ -300,6 +303,7 @@ enum {
> > > > OPT_LIST_SUBTESTS = 500,
> > > > OPT_DESCRIBE_SUBTESTS,
> > > > OPT_RUN_SUBTEST,
> > > > + OPT_RUN_DYNAMIC_SUBTEST,
> > > > OPT_DESCRIPTION,
> > > > OPT_DEBUG,
> > > > OPT_INTERACTIVE_DEBUG,
> > > > @@ -323,6 +327,8 @@ char *igt_frame_dump_path;
> > > >
> > > > static bool stderr_needs_sentinel = false;
> > > >
> > > > +static int _igt_dynamic_tests_executed = -1;
> > > > +
> > > > const char *igt_test_name(void)
> > > > {
> > > > return command_str;
> > > > @@ -354,7 +360,9 @@ static void _igt_log_buffer_dump(void)
> > > > {
> > > > uint8_t i;
> > > >
> > > > - if (in_subtest)
> > > > + if (in_dynamic_subtest)
> > > > + fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
> > > > + else if (in_subtest)
> > > > fprintf(stderr, "Subtest %s failed.\n", in_subtest);
> > > > else
> > > > fprintf(stderr, "Test %s failed.\n", command_str);
> > > > @@ -619,6 +627,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
> > > > fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
> > > > fprintf(f, " --list-subtests\n"
> > > > " --run-subtest <pattern>\n"
> > > > + " --dynamic-subtest <pattern>\n"
> > > > " --debug[=log-domain]\n"
> > > > " --interactive-debug[=domain]\n"
> > > > " --skip-crc-compare\n"
> > > > @@ -739,6 +748,7 @@ static int common_init(int *argc, char **argv,
> > > > {"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS},
> > > > {"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
> > > > {"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST},
> > > > + {"dynamic-subtest", required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
> > >
> > > This deviates from --run-subtests but even OPT_RUN_DYNAMIC_SUBTEST has
> > > "run" in it. I guess you did it to preserve '--r' shorthand?
> >
> > Yes :P
> >
> > > Also interaction betwen --rub-subtest and igt_dynamic_subtest_container
> > > is not very obvious.
> > >
> > > Same for using both --run-subtest and --dynamic-subtest.
> > >
> > > Maybe it's time to introduce short options?
> > >
> > > > {"help-description", no_argument, NULL, OPT_DESCRIPTION},
> > > > {"debug", optional_argument, NULL, OPT_DEBUG},
> > > > {"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
> > > > @@ -858,6 +868,11 @@ static int common_init(int *argc, char **argv,
> > > > if (!list_subtests)
> > > > run_single_subtest = strdup(optarg);
> > > > break;
> > > > + case OPT_RUN_DYNAMIC_SUBTEST:
> > > > + assert(optarg);
> > > > + if (!list_subtests)
> > > > + run_single_dynamic_subtest = strdup(optarg);
> > > > + break;
> > > > case OPT_DESCRIPTION:
> > > > print_test_description();
> > > > ret = -1;
> > > > @@ -1107,6 +1122,41 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> > > > return (in_subtest = subtest_name);
> > > > }
> > > >
> > > > +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
> > > > +{
> > > > + int i;
> > > > +
> > > > + assert(in_subtest);
> > > > + assert(_igt_dynamic_tests_executed >= 0);
> > > > +
> > > > + /* check the dynamic_subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
> > > > + for (i = 0; dynamic_subtest_name[i] != '\0'; i++)
> > > > + if (dynamic_subtest_name[i] != '_' && dynamic_subtest_name[i] != '-'
> > > > + && !isalnum(dynamic_subtest_name[i])) {
> > > > + igt_critical("Invalid dynamic subtest name \"%s\".\n",
> > > > + dynamic_subtest_name);
> > > > + igt_exit();
> > > > + }
> > >
> > > Same logic as for normal subtests, could use a helper?
> >
> > Yes! I'll do that for next revision.
> >
> >
> > >
> > > > +
> > > > + if (run_single_dynamic_subtest &&
> > > > + uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
> > > > + return false;
> > > > +
> > > > + igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
> > > > + command_str, dynamic_subtest_name);
> > > > + igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
> > > > + fflush(stdout);
> > > > + if (stderr_needs_sentinel)
> > > > + fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
> > > > +
> > > > + _igt_log_buffer_reset();
> > > > +
> > > > + _igt_dynamic_tests_executed++;
> > > > +
> > > > + igt_gettime(&dynamic_subtest_time);
> > > > + return (in_dynamic_subtest = dynamic_subtest_name);
> > > > +}
> > > > +
> > > > /**
> > > > * igt_subtest_name:
> > > > *
> > > > @@ -1161,26 +1211,50 @@ void __igt_subtest_group_restore(int save, int desc)
> > > > static bool skipped_one = false;
> > > > static bool succeeded_one = false;
> > > > static bool failed_one = false;
> > > > +static bool dynamic_failed_one = false;
> > > > +
> > > > +bool __igt_enter_dynamic_container(void)
> > > > +{
> > > > + _igt_dynamic_tests_executed = 0;
> > > > + dynamic_failed_one = false;
> > > > +
> > > > + return true;
> > > > +}
> > > >
> > > > static void exit_subtest(const char *) __attribute__((noreturn));
> > > > static void exit_subtest(const char *result)
> > > > {
> > > > struct timespec now;
> > > > + const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
> > > > + const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
> > > > + struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
> > > > + jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subtest_jmpbuf : &igt_subtest_jmpbuf;
> > > >
> > > > igt_gettime(&now);
> > > > - igt_info("%sSubtest %s: %s (%.3fs)%s\n",
> > > > +
> > > > + igt_info("%s%s %s: %s (%.3fs)%s\n",
> > > > (!__igt_plain_output) ? "\x1b[1m" : "",
> > > > - in_subtest, result, igt_time_elapsed(&subtest_time, &now),
> > > > + subtest_text, *subtest_name, result,
> > > > + igt_time_elapsed(thentime, &now),
> > > > (!__igt_plain_output) ? "\x1b[0m" : "");
> > > > fflush(stdout);
> > > > if (stderr_needs_sentinel)
> > > > - fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
> > > > - in_subtest, result, igt_time_elapsed(&subtest_time, &now));
> > > > + fprintf(stderr, "%s %s: %s (%.3fs)\n",
> > > > + subtest_text, *subtest_name,
> > > > + result, igt_time_elapsed(thentime, &now));
> > > >
> > > > igt_terminate_spins();
> > > >
> > > > - in_subtest = NULL;
> > > > - siglongjmp(igt_subtest_jmpbuf, 1);
> > > > + if (!in_dynamic_subtest)
> > > > + _igt_dynamic_tests_executed = -1;
> > > > +
> > > > + /* Don't keep the above text in the log, the container would print it again otherwise */
> > > > + if (in_dynamic_subtest)
> > > > + _igt_log_buffer_reset();
> > > > +
> > > > + *subtest_name = NULL;
> > > > +
> > > > + siglongjmp(*jmptarget, 1);
> > > > }
> > > >
> > > > /**
> > > > @@ -1211,6 +1285,7 @@ void igt_skip(const char *f, ...)
> > > > }
> > > >
> > > > if (in_subtest) {
> > > > + /* Doing the same even if inside a dynamic subtest */
> > > > exit_subtest("SKIP");
> > > > } else if (test_with_subtests) {
> > > > skip_subtests_henceforth = SKIP;
> > > > @@ -1267,7 +1342,22 @@ void __igt_skip_check(const char *file, const int line,
> > > > */
> > > > void igt_success(void)
> > > > {
> > > > - succeeded_one = true;
> > > > + if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
> > > > + /*
> > > > + * We're exiting a dynamic container, yield a result
> > > > + * according to the dynamic tests that got
> > > > + * executed.
> > > > + */
> > > > + if (dynamic_failed_one)
> > > > + igt_fail(IGT_EXIT_FAILURE);
> > > > +
> > > > + if (_igt_dynamic_tests_executed == 0)
> > > > + igt_skip("No dynamic tests executed.\n");
> > > > + }
> > > > +
> > > > + if (!in_dynamic_subtest)
> > > > + succeeded_one = true;
> > > > +
> > > > if (in_subtest)
> > > > exit_subtest("SUCCESS");
> > > > }
> > > > @@ -1298,10 +1388,17 @@ void igt_fail(int exitcode)
> > > > if (in_atexit_handler)
> > > > _exit(IGT_EXIT_FAILURE);
> > > >
> > > > - if (!failed_one)
> > > > - igt_exitcode = exitcode;
> > > > + if (in_dynamic_subtest) {
> > > > + dynamic_failed_one = true;
> > > > + } else {
> > > > + /* Dynamic subtest containers must not fail explicitly */
> > > > + assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
> > > > +
> > > > + if (!failed_one)
> > > > + igt_exitcode = exitcode;
> > > >
> > > > - failed_one = true;
> > > > + failed_one = true;
> > > > + }
> > > >
> > > > /* Silent exit, parent will do the yelling. */
> > > > if (test_child)
> > > > diff --git a/lib/igt_core.h b/lib/igt_core.h
> > > > index 177d2431..21289c8e 100644
> > > > --- a/lib/igt_core.h
> > > > +++ b/lib/igt_core.h
> > > > @@ -144,6 +144,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
> > > >
> > > > /* subtest infrastructure */
> > > > jmp_buf igt_subtest_jmpbuf;
> > > > +jmp_buf igt_dynamic_subtest_jmpbuf;
> > > > typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
> > > > #define IGT_OPT_HANDLER_SUCCESS 0
> > > > #define IGT_OPT_HANDLER_ERROR -2
> > > > @@ -175,6 +176,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
> > > > igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
> > > >
> > > > bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
> > > > +bool __igt_enter_dynamic_container(void);
> > > > +bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
> > > > #define __igt_tokencat2(x, y) x ## y
> > > >
> > > > /**
> > > > @@ -224,6 +227,92 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
> > > > #define igt_subtest_f(f...) \
> > > > __igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
> > > >
> > > > +/**
> > > > + * igt_dynamic_subtest_container:
> > > > + * @name: name of the subtest
> > > > + *
> > > > + * This is a magic control flow block which denotes a subtest code
> > > > + * block that contains dynamic subtests. The _f variant accepts a
> > > > + * printf format string, which is useful for constructing
> > > > + * combinatorial tests.
> > > > + *
> > > > + * Within a dynamic subtest container, explicit failure
> > > > + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> > > > + * will produce test results. igt_skip()/igt_require() is allowed.
> > > > + *
> > > > + * This is a simpler version of igt_dynamic_subtest_container_f()
> > > > + */
> > > > +#define igt_dynamic_subtest_container(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
> > > > + __igt_enter_dynamic_container() && \
> > > > + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> > > > + igt_success())
> > > > +#define __igt_dynamic_subtest_container_f(tmp, format...) \
> > > > + for (char tmp [256]; \
> > > > + snprintf( tmp , sizeof( tmp ), \
> > > > + format), \
> > > > + __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
> > > > + __igt_enter_dynamic_container() && \
> > > > + (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
> > > > + igt_success())
> > > > +
> > > > +/**
> > > > + * igt_dynamic_subtest_container_f:
> > > > + * @...: format string and optional arguments
> > > > + *
> > > > + * This is a magic control flow block which denotes a subtest code
> > > > + * block that contains dynamic subtests. The _f variant accepts a
> > > > + * printf format string, which is useful for constructing
> > > > + * combinatorial tests.
> > > > + *
> > > > + * Within a dynamic subtest container, explicit failure
> > > > + * (e.g. igt_assert) is not allowed, only dynamic subtests themselves
> > > > + * will produce test results. igt_skip()/igt_require() is allowed.
> > > > + *
> > > > + * Like igt_dynamic_subtest_container(), but also accepts a printf
> > > > + * format string instead of a static string.
> > > > + */
> > >
> > > I think we need a bit more of explanation here. It's hard to understand
> > > where the asserts go and that this is more like a subtest group and you
> > > have to nest igt_dynamic_subtest under it.
> > >
> > > I think some pseudocode could be usefule, e.g.:
> > >
> > > igt_main {
> > > igt_dynamic_subtest_container("engine-tests") {
> > > /* requires ok, no asserts */
> > > igt_require(is_awesome(fd));
> > >
> > > for_each_engine(e) {
> > > igt_dynamic_subtest_f("%s", e->name) {
> > > /* asserts ok! */
> > > igt_fail();
> > > }
> > > }
> > > }
> > > }
> > >
> > > It's also not very clear how this will be reported out.
> >
> > Yeah, your recent examples of embedding code into documentation with
> > the igt_describe code now makes it possible for me to write better
> > docs. Noted.
> >
> > > Or maybe even use "group" instead of "container" here to build the
> > > parallel with normal subtest group, the obvious difference being that it
> > > takes name.
> >
> > The pertinent issue is hiding right here in plain view:
> >
> > Naming.
> >
> > The main relative for the "container" is not subtest group, but
> > subtest. After all, it is an execution point, listed with
> > --list-subtests and executed with --run-subtest.
> >
> > igt_dynamic_subtest_container is the best name I could come up with,
> > and it absolutely sucks at conveying the purpose. Deciding on the name
> > should be considered to be the most important part of reviewing this
> > series, suggestions for that are very welcome!
>
> Ok, this is going to look a bit ridiculous, but read this out:
>
> igt_subtest_with_dynamic_subsubtests("xyz");
> - conveys that it's a subtest, so will work with --run-subtest
> - makes obvious that it's special because it *contains* something "dynamic"
> - makes it clear that subsubtests nest under subtests
>
> igt_dynamic_subsubtest - double sub-, but clarifies nesting
>
> I am not able to come up with anything shorter that this, that would not
> be just plain confusing.
Those sound good enough. Unless anything better appears, I'm going
with those.
--
Petri Latvala
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread
* [igt-dev] [PATCH i-g-t 02/10] lib/tests: Unit tests for dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 03/10] lib/tests: Test that igt_describe works with " Petri Latvala
` (9 subsequent siblings)
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
lib/tests/igt_dynamic_subtests.c | 186 +++++++++++++++++++++++++++++++
lib/tests/meson.build | 1 +
2 files changed, 187 insertions(+)
create mode 100644 lib/tests/igt_dynamic_subtests.c
diff --git a/lib/tests/igt_dynamic_subtests.c b/lib/tests/igt_dynamic_subtests.c
new file mode 100644
index 00000000..0e693744
--- /dev/null
+++ b/lib/tests/igt_dynamic_subtests.c
@@ -0,0 +1,186 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <errno.h>
+#include <sys/wait.h>
+
+#include "igt_core.h"
+#include "drmtest.h"
+
+#include "igt_tests_common.h"
+
+static int do_fork(void (*test_to_run)(void))
+{
+ int pid, status;
+
+ switch (pid = fork()) {
+ case -1:
+ internal_assert(0);
+ case 0:
+ test_to_run();
+ default:
+ while (waitpid(pid, &status, 0) == -1 &&
+ errno == EINTR)
+ ;
+
+ return status;
+ }
+}
+
+static void dynamic_subtest_in_normal_subtest(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_subtest("normal-subtest") {
+ igt_dynamic_subtest("dynamic") {
+ igt_info("Dynamic subtest in normal subtest\n");
+ }
+ }
+
+ igt_exit();
+}
+
+static void invalid_dynamic_subtest_name(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_dynamic_subtest_container("subtest") {
+ igt_dynamic_subtest("# invalid name !") {
+ igt_info("Invalid dynamic subtest name test\n");
+ }
+ }
+
+ igt_exit();
+}
+
+static void dynamic_subtest_in_toplevel(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_dynamic_subtest("dynamic-subtest-in-toplevel") {
+ igt_info("Dynamic subtests need to be in a subtest\n");
+ }
+
+ igt_exit();
+}
+
+static void subtest_itself_failing(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_dynamic_subtest_container("subtest") {
+ igt_assert(false);
+ }
+
+ igt_exit();
+}
+
+static void subtest_itself_skipping(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_dynamic_subtest_container("subtest") {
+ igt_skip("Skipping\n");
+ }
+
+ igt_exit();
+}
+
+static void dynamic_subtest_failure_leads_to_fail(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_dynamic_subtest_container("subtest") {
+ igt_dynamic_subtest("dynamic") {
+ igt_assert(false);
+ }
+ }
+
+ igt_exit();
+}
+
+static void no_dynamic_subtests_entered_leads_to_skip(void)
+{
+ char prog[] = "igt_no_exit";
+ char *fake_argv[] = {prog};
+ int fake_argc = ARRAY_SIZE(fake_argv);
+
+ igt_subtest_init(fake_argc, fake_argv);
+
+ igt_dynamic_subtest_container("subtest") {
+ }
+
+ igt_exit();
+}
+
+int main(int argc, char **argv)
+{
+ int ret;
+
+ ret = do_fork(dynamic_subtest_in_normal_subtest);
+ internal_assert_wsignaled(ret, SIGABRT);
+
+ ret = do_fork(invalid_dynamic_subtest_name);
+ internal_assert_wsignaled(ret, SIGABRT);
+
+ ret = do_fork(dynamic_subtest_in_toplevel);
+ internal_assert_wsignaled(ret, SIGABRT);
+
+ ret = do_fork(subtest_itself_failing);
+ internal_assert_wsignaled(ret, SIGABRT);
+
+ ret = do_fork(subtest_itself_skipping);
+ internal_assert_wexited(ret, IGT_EXIT_SKIP);
+
+ ret = do_fork(dynamic_subtest_failure_leads_to_fail);
+ internal_assert_wexited(ret, IGT_EXIT_FAILURE);
+
+ ret = do_fork(no_dynamic_subtests_entered_leads_to_skip);
+ internal_assert_wexited(ret, IGT_EXIT_SKIP);
+
+ return 0;
+}
diff --git a/lib/tests/meson.build b/lib/tests/meson.build
index 0c35da30..e5066665 100644
--- a/lib/tests/meson.build
+++ b/lib/tests/meson.build
@@ -4,6 +4,7 @@ lib_tests = [
'igt_can_fail_simple',
'igt_conflicting_args',
'igt_describe',
+ 'igt_dynamic_subtests',
'igt_edid',
'igt_exit_handler',
'igt_fork',
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 03/10] lib/tests: Test that igt_describe works with dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 01/10] lib: Introduce dynamic subtests Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 02/10] lib/tests: Unit tests for " Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-28 12:33 ` Arkadiusz Hiler
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 04/10] runner/resultgen: Refactor output parsing Petri Latvala
` (8 subsequent siblings)
11 siblings, 1 reply; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
lib/tests/igt_describe.c | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/tests/igt_describe.c b/lib/tests/igt_describe.c
index 2ea47e9d..70ec055e 100644
--- a/lib/tests/igt_describe.c
+++ b/lib/tests/igt_describe.c
@@ -87,6 +87,13 @@ static void fake_main(int argc, char **argv) {
igt_subtest("F")
;
+ igt_describe("Dynamic container");
+ igt_dynamic_subtest_container("G") {
+ printf("should not be executed!\n");
+ igt_dynamic_subtest("should-not-list")
+ printf("should not be executed!\n");
+ }
+
igt_exit();
}
@@ -129,7 +136,10 @@ static const char DESCRIBE_ALL_OUTPUT[] = \
"\n"
"SUB F ../lib/tests/igt_describe.c:87:\n"
" verylongwordthatshoudlbeprintedeventhoughitspastthewrppinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n"
- " verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n\n";
+ " verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n"
+ "\n"
+ "SUB G ../lib/tests/igt_describe.c:91:\n"
+ " Dynamic container\n\n";
static const char JUST_C_OUTPUT[] = \
"the top level description\n"
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* Re: [igt-dev] [PATCH i-g-t 03/10] lib/tests: Test that igt_describe works with dynamic subtests
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 03/10] lib/tests: Test that igt_describe works with " Petri Latvala
@ 2019-08-28 12:33 ` Arkadiusz Hiler
2019-08-28 12:39 ` Petri Latvala
0 siblings, 1 reply; 20+ messages in thread
From: Arkadiusz Hiler @ 2019-08-28 12:33 UTC (permalink / raw)
To: Petri Latvala; +Cc: igt-dev
On Fri, Aug 16, 2019 at 12:34:19PM +0300, Petri Latvala wrote:
> Signed-off-by: Petri Latvala <petri.latvala@intel.com>
> ---
> lib/tests/igt_describe.c | 12 +++++++++++-
> 1 file changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/lib/tests/igt_describe.c b/lib/tests/igt_describe.c
> index 2ea47e9d..70ec055e 100644
> --- a/lib/tests/igt_describe.c
> +++ b/lib/tests/igt_describe.c
> @@ -87,6 +87,13 @@ static void fake_main(int argc, char **argv) {
> igt_subtest("F")
> ;
>
> + igt_describe("Dynamic container");
> + igt_dynamic_subtest_container("G") {
> + printf("should not be executed!\n");
> + igt_dynamic_subtest("should-not-list")
Have you tested this for:
igt_describe("Dynamic subtest");
igt_dynamic_subtest("should-not-list") {}
I think this has potential to mess up few things and should be
disallowed with a loud error message.
> + printf("should not be executed!\n");
> + }
> +
> igt_exit();
> }
>
> @@ -129,7 +136,10 @@ static const char DESCRIBE_ALL_OUTPUT[] = \
> "\n"
> "SUB F ../lib/tests/igt_describe.c:87:\n"
> " verylongwordthatshoudlbeprintedeventhoughitspastthewrppinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n"
> - " verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n\n";
> + " verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n"
> + "\n"
> + "SUB G ../lib/tests/igt_describe.c:91:\n"
> + " Dynamic container\n\n";
>
> static const char JUST_C_OUTPUT[] = \
> "the top level description\n"
> --
> 2.19.1
>
> _______________________________________________
> igt-dev mailing list
> igt-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/igt-dev
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread* Re: [igt-dev] [PATCH i-g-t 03/10] lib/tests: Test that igt_describe works with dynamic subtests
2019-08-28 12:33 ` Arkadiusz Hiler
@ 2019-08-28 12:39 ` Petri Latvala
0 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-28 12:39 UTC (permalink / raw)
To: Arkadiusz Hiler; +Cc: igt-dev
On Wed, Aug 28, 2019 at 03:33:09PM +0300, Arkadiusz Hiler wrote:
> On Fri, Aug 16, 2019 at 12:34:19PM +0300, Petri Latvala wrote:
> > Signed-off-by: Petri Latvala <petri.latvala@intel.com>
> > ---
> > lib/tests/igt_describe.c | 12 +++++++++++-
> > 1 file changed, 11 insertions(+), 1 deletion(-)
> >
> > diff --git a/lib/tests/igt_describe.c b/lib/tests/igt_describe.c
> > index 2ea47e9d..70ec055e 100644
> > --- a/lib/tests/igt_describe.c
> > +++ b/lib/tests/igt_describe.c
> > @@ -87,6 +87,13 @@ static void fake_main(int argc, char **argv) {
> > igt_subtest("F")
> > ;
> >
> > + igt_describe("Dynamic container");
> > + igt_dynamic_subtest_container("G") {
> > + printf("should not be executed!\n");
>
> > + igt_dynamic_subtest("should-not-list")
>
> Have you tested this for:
> igt_describe("Dynamic subtest");
> igt_dynamic_subtest("should-not-list") {}
>
>
> I think this has potential to mess up few things and should be
> disallowed with a loud error message.
You mean someone accidentally calling igt_describe for a dynamic
subtest, expecting it to appear in docs somewhere? Yeah that should be
easy, coming up in next revision.
--
Petri Latvala
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread
* [igt-dev] [PATCH i-g-t 04/10] runner/resultgen: Refactor output parsing
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (2 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 03/10] lib/tests: Test that igt_describe works with " Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-28 11:48 ` Arkadiusz Hiler
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 05/10] runner/json_tests: Adapt to better " Petri Latvala
` (7 subsequent siblings)
11 siblings, 1 reply; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Instead of searching back and forth for proper lines, first find all
lines that we could be interested in with one pass through the output,
and use the positions of found lines to delimit the extracted outputs.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
runner/resultgen.c | 259 +++++++++++++++++++++++----------------------
1 file changed, 134 insertions(+), 125 deletions(-)
diff --git a/runner/resultgen.c b/runner/resultgen.c
index 58b95220..dabf8b98 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -10,6 +10,7 @@
#include <json.h>
+#include "igt_aux.h"
#include "igt_core.h"
#include "resultgen.h"
#include "settings.h"
@@ -59,28 +60,7 @@ static char *find_line_starting_with(char *haystack, const char *needle, char *e
return NULL;
}
-static char *find_line_starting_with_either(char *haystack,
- const char *needle1,
- const char *needle2,
- char *end)
-{
- while (haystack < end) {
- char *line_end = memchr(haystack, '\n', end - haystack);
- size_t linelen = line_end != NULL ? line_end - haystack : end - haystack;
- if ((linelen >= strlen(needle1) && !memcmp(haystack, needle1, strlen(needle1))) ||
- (linelen >= strlen(needle2) && !memcmp(haystack, needle2, strlen(needle2))))
- return haystack;
-
- if (line_end == NULL)
- return NULL;
-
- haystack = line_end + 1;
- }
-
- return NULL;
-}
-
-static char *next_line(char *line, char *bufend)
+static const char *next_line(const char *line, const char *bufend)
{
char *ret;
@@ -97,45 +77,6 @@ static char *next_line(char *line, char *bufend)
return NULL;
}
-static char *find_line_after_last(char *begin,
- const char *needle1,
- const char *needle2,
- char *end)
-{
- char *one, *two;
- char *current_pos = begin;
- char *needle1_newline = malloc(strlen(needle1) + 2);
- char *needle2_newline = malloc(strlen(needle2) + 2);
-
- needle1_newline[0] = needle2_newline[0] = '\n';
- strcpy(needle1_newline + 1, needle1);
- strcpy(needle2_newline + 1, needle2);
-
- while (true) {
- one = memmem(current_pos, end - current_pos, needle1_newline, strlen(needle1_newline));
- two = memmem(current_pos, end - current_pos, needle2_newline, strlen(needle2_newline));
- if (one == NULL && two == NULL)
- break;
-
- if (one != NULL && current_pos < one)
- current_pos = one;
- if (two != NULL && current_pos < two)
- current_pos = two;
-
- one = next_line(current_pos, end);
- if (one != NULL)
- current_pos = one;
- }
- free(needle1_newline);
- free(needle2_newline);
-
- one = memchr(current_pos, '\n', end - current_pos);
- if (one != NULL)
- return ++one;
-
- return current_pos;
-}
-
static size_t count_lines(const char *buf, const char *bufend)
{
size_t ret = 0;
@@ -166,7 +107,7 @@ static const struct {
{ "CRASH", "crash" },
{ "TIMEOUT", "timeout" },
};
-static void parse_result_string(char *resultstring, size_t len, const char **result, double *time)
+static void parse_result_string(const char *resultstring, size_t len, const char **result, double *time)
{
size_t i;
size_t wordlen = 0;
@@ -206,19 +147,20 @@ static void parse_result_string(char *resultstring, size_t len, const char **res
}
}
-static void parse_subtest_result(char *subtest, const char **result, double *time, char *buf, char *bufend)
+static void parse_subtest_result(const char *subtest,
+ const char **result,
+ double *time,
+ const char *line,
+ const char *bufend)
{
- char *line;
- char *line_end;
- char *resultstring;
- size_t linelen;
+ const char *resultstring;
size_t subtestlen = strlen(subtest);
+ const char *line_end;
+ size_t linelen;
- *result = NULL;
+ *result = "incomplete";
*time = 0.0;
- if (!buf) return;
-
/*
* The result line structure is:
*
@@ -235,18 +177,17 @@ static void parse_subtest_result(char *subtest, const char **result, double *tim
* Subtest subtestname: PASS (0.003s)
*/
- line = find_line_starting_with(buf, SUBTEST_RESULT, bufend);
- if (!line) {
- *result = "incomplete";
+ if (!line)
return;
- }
line_end = memchr(line, '\n', bufend - line);
linelen = line_end != NULL ? line_end - line : bufend - line;
if (strlen(SUBTEST_RESULT) + subtestlen + strlen(": ") > linelen ||
- strncmp(line + strlen(SUBTEST_RESULT), subtest, subtestlen))
- return parse_subtest_result(subtest, result, time, line + linelen, bufend);
+ strncmp(line + strlen(SUBTEST_RESULT), subtest, subtestlen)) {
+ /* This is not the correct result line */
+ return;
+ }
resultstring = line + strlen(SUBTEST_RESULT) + subtestlen + strlen(": ");
parse_result_string(resultstring, linelen - (resultstring - line), result, time);
@@ -308,6 +249,59 @@ static void set_runtime(struct json_object *obj, double time)
json_object_new_double(time));
}
+struct match_item
+{
+ const char *where;
+ const char *what;
+};
+
+struct matches
+{
+ struct match_item *items;
+ size_t size;
+};
+
+static void match_add(struct matches *matches, const char *where, const char *what)
+{
+ struct match_item newitem = { where, what };
+
+ matches->size++;
+ matches->items = realloc(matches->items, matches->size * sizeof(*matches->items));
+ matches->items[matches->size - 1] = newitem;
+}
+
+static struct matches find_matches(const char *buf, const char *bufend,
+ const char **needles)
+{
+ struct matches ret = {};
+
+ while (buf < bufend) {
+ const char **needle;
+
+ for (needle = needles; *needle; needle++) {
+ if (bufend - buf < strlen(*needle))
+ continue;
+
+ if (!memcmp(buf, *needle, strlen(*needle))) {
+ match_add(&ret, buf, *needle);
+ goto end_find;
+ }
+ }
+
+ end_find:
+ buf = next_line(buf, bufend);
+ if (!buf)
+ break;
+ }
+
+ return ret;
+}
+
+static void free_matches(struct matches *matches)
+{
+ free(matches->items);
+}
+
static bool fill_from_output(int fd, const char *binary, const char *key,
struct subtests *subtests,
struct json_object *tests)
@@ -318,7 +312,13 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
char *igt_version = NULL;
size_t igt_version_len = 0;
struct json_object *current_test = NULL;
- size_t i;
+ const char *needles[] = {
+ STARTING_SUBTEST,
+ SUBTEST_RESULT,
+ NULL
+ };
+ struct matches matches = {};
+ size_t i, k;
if (fstat(fd, &statbuf))
return false;
@@ -363,10 +363,13 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
return true;
}
+ matches = find_matches(buf, bufend, needles);
+
for (i = 0; i < subtests->size; i++) {
char *this_sub_begin, *this_sub_result;
+ int begin_idx = -1, result_idx = -1;
const char *resulttext;
- char *beg, *end, *startline;
+ const char *beg, *end;
double time;
int begin_len;
int result_len;
@@ -382,66 +385,68 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
return false;
}
- beg = find_line_starting_with(buf, this_sub_begin, bufend);
- end = find_line_starting_with(buf, this_sub_result, bufend);
- startline = beg;
+ for (k = 0; k < matches.size; k++) {
+ if (matches.items[k].what == STARTING_SUBTEST &&
+ !memcmp(matches.items[k].where,
+ this_sub_begin,
+ min(begin_len, bufend - matches.items[k].where))) {
+ beg = matches.items[k].where;
+ begin_idx = k;
+ }
+ if (matches.items[k].what == SUBTEST_RESULT &&
+ !memcmp(matches.items[k].where,
+ this_sub_result,
+ min(result_len, bufend - matches.items[k].where))) {
+ end = matches.items[k].where;
+ result_idx = k;
+ }
+ }
free(this_sub_begin);
free(this_sub_result);
- if (beg == NULL && end == NULL) {
+ if (begin_idx < 0 && result_idx < 0) {
/* No output at all */
beg = bufend;
end = bufend;
- }
-
- if (beg == NULL) {
+ } else if (begin_idx < 0) {
/*
- * Subtest didn't start, probably skipped from
- * fixture already. Start from the result
- * line, it gets adjusted below.
+ * Subtest didn't start, but we have the
+ * result. Probably because an igt_fixture
+ * made it fail/skip.
+ *
+ * Start the output right after the previous
+ * subtest result/start.
*/
- beg = end;
+ if (result_idx > 0)
+ beg = next_line(matches.items[result_idx - 1].where, bufend);
+ else
+ beg = buf;
+ } else {
+ /* Stretch output beginning backwards */
+ if (begin_idx == 0)
+ beg = buf;
+ else
+ beg = next_line(matches.items[begin_idx - 1].where, bufend);
}
- /* Include the output after the previous subtest output */
- beg = find_line_after_last(buf,
- STARTING_SUBTEST,
- SUBTEST_RESULT,
- beg);
-
- if (end == NULL) {
- /* Incomplete result. Find the next starting subtest or result. */
- end = next_line(startline, bufend);
- if (end != NULL) {
- end = find_line_starting_with_either(end,
- STARTING_SUBTEST,
- SUBTEST_RESULT,
- bufend);
- }
- if (end == NULL) {
- end = bufend;
- }
- } else {
+ if (result_idx < 0 && begin_idx < 0) {
+ /* No output at all: Already handled above */
+ } else if (result_idx < 0) {
/*
- * Now pointing to the line where this sub's
- * result is. We need to include that of
- * course.
+ * Incomplete result. Include output up to the
+ * next starting subtest or result.
*/
- char *nexttest = next_line(end, bufend);
-
- /* Stretch onwards until the next subtest begins or ends */
- if (nexttest != NULL) {
- nexttest = find_line_starting_with_either(nexttest,
- STARTING_SUBTEST,
- SUBTEST_RESULT,
- bufend);
- }
- if (nexttest != NULL) {
- end = nexttest;
- } else {
+ if (begin_idx < matches.size - 1)
+ end = matches.items[begin_idx + 1].where;
+ else
+ end = bufend;
+ } else {
+ /* Stretch output to the next starting subtest or result. */
+ if (result_idx < matches.size - 1)
+ end = matches.items[result_idx + 1].where;
+ else
end = bufend;
- }
}
json_object_object_add(current_test, key,
@@ -454,12 +459,16 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
}
if (!json_object_object_get_ex(current_test, "result", NULL)) {
- parse_subtest_result(subtests->names[i], &resulttext, &time, beg, end);
+ parse_subtest_result(subtests->names[i],
+ &resulttext, &time,
+ result_idx < 0 ? NULL : matches.items[result_idx].where,
+ end);
set_result(current_test, resulttext);
set_runtime(current_test, time);
}
}
+ free_matches(&matches);
return true;
}
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 05/10] runner/json_tests: Adapt to better output parsing
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (3 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 04/10] runner/resultgen: Refactor output parsing Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 06/10] runner: Parse dynamic subtest outputs and results Petri Latvala
` (6 subsequent siblings)
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Since we actually include the output before the subtest begins now,
add it to the reference.jsons where applicable.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
.../json_tests_data/aborted-after-a-test/reference.json | 2 +-
runner/json_tests_data/dmesg-escapes/reference.json | 2 +-
runner/json_tests_data/dmesg-results/reference.json | 8 ++++----
.../dmesg-warn-level-one-piglit-style/reference.json | 4 ++--
.../dmesg-warn-level-piglit-style/reference.json | 4 ++--
runner/json_tests_data/dmesg-warn-level/reference.json | 4 ++--
runner/json_tests_data/normal-run/reference.json | 8 ++++----
.../notrun-results-multiple-mode/reference.json | 2 +-
runner/json_tests_data/notrun-results/reference.json | 2 +-
runner/json_tests_data/piglit-style-dmesg/reference.json | 8 ++++----
.../warnings-with-dmesg-warns/reference.json | 8 ++++----
runner/json_tests_data/warnings/reference.json | 8 ++++----
12 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/runner/json_tests_data/aborted-after-a-test/reference.json b/runner/json_tests_data/aborted-after-a-test/reference.json
index 06a8bff6..583242c7 100644
--- a/runner/json_tests_data/aborted-after-a-test/reference.json
+++ b/runner/json_tests_data/aborted-after-a-test/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
diff --git a/runner/json_tests_data/dmesg-escapes/reference.json b/runner/json_tests_data/dmesg-escapes/reference.json
index a6a70109..70d6b366 100644
--- a/runner/json_tests_data/dmesg-escapes/reference.json
+++ b/runner/json_tests_data/dmesg-escapes/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json
index 445cd0a0..0edbae93 100644
--- a/runner/json_tests_data/dmesg-results/reference.json
+++ b/runner/json_tests_data/dmesg-results/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"dmesg-warn",
"time":{
@@ -23,7 +23,7 @@
"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
},
"igt@successtest@second-subtest":{
- "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"dmesg-fail",
"time":{
@@ -49,7 +49,7 @@
"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
},
"igt@skippers@skip-one":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
@@ -61,7 +61,7 @@
"dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-two":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
index 520edf08..4ccb18ae 100644
--- a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"dmesg-warn",
"time":{
@@ -70,4 +70,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
index 8cd2eeee..d706ee4c 100644
--- a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"dmesg-warn",
"time":{
@@ -70,4 +70,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/runner/json_tests_data/dmesg-warn-level/reference.json b/runner/json_tests_data/dmesg-warn-level/reference.json
index 93d9c6ec..11cc39d9 100644
--- a/runner/json_tests_data/dmesg-warn-level/reference.json
+++ b/runner/json_tests_data/dmesg-warn-level/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"dmesg-warn",
"time":{
@@ -70,4 +70,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/runner/json_tests_data/normal-run/reference.json b/runner/json_tests_data/normal-run/reference.json
index 5e62579b..982038f9 100644
--- a/runner/json_tests_data/normal-run/reference.json
+++ b/runner/json_tests_data/normal-run/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
@@ -22,7 +22,7 @@
"dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
},
"igt@successtest@second-subtest":{
- "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"fail",
"time":{
@@ -46,7 +46,7 @@
"dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-one":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
@@ -58,7 +58,7 @@
"dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-two":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/reference.json b/runner/json_tests_data/notrun-results-multiple-mode/reference.json
index de1c3c31..492c0a9e 100644
--- a/runner/json_tests_data/notrun-results-multiple-mode/reference.json
+++ b/runner/json_tests_data/notrun-results-multiple-mode/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
diff --git a/runner/json_tests_data/notrun-results/reference.json b/runner/json_tests_data/notrun-results/reference.json
index 6b5ff69b..49a2f693 100644
--- a/runner/json_tests_data/notrun-results/reference.json
+++ b/runner/json_tests_data/notrun-results/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
diff --git a/runner/json_tests_data/piglit-style-dmesg/reference.json b/runner/json_tests_data/piglit-style-dmesg/reference.json
index 59f46595..45d6108e 100644
--- a/runner/json_tests_data/piglit-style-dmesg/reference.json
+++ b/runner/json_tests_data/piglit-style-dmesg/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"warn",
"time":{
@@ -22,7 +22,7 @@
"dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
},
"igt@successtest@second-subtest":{
- "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
@@ -46,7 +46,7 @@
"dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-one":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
@@ -58,7 +58,7 @@
"dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-two":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
index f70990e8..fa571703 100644
--- a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"dmesg-warn",
"time":{
@@ -23,7 +23,7 @@
"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
},
"igt@successtest@second-subtest":{
- "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
@@ -47,7 +47,7 @@
"dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-one":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
@@ -59,7 +59,7 @@
"dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-two":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
diff --git a/runner/json_tests_data/warnings/reference.json b/runner/json_tests_data/warnings/reference.json
index dade0439..53e0c3c7 100644
--- a/runner/json_tests_data/warnings/reference.json
+++ b/runner/json_tests_data/warnings/reference.json
@@ -10,7 +10,7 @@
},
"tests":{
"igt@successtest@first-subtest":{
- "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"warn",
"time":{
@@ -22,7 +22,7 @@
"dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
},
"igt@successtest@second-subtest":{
- "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"pass",
"time":{
@@ -46,7 +46,7 @@
"dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-one":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
@@ -58,7 +58,7 @@
"dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
},
"igt@skippers@skip-two":{
- "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nTest requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
"igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
"result":"skip",
"time":{
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 06/10] runner: Parse dynamic subtest outputs and results
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (4 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 05/10] runner/json_tests: Adapt to better " Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 07/10] runner/json_tests: Test dynamic subtests Petri Latvala
` (5 subsequent siblings)
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
If binary 'bin' has a subtest 'sub', which has dynamic subtests 'foo'
and 'bar', results.json will now have "subtests" by the names
igt@bin@sub@foo and igt@bin@sub@bar, with data as expected of normal
subtests.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
runner/job_list.c | 11 ++
runner/job_list.h | 3 +
runner/output_strings.h | 29 +++-
runner/resultgen.c | 377 ++++++++++++++++++++++++++++++++--------
4 files changed, 346 insertions(+), 74 deletions(-)
diff --git a/runner/job_list.c b/runner/job_list.c
index 5283fd72..93cede75 100644
--- a/runner/job_list.c
+++ b/runner/job_list.c
@@ -357,6 +357,17 @@ void generate_piglit_name(const char *binary, const char *subtest,
free(lc_subtest);
}
+void generate_piglit_name_for_dynamic(const char *base_piglit_name,
+ const char *dynamic_subtest,
+ char *namebuf, size_t namebuf_size)
+{
+ char *lc_dynamic = lowercase(dynamic_subtest);
+
+ snprintf(namebuf, namebuf_size, "%s@%s", base_piglit_name, lc_dynamic);
+
+ free(lc_dynamic);
+}
+
void init_job_list(struct job_list *job_list)
{
memset(job_list, 0, sizeof(*job_list));
diff --git a/runner/job_list.h b/runner/job_list.h
index 39c9b863..30b4a252 100644
--- a/runner/job_list.h
+++ b/runner/job_list.h
@@ -29,6 +29,9 @@ struct job_list
void generate_piglit_name(const char *binary, const char *subtest,
char *namebuf, size_t namebuf_size);
+void generate_piglit_name_for_dynamic(const char *base_piglit_name,
+ const char *dynamic_subtest,
+ char *namebuf, size_t namebuf_size);
void init_job_list(struct job_list *job_list);
void free_job_list(struct job_list *job_list);
diff --git a/runner/output_strings.h b/runner/output_strings.h
index 1e52c2ce..892895ea 100644
--- a/runner/output_strings.h
+++ b/runner/output_strings.h
@@ -15,10 +15,28 @@ static const char STARTING_SUBTEST[] = "Starting subtest: ";
*
* Examples:
* Subtest subtestname: SKIP
- * Subtest subtestname: PASS (0.003s)
+ * Subtest subtestname: SUCCESS (0.003s)
*/
static const char SUBTEST_RESULT[] = "Subtest ";
+/*
+ * Output when a dynamic subtest has begun. Is followed by the subtest name.
+ *
+ * Example:
+ * Starting dynamic subtest: subtestname
+ */
+static const char STARTING_DYNAMIC_SUBTEST[] = "Starting dynamic subtest: ";
+
+/*
+ * Output when a dynamic subtest has ended. Is followed by the subtest name
+ * and optionally its runtime.
+ *
+ * Examples:
+ * Dynamic subtest subtestname: SKIP
+ * Dynamic subtest subtestname: SUCCESS (0.003s)
+ */
+static const char DYNAMIC_SUBTEST_RESULT[] = "Dynamic subtest ";
+
/*
* Output in dmesg when a subtest has begin. Is followed by the subtest name.
*
@@ -27,6 +45,15 @@ static const char SUBTEST_RESULT[] = "Subtest ";
*/
static const char STARTING_SUBTEST_DMESG[] = ": starting subtest ";
+/*
+ * Output in dmesg when a dynamic subtest has begin. Is followed by
+ * the subtest name.
+ *
+ * Example:
+ * [IGT] test-binary-name: starting dynamic subtest subtestname
+ */
+static const char STARTING_DYNAMIC_SUBTEST_DMESG[] = ": starting dynamic subtest ";
+
/*
* Output when a test process is executed.
*
diff --git a/runner/resultgen.c b/runner/resultgen.c
index dabf8b98..ec74ddf1 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -23,9 +23,16 @@ _Static_assert(INCOMPLETE_EXITCODE != IGT_EXIT_SKIP, "exit code clash");
_Static_assert(INCOMPLETE_EXITCODE != IGT_EXIT_SUCCESS, "exit code clash");
_Static_assert(INCOMPLETE_EXITCODE != IGT_EXIT_INVALID, "exit code clash");
-struct subtests
+struct subtest
{
- char **names;
+ char *name;
+ char **dynamic_names;
+ size_t dynamic_size;
+};
+
+struct subtest_list
+{
+ struct subtest *subs;
size_t size;
};
@@ -36,6 +43,67 @@ struct results
struct json_object *runtimes;
};
+static void add_dynamic_subtest(struct subtest *subtest, char *dynamic)
+{
+ size_t len = strlen(dynamic);
+ size_t i;
+
+ if (len == 0)
+ return;
+
+ if (dynamic[len - 1] == '\n')
+ dynamic[len - 1] = '\0';
+
+ /* Don't add if we already have this one */
+ for (i = 0; i < subtest->dynamic_size; i++)
+ if (!strcmp(dynamic, subtest->dynamic_names[i]))
+ return;
+
+ subtest->dynamic_size++;
+ subtest->dynamic_names = realloc(subtest->dynamic_names, sizeof(*subtest->dynamic_names) * subtest->dynamic_size);
+ subtest->dynamic_names[subtest->dynamic_size - 1] = dynamic;
+}
+
+static void add_subtest(struct subtest_list *subtests, char *subtest)
+{
+ size_t len = strlen(subtest);
+ size_t i;
+
+ if (len == 0)
+ return;
+
+ if (subtest[len - 1] == '\n')
+ subtest[len - 1] = '\0';
+
+ /* Don't add if we already have this subtest */
+ for (i = 0; i < subtests->size; i++)
+ if (!strcmp(subtest, subtests->subs[i].name))
+ return;
+
+ subtests->size++;
+ subtests->subs = realloc(subtests->subs, sizeof(*subtests->subs) * subtests->size);
+ memset(&subtests->subs[subtests->size - 1], 0, sizeof(struct subtest));
+ subtests->subs[subtests->size - 1].name = subtest;
+}
+
+static void free_subtest(struct subtest *subtest)
+{
+ size_t i;
+
+ for (i = 0; i < subtest->dynamic_size; i++)
+ free(subtest->dynamic_names[i]);
+ free(subtest->dynamic_names);
+}
+
+static void free_subtests(struct subtest_list *subtests)
+{
+ size_t i;
+
+ for (i = 0; i < subtests->size; i++)
+ free_subtest(&subtests->subs[i]);
+ free(subtests->subs);
+}
+
/*
* A lot of string handling here operates on an mmapped buffer, and
* thus we can't assume null-terminated strings. Buffers will be
@@ -77,17 +145,6 @@ static const char *next_line(const char *line, const char *bufend)
return NULL;
}
-static size_t count_lines(const char *buf, const char *bufend)
-{
- size_t ret = 0;
- while (buf < bufend && (buf = memchr(buf, '\n', bufend - buf)) != NULL) {
- ret++;
- buf++;
- }
-
- return ret;
-}
-
static void append_line(char **buf, size_t *buflen, char *line)
{
size_t linelen = strlen(line);
@@ -148,6 +205,7 @@ static void parse_result_string(const char *resultstring, size_t len, const char
}
static void parse_subtest_result(const char *subtest,
+ const char *resulttextprefix,
const char **result,
double *time,
const char *line,
@@ -175,6 +233,10 @@ static void parse_subtest_result(const char *subtest,
*
* Example:
* Subtest subtestname: PASS (0.003s)
+ *
+ * For dynamic subtests the same structure applies, but the
+ * string "Subtest " is "Dynamic subtest "
+ * instead. (`DYNAMIC_SUBTEST_RESULT` from output_strings.h)
*/
if (!line)
@@ -183,13 +245,13 @@ static void parse_subtest_result(const char *subtest,
line_end = memchr(line, '\n', bufend - line);
linelen = line_end != NULL ? line_end - line : bufend - line;
- if (strlen(SUBTEST_RESULT) + subtestlen + strlen(": ") > linelen ||
- strncmp(line + strlen(SUBTEST_RESULT), subtest, subtestlen)) {
+ if (strlen(resulttextprefix) + subtestlen + strlen(": ") > linelen ||
+ strncmp(line + strlen(resulttextprefix), subtest, subtestlen)) {
/* This is not the correct result line */
return;
}
- resultstring = line + strlen(SUBTEST_RESULT) + subtestlen + strlen(": ");
+ resultstring = line + strlen(resulttextprefix) + subtestlen + strlen(": ");
parse_result_string(resultstring, linelen - (resultstring - line), result, time);
}
@@ -303,7 +365,7 @@ static void free_matches(struct matches *matches)
}
static bool fill_from_output(int fd, const char *binary, const char *key,
- struct subtests *subtests,
+ struct subtest_list *subtests,
struct json_object *tests)
{
char *buf, *bufend, *nullchr;
@@ -315,6 +377,8 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
const char *needles[] = {
STARTING_SUBTEST,
SUBTEST_RESULT,
+ STARTING_DYNAMIC_SUBTEST,
+ DYNAMIC_SUBTEST_RESULT,
NULL
};
struct matches matches = {};
@@ -374,11 +438,11 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
int begin_len;
int result_len;
- generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name));
+ generate_piglit_name(binary, subtests->subs[i].name, piglit_name, sizeof(piglit_name));
current_test = get_or_create_json_object(tests, piglit_name);
- begin_len = asprintf(&this_sub_begin, "%s%s\n", STARTING_SUBTEST, subtests->names[i]);
- result_len = asprintf(&this_sub_result, "%s%s: ", SUBTEST_RESULT, subtests->names[i]);
+ begin_len = asprintf(&this_sub_begin, "%s%s\n", STARTING_SUBTEST, subtests->subs[i].name);
+ result_len = asprintf(&this_sub_result, "%s%s: ", SUBTEST_RESULT, subtests->subs[i].name);
if (begin_len < 0 || result_len < 0) {
fprintf(stderr, "Failure generating strings\n");
@@ -437,8 +501,13 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
* Incomplete result. Include output up to the
* next starting subtest or result.
*/
- if (begin_idx < matches.size - 1)
- end = matches.items[begin_idx + 1].where;
+ for (k = begin_idx + 1; k < matches.size; k++) {
+ if (matches.items[k].what == STARTING_SUBTEST ||
+ matches.items[k].what == SUBTEST_RESULT)
+ break;
+ }
+ if (k < matches.size)
+ end = matches.items[k].where;
else
end = bufend;
} else {
@@ -459,13 +528,104 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
}
if (!json_object_object_get_ex(current_test, "result", NULL)) {
- parse_subtest_result(subtests->names[i],
+ parse_subtest_result(subtests->subs[i].name,
+ SUBTEST_RESULT,
&resulttext, &time,
result_idx < 0 ? NULL : matches.items[result_idx].where,
end);
set_result(current_test, resulttext);
set_runtime(current_test, time);
}
+
+ /*
+ * Look for dynamic subtests: If any, they are within
+ * the subtest output.
+ */
+ if (result_idx < 0) {
+ /* If the subtest itself is incomplete, stop at the next start of a subtest */
+ for (result_idx = begin_idx + 1;
+ result_idx < matches.size;
+ result_idx++) {
+ if (matches.items[result_idx].what == STARTING_SUBTEST)
+ break;
+ }
+ }
+
+ for (k = begin_idx + 1; k < result_idx; k++) {
+ if (matches.items[k].what == STARTING_DYNAMIC_SUBTEST) {
+ struct json_object *current_dynamic_test = NULL;
+ int dyn_result_idx = -1;
+ char dynamic_name[256];
+ char dynamic_piglit_name[256];
+ char *this_dyn_result;
+ int dyn_result_len;
+ const char *dynbeg, *dynend;
+ int n;
+
+ if (sscanf(matches.items[k].where + strlen(STARTING_DYNAMIC_SUBTEST), "%s", dynamic_name) != 1) {
+ /* Cannot parse name, just ignore this one */
+ continue;
+ }
+
+ dyn_result_len = asprintf(&this_dyn_result, "%s%s: ", DYNAMIC_SUBTEST_RESULT, dynamic_name);
+ if (dyn_result_len < 0)
+ continue;
+
+ for (n = k + 1; n < result_idx; n++) {
+ if (matches.items[n].what == DYNAMIC_SUBTEST_RESULT &&
+ !memcmp(matches.items[n].where,
+ this_dyn_result,
+ min(dyn_result_len, bufend - matches.items[n].where))) {
+ dyn_result_idx = n;
+ break;
+ }
+ }
+
+ free(this_dyn_result);
+
+ if (k == 0)
+ dynbeg = beg;
+ else
+ dynbeg = next_line(matches.items[k - 1].where, end);
+
+ if (dyn_result_idx < 0) {
+ if (k < matches.size - 1)
+ dynend = matches.items[k + 1].where;
+ else
+ dynend = end;
+ } else {
+ if (dyn_result_idx < matches.size - 1)
+ dynend = matches.items[dyn_result_idx + 1].where;
+ else
+ dynend = end;
+ }
+
+ generate_piglit_name_for_dynamic(piglit_name, dynamic_name, dynamic_piglit_name, sizeof(dynamic_piglit_name));
+
+ add_dynamic_subtest(&subtests->subs[i], strdup(dynamic_name));
+ current_dynamic_test = get_or_create_json_object(tests, dynamic_piglit_name);
+
+ json_object_object_add(current_dynamic_test, key,
+ json_object_new_string_len(dynbeg, dynend - dynbeg));
+ if (igt_version)
+ json_object_object_add(current_dynamic_test, "igt-version",
+ json_object_new_string_len(igt_version,
+ igt_version_len));
+
+ if (!json_object_object_get_ex(current_dynamic_test, "result", NULL)) {
+ const char *dynresulttext;
+ double dyntime;
+
+ parse_subtest_result(dynamic_name,
+ DYNAMIC_SUBTEST_RESULT,
+ &dynresulttext, &dyntime,
+ dyn_result_idx < 0 ? NULL : matches.items[dyn_result_idx].where,
+ dynend);
+ set_result(current_dynamic_test, dynresulttext);
+ set_runtime(current_dynamic_test, dyntime);
+ }
+ }
+ }
}
free_matches(&matches);
@@ -613,18 +773,28 @@ static void add_dmesg(struct json_object *obj,
static void add_empty_dmesgs_where_missing(struct json_object *tests,
char *binary,
- struct subtests *subtests)
+ struct subtest_list *subtests)
{
struct json_object *current_test;
char piglit_name[256];
- size_t i;
+ char dynamic_piglit_name[256];
+ size_t i, k;
for (i = 0; i < subtests->size; i++) {
- generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name));
+ generate_piglit_name(binary, subtests->subs[i].name, piglit_name, sizeof(piglit_name));
current_test = get_or_create_json_object(tests, piglit_name);
if (!json_object_object_get_ex(current_test, "dmesg", NULL)) {
add_dmesg(current_test, "", 0, NULL, 0);
}
+
+ for (k = 0; k < subtests->subs[i].dynamic_size; k++) {
+ generate_piglit_name_for_dynamic(piglit_name, subtests->subs[i].dynamic_names[k],
+ dynamic_piglit_name, sizeof(dynamic_piglit_name));
+ current_test = get_or_create_json_object(tests, dynamic_piglit_name);
+ if (!json_object_object_get_ex(current_test, "dmesg", NULL)) {
+ add_dmesg(current_test, "", 0, NULL, 0);
+ }
+ }
}
}
@@ -632,14 +802,20 @@ static void add_empty_dmesgs_where_missing(struct json_object *tests,
static bool fill_from_dmesg(int fd,
struct settings *settings,
char *binary,
- struct subtests *subtests,
+ struct subtest_list *subtests,
struct json_object *tests)
{
- char *line = NULL, *warnings = NULL, *dmesg = NULL;
- size_t linelen = 0, warningslen = 0, dmesglen = 0;
+ char *line = NULL;
+ char *warnings = NULL, *dynamic_warnings = NULL;
+ char *dmesg = NULL, *dynamic_dmesg = NULL;
+ size_t linelen = 0;
+ size_t warningslen = 0, dynamic_warnings_len = 0;
+ size_t dmesglen = 0, dynamic_dmesg_len = 0;
struct json_object *current_test = NULL;
+ struct json_object *current_dynamic_test = NULL;
FILE *f = fdopen(fd, "r");
char piglit_name[256];
+ char dynamic_piglit_name[256];
ssize_t read;
size_t i;
GRegex *re;
@@ -658,7 +834,7 @@ static bool fill_from_dmesg(int fd,
unsigned flags;
unsigned long long ts_usec;
char continuation;
- char *message, *subtest;
+ char *message, *subtest, *dynamic_subtest;
if (!parse_dmesg_line(line, &flags, &ts_usec, &continuation, &message))
continue;
@@ -674,6 +850,15 @@ static bool fill_from_dmesg(int fd,
free(warnings);
dmesg = warnings = NULL;
dmesglen = warningslen = 0;
+
+ if (current_dynamic_test != NULL)
+ add_dmesg(current_dynamic_test, dynamic_dmesg, dynamic_dmesg_len, dynamic_warnings, dynamic_warnings_len);
+
+ free(dynamic_dmesg);
+ free(dynamic_warnings);
+ dynamic_dmesg = dynamic_warnings = NULL;
+ dynamic_dmesg_len = dynamic_warnings_len = 0;
+ current_dynamic_test = NULL;
}
subtest += strlen(STARTING_SUBTEST_DMESG);
@@ -681,24 +866,50 @@ static bool fill_from_dmesg(int fd,
current_test = get_or_create_json_object(tests, piglit_name);
}
+ if (current_test != NULL &&
+ (dynamic_subtest = strstr(message, STARTING_DYNAMIC_SUBTEST_DMESG)) != NULL) {
+ if (current_dynamic_test != NULL) {
+ /* Done with the previous dynamic subtest, file up */
+ add_dmesg(current_dynamic_test, dynamic_dmesg, dynamic_dmesg_len, dynamic_warnings, dynamic_warnings_len);
+
+ free(dynamic_dmesg);
+ free(dynamic_warnings);
+ dynamic_dmesg = dynamic_warnings = NULL;
+ dynamic_dmesg_len = dynamic_warnings_len = 0;
+ }
+
+ dynamic_subtest += strlen(STARTING_DYNAMIC_SUBTEST_DMESG);
+ generate_piglit_name_for_dynamic(piglit_name, dynamic_subtest, dynamic_piglit_name, sizeof(dynamic_piglit_name));
+ current_dynamic_test = get_or_create_json_object(tests, dynamic_piglit_name);
+ }
+
if (settings->piglit_style_dmesg) {
if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' &&
g_regex_match(re, message, 0, NULL)) {
append_line(&warnings, &warningslen, formatted);
+ if (current_test != NULL)
+ append_line(&dynamic_warnings, &dynamic_warnings_len, formatted);
}
} else {
if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' &&
!g_regex_match(re, message, 0, NULL)) {
append_line(&warnings, &warningslen, formatted);
+ if (current_test != NULL)
+ append_line(&dynamic_warnings, &dynamic_warnings_len, formatted);
}
}
append_line(&dmesg, &dmesglen, formatted);
+ if (current_test != NULL)
+ append_line(&dynamic_dmesg, &dynamic_dmesg_len, formatted);
free(formatted);
}
free(line);
if (current_test != NULL) {
add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen);
+ if (current_dynamic_test != NULL) {
+ add_dmesg(current_dynamic_test, dynamic_dmesg, dynamic_dmesg_len, dynamic_warnings, dynamic_warnings_len);
+ }
} else {
/*
* Didn't get any subtest messages at all. If there
@@ -706,7 +917,7 @@ static bool fill_from_dmesg(int fd,
* them.
*/
for (i = 0; i < subtests->size; i++) {
- generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name));
+ generate_piglit_name(binary, subtests->subs[i].name, piglit_name, sizeof(piglit_name));
current_test = get_or_create_json_object(tests, piglit_name);
/*
* Don't bother with warnings, any subtests
@@ -726,7 +937,9 @@ static bool fill_from_dmesg(int fd,
add_empty_dmesgs_where_missing(tests, binary, subtests);
free(dmesg);
+ free(dynamic_dmesg);
free(warnings);
+ free(dynamic_warnings);
g_regex_unref(re);
fclose(f);
return true;
@@ -748,39 +961,9 @@ static const char *result_from_exitcode(int exitcode)
}
}
-static void add_subtest(struct subtests *subtests, char *subtest)
-{
- size_t len = strlen(subtest);
- size_t i;
-
- if (len == 0)
- return;
-
- if (subtest[len - 1] == '\n')
- subtest[len - 1] = '\0';
-
- /* Don't add if we already have this subtest */
- for (i = 0; i < subtests->size; i++)
- if (!strcmp(subtest, subtests->names[i]))
- return;
-
- subtests->size++;
- subtests->names = realloc(subtests->names, sizeof(*subtests->names) * subtests->size);
- subtests->names[subtests->size - 1] = subtest;
-}
-
-static void free_subtests(struct subtests *subtests)
-{
- size_t i;
-
- for (i = 0; i < subtests->size; i++)
- free(subtests->names[i]);
- free(subtests->names);
-}
-
static void fill_from_journal(int fd,
struct job_list_entry *entry,
- struct subtests *subtests,
+ struct subtest_list *subtests,
struct results *results)
{
FILE *f = fdopen(fd, "r");
@@ -820,7 +1003,7 @@ static void fill_from_journal(int fd,
if (subtests->size) {
/* Assign the timeout to the previously appeared subtest */
- char *last_subtest = subtests->names[subtests->size - 1];
+ char *last_subtest = subtests->subs[subtests->size - 1].name;
char piglit_name[256];
char *p = strchr(line, '(');
double time = 0.0;
@@ -873,6 +1056,30 @@ static void fill_from_journal(int fd,
fclose(f);
}
+static bool stderr_contains_warnings(const char *beg, const char *end)
+{
+ const char *needles[] = {
+ STARTING_SUBTEST,
+ SUBTEST_RESULT,
+ STARTING_DYNAMIC_SUBTEST,
+ DYNAMIC_SUBTEST_RESULT,
+ NULL
+ };
+ struct matches matches;
+ size_t i = 0;
+
+ matches = find_matches(beg, end, needles);
+
+ while (i < matches.size) {
+ if (matches.items[i].where != beg)
+ return true;
+ beg = next_line(beg, end);
+ i++;
+ }
+
+ return false;
+}
+
static void override_result_single(struct json_object *obj)
{
const char *errtext = "", *result = "";
@@ -887,7 +1094,7 @@ static void override_result_single(struct json_object *obj)
dmesgwarns = true;
if (!strcmp(result, "pass") &&
- count_lines(errtext, errtext + strlen(errtext)) > 2) {
+ stderr_contains_warnings(errtext, errtext + strlen(errtext))) {
set_result(obj, "warn");
result = "warn";
}
@@ -902,12 +1109,13 @@ static void override_result_single(struct json_object *obj)
}
static void override_results(char *binary,
- struct subtests *subtests,
+ struct subtest_list *subtests,
struct json_object *tests)
{
struct json_object *obj;
char piglit_name[256];
- size_t i;
+ char dynamic_piglit_name[256];
+ size_t i, k;
if (subtests->size == 0) {
generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name));
@@ -917,9 +1125,16 @@ static void override_results(char *binary,
}
for (i = 0; i < subtests->size; i++) {
- generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name));
+ generate_piglit_name(binary, subtests->subs[i].name, piglit_name, sizeof(piglit_name));
obj = get_or_create_json_object(tests, piglit_name);
override_result_single(obj);
+
+ for (k = 0; k < subtests->subs[i].dynamic_size; k++) {
+ generate_piglit_name_for_dynamic(piglit_name, subtests->subs[i].dynamic_names[k],
+ dynamic_piglit_name, sizeof(dynamic_piglit_name));
+ obj = get_or_create_json_object(tests, dynamic_piglit_name);
+ override_result_single(obj);
+ }
}
}
@@ -964,13 +1179,14 @@ static void add_result_to_totals(struct json_object *totals,
}
static void add_to_totals(const char *binary,
- struct subtests *subtests,
+ struct subtest_list *subtests,
struct results *results)
{
struct json_object *test, *resultobj, *emptystrtotal, *roottotal, *binarytotal;
char piglit_name[256];
+ char dynamic_piglit_name[256];
const char *result;
- size_t i;
+ size_t i, k;
generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name));
emptystrtotal = get_totals_object(results->totals, "");
@@ -991,7 +1207,7 @@ static void add_to_totals(const char *binary,
}
for (i = 0; i < subtests->size; i++) {
- generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name));
+ generate_piglit_name(binary, subtests->subs[i].name, piglit_name, sizeof(piglit_name));
test = get_or_create_json_object(results->tests, piglit_name);
if (!json_object_object_get_ex(test, "result", &resultobj)) {
fprintf(stderr, "Warning: No results set for %s\n", piglit_name);
@@ -1001,6 +1217,21 @@ static void add_to_totals(const char *binary,
add_result_to_totals(emptystrtotal, result);
add_result_to_totals(roottotal, result);
add_result_to_totals(binarytotal, result);
+
+ for (k = 0; k < subtests->subs[i].dynamic_size; k++) {
+ generate_piglit_name_for_dynamic(piglit_name, subtests->subs[i].dynamic_names[k],
+ dynamic_piglit_name, sizeof(dynamic_piglit_name));
+ test = get_or_create_json_object(results->tests, dynamic_piglit_name);
+ if (!json_object_object_get_ex(test, "result", &resultobj)) {
+ fprintf(stderr, "Warning: No results set for %s\n", dynamic_piglit_name);
+ return;
+ }
+ result = json_object_get_string(resultobj);
+ add_result_to_totals(emptystrtotal, result);
+ add_result_to_totals(roottotal, result);
+ add_result_to_totals(binarytotal, result);
+ }
+
}
}
@@ -1010,7 +1241,7 @@ static bool parse_test_directory(int dirfd,
struct results *results)
{
int fds[_F_LAST];
- struct subtests subtests = {};
+ struct subtest_list subtests = {};
bool status = true;
if (!open_output_files(dirfd, fds, false)) {
@@ -1046,7 +1277,7 @@ static void try_add_notrun_results(const struct job_list_entry *entry,
const struct settings *settings,
struct results *results)
{
- struct subtests subtests = {};
+ struct subtest_list subtests = {};
struct json_object *current_test;
size_t i;
@@ -1183,7 +1414,7 @@ struct json_object *generate_results_json(int dirfd)
if ((fd = openat(dirfd, "aborted.txt", O_RDONLY)) >= 0) {
char buf[4096];
char piglit_name[] = "igt@runner@aborted";
- struct subtests abortsub = {};
+ struct subtest_list abortsub = {};
struct json_object *aborttest = get_or_create_json_object(results.tests, piglit_name);
ssize_t s;
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 07/10] runner/json_tests: Test dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (5 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 06/10] runner: Parse dynamic subtest outputs and results Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 08/10] kms_plane_cursor: Use " Petri Latvala
` (4 subsequent siblings)
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
.../dynamic-subtests/0/dmesg.txt | 7 +
.../dynamic-subtests/0/err.txt | 36 ++++
.../dynamic-subtests/0/journal.txt | 2 +
.../dynamic-subtests/0/out.txt | 19 ++
.../dynamic-subtests/1/dmesg.txt | 5 +
.../dynamic-subtests/1/err.txt | 2 +
.../dynamic-subtests/1/journal.txt | 2 +
.../dynamic-subtests/1/out.txt | 5 +
.../dynamic-subtests/2/dmesg.txt | 11 ++
.../dynamic-subtests/2/err.txt | 8 +
.../dynamic-subtests/2/journal.txt | 4 +
.../dynamic-subtests/2/out.txt | 10 ++
.../dynamic-subtests/README.txt | 2 +
.../dynamic-subtests/endtime.txt | 1 +
.../dynamic-subtests/joblist.txt | 3 +
.../dynamic-subtests/metadata.txt | 12 ++
.../dynamic-subtests/reference.json | 168 ++++++++++++++++++
.../dynamic-subtests/starttime.txt | 1 +
.../dynamic-subtests/uname.txt | 1 +
runner/runner_json_tests.c | 3 +-
20 files changed, 301 insertions(+), 1 deletion(-)
create mode 100644 runner/json_tests_data/dynamic-subtests/0/dmesg.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/0/err.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/0/journal.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/0/out.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/1/dmesg.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/1/err.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/1/journal.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/1/out.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/2/dmesg.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/2/err.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/2/journal.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/2/out.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/README.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/endtime.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/joblist.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/metadata.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/reference.json
create mode 100644 runner/json_tests_data/dynamic-subtests/starttime.txt
create mode 100644 runner/json_tests_data/dynamic-subtests/uname.txt
diff --git a/runner/json_tests_data/dynamic-subtests/0/dmesg.txt b/runner/json_tests_data/dynamic-subtests/0/dmesg.txt
new file mode 100644
index 00000000..c87b4271
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/0/dmesg.txt
@@ -0,0 +1,7 @@
+6,1157,23426155175691,-;Console: switching to colour dummy device 80x25
+14,1158,23426155175708,-;[IGT] dynamic: executing
+14,1159,23426155184875,-;[IGT] dynamic: starting subtest debug-log-checking
+14,1160,23426155184895,-;[IGT] dynamic: starting dynamic subtest this-is-dynamic-1
+14,1161,23426155240164,-;[IGT] dynamic: starting dynamic subtest this-is-dynamic-2
+14,1162,23426155293846,-;[IGT] dynamic: exiting, ret=98
+6,1163,23426155294003,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dynamic-subtests/0/err.txt b/runner/json_tests_data/dynamic-subtests/0/err.txt
new file mode 100644
index 00000000..3c280788
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/0/err.txt
@@ -0,0 +1,36 @@
+Starting subtest: debug-log-checking
+Starting dynamic subtest: this-is-dynamic-1
+(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ../runner/testdata/dynamic.c:8:
+(dynamic:20904) CRITICAL: Failed assertion: false
+Dynamic subtest this-is-dynamic-1 failed.
+**** DEBUG ****
+(dynamic:20904) DEBUG: This print is from 1
+(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ../runner/testdata/dynamic.c:8:
+(dynamic:20904) CRITICAL: Failed assertion: false
+(dynamic:20904) igt_core-INFO: Stack trace:
+(dynamic:20904) igt_core-INFO: #0 ../lib/igt_core.c:1607 __igt_fail_assert()
+(dynamic:20904) igt_core-INFO: #1 ../runner/testdata/dynamic.c:11 __real_main3()
+(dynamic:20904) igt_core-INFO: #2 ../runner/testdata/dynamic.c:3 main()
+(dynamic:20904) igt_core-INFO: #3 ../csu/libc-start.c:342 __libc_start_main()
+(dynamic:20904) igt_core-INFO: #4 [_start+0x2a]
+**** END ****
+Dynamic subtest this-is-dynamic-1: FAIL (0.055s)
+Starting dynamic subtest: this-is-dynamic-2
+(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ../runner/testdata/dynamic.c:13:
+(dynamic:20904) CRITICAL: Failed assertion: false
+Dynamic subtest this-is-dynamic-2 failed.
+**** DEBUG ****
+(dynamic:20904) DEBUG: This print is from 2
+(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ../runner/testdata/dynamic.c:13:
+(dynamic:20904) CRITICAL: Failed assertion: false
+(dynamic:20904) igt_core-INFO: Stack trace:
+(dynamic:20904) igt_core-INFO: #0 ../lib/igt_core.c:1607 __igt_fail_assert()
+(dynamic:20904) igt_core-INFO: #1 ../runner/testdata/dynamic.c:5 __real_main3()
+(dynamic:20904) igt_core-INFO: #2 ../runner/testdata/dynamic.c:3 main()
+(dynamic:20904) igt_core-INFO: #3 ../csu/libc-start.c:342 __libc_start_main()
+(dynamic:20904) igt_core-INFO: #4 [_start+0x2a]
+**** END ****
+Dynamic subtest this-is-dynamic-2: FAIL (0.054s)
+Subtest debug-log-checking failed.
+No log.
+Subtest debug-log-checking: FAIL (0.109s)
diff --git a/runner/json_tests_data/dynamic-subtests/0/journal.txt b/runner/json_tests_data/dynamic-subtests/0/journal.txt
new file mode 100644
index 00000000..aa8cef0c
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/0/journal.txt
@@ -0,0 +1,2 @@
+debug-log-checking
+exit:98 (0.130s)
diff --git a/runner/json_tests_data/dynamic-subtests/0/out.txt b/runner/json_tests_data/dynamic-subtests/0/out.txt
new file mode 100644
index 00000000..286da3f3
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/0/out.txt
@@ -0,0 +1,19 @@
+IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: debug-log-checking
+Starting dynamic subtest: this-is-dynamic-1
+Stack trace:
+ #0 ../lib/igt_core.c:1607 __igt_fail_assert()
+ #1 ../runner/testdata/dynamic.c:11 __real_main3()
+ #2 ../runner/testdata/dynamic.c:3 main()
+ #3 ../csu/libc-start.c:342 __libc_start_main()
+ #4 [_start+0x2a]
+Dynamic subtest this-is-dynamic-1: FAIL (0.055s)
+Starting dynamic subtest: this-is-dynamic-2
+Stack trace:
+ #0 ../lib/igt_core.c:1607 __igt_fail_assert()
+ #1 ../runner/testdata/dynamic.c:5 __real_main3()
+ #2 ../runner/testdata/dynamic.c:3 main()
+ #3 ../csu/libc-start.c:342 __libc_start_main()
+ #4 [_start+0x2a]
+Dynamic subtest this-is-dynamic-2: FAIL (0.054s)
+Subtest debug-log-checking: FAIL (0.109s)
diff --git a/runner/json_tests_data/dynamic-subtests/1/dmesg.txt b/runner/json_tests_data/dynamic-subtests/1/dmesg.txt
new file mode 100644
index 00000000..d6e16a46
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,1164,23426155304955,-;Console: switching to colour dummy device 80x25
+14,1165,23426155304968,-;[IGT] dynamic: executing
+14,1166,23426155308644,-;[IGT] dynamic: starting subtest empty-container
+14,1167,23426155308671,-;[IGT] dynamic: exiting, ret=77
+6,1168,23426155308822,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dynamic-subtests/1/err.txt b/runner/json_tests_data/dynamic-subtests/1/err.txt
new file mode 100644
index 00000000..6247e714
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: empty-container
+Subtest empty-container: SKIP (0.000s)
diff --git a/runner/json_tests_data/dynamic-subtests/1/journal.txt b/runner/json_tests_data/dynamic-subtests/1/journal.txt
new file mode 100644
index 00000000..3e4ce5c4
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/1/journal.txt
@@ -0,0 +1,2 @@
+empty-container
+exit:77 (0.014s)
diff --git a/runner/json_tests_data/dynamic-subtests/1/out.txt b/runner/json_tests_data/dynamic-subtests/1/out.txt
new file mode 100644
index 00000000..704734dc
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/1/out.txt
@@ -0,0 +1,5 @@
+IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: empty-container
+This should skip
+No dynamic tests executed.
+Subtest empty-container: SKIP (0.000s)
diff --git a/runner/json_tests_data/dynamic-subtests/2/dmesg.txt b/runner/json_tests_data/dynamic-subtests/2/dmesg.txt
new file mode 100644
index 00000000..046ef579
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/2/dmesg.txt
@@ -0,0 +1,11 @@
+6,1157,23426155175691,-;Console: switching to colour dummy device 80x25
+14,1158,23426155175708,-;[IGT] dynamic: executing
+14,1159,23426155184875,-;[IGT] dynamic: starting subtest normal
+6,1160,23426155184895,-;Dmesg output for normal
+6,1160,23426155184895,-;[IGT] dynamic: starting dynamic subtest normal-dynamic-subtest
+14,1159,23426155184875,-;[IGT] dynamic: starting subtest incomplete
+14,1160,23426155184895,-;[IGT] dynamic: starting dynamic subtest this-is-incomplete
+6,1160,23426155184895,-;Dmesg output for incomplete
+14,1161,23426155240164,-;[IGT] dynamic: starting subtest resume
+14,1162,23426155293846,-;[IGT] dynamic: exiting, ret=0
+6,1163,23426155294003,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dynamic-subtests/2/err.txt b/runner/json_tests_data/dynamic-subtests/2/err.txt
new file mode 100644
index 00000000..1eae6061
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/2/err.txt
@@ -0,0 +1,8 @@
+Starting subtest: normal
+Starting dynamic subtest: normal-dynamic-subtest
+Dynamic subtest normal-dynamic-subtest: SUCCESS (0.055s)
+Subtest normal: SUCCESS (0.100s)
+Starting subtest: incomplete
+Starting dynamic subtest: this-is-incomplete
+Starting subtest: resume
+Subtest resume: SUCCESS (0.109s)
diff --git a/runner/json_tests_data/dynamic-subtests/2/journal.txt b/runner/json_tests_data/dynamic-subtests/2/journal.txt
new file mode 100644
index 00000000..9469e227
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/2/journal.txt
@@ -0,0 +1,4 @@
+normal
+incomplete
+resume
+exit:0 (0.130s)
diff --git a/runner/json_tests_data/dynamic-subtests/2/out.txt b/runner/json_tests_data/dynamic-subtests/2/out.txt
new file mode 100644
index 00000000..3063844b
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/2/out.txt
@@ -0,0 +1,10 @@
+IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: normal
+Starting dynamic subtest: normal-dynamic-subtest
+Dynamic subtest normal-dynamic-subtest: SUCCESS (0.055s)
+Subtest normal: SUCCESS (0.100s)
+Starting subtest: incomplete
+Starting dynamic subtest: this-is-incomplete
+This is some output
+Starting subtest: resume
+Subtest resume: SUCCESS (0.109s)
diff --git a/runner/json_tests_data/dynamic-subtests/README.txt b/runner/json_tests_data/dynamic-subtests/README.txt
new file mode 100644
index 00000000..3a63ab17
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/README.txt
@@ -0,0 +1,2 @@
+A test with dynamic subtests should generate separate subresults for
+the dynamic tests.
diff --git a/runner/json_tests_data/dynamic-subtests/endtime.txt b/runner/json_tests_data/dynamic-subtests/endtime.txt
new file mode 100644
index 00000000..5c7608b5
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/endtime.txt
@@ -0,0 +1 @@
+1560163492.410489
diff --git a/runner/json_tests_data/dynamic-subtests/joblist.txt b/runner/json_tests_data/dynamic-subtests/joblist.txt
new file mode 100644
index 00000000..4fdcc5f1
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/joblist.txt
@@ -0,0 +1,3 @@
+dynamic debug-log-checking
+dynamic empty-container
+dynamic normal,incomplete,resume
diff --git a/runner/json_tests_data/dynamic-subtests/metadata.txt b/runner/json_tests_data/dynamic-subtests/metadata.txt
new file mode 100644
index 00000000..8c77de01
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/metadata.txt
@@ -0,0 +1,12 @@
+abort_mask : 0
+name : dynamic-subtests
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 1
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/dynamic-subtests/reference.json b/runner/json_tests_data/dynamic-subtests/reference.json
new file mode 100644
index 00000000..274173f7
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/reference.json
@@ -0,0 +1,168 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":10,
+ "name":"dynamic-subtests",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1560163492.266377,
+ "end":1560163492.4104891
+ },
+ "tests":{
+ "igt@dynamic@debug-log-checking":{
+ "out":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: debug-log-checking\nStarting dynamic subtest: this-is-dynamic-1\nStack trace:\n #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n #1 ..\/runner\/testdata\/dynamic.c:11 __real_main3()\n #2 ..\/runner\/testdata\/dynamic.c:3 main()\n #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n #4 [_start+0x2a]\nDynamic subtest this-is-dynamic-1: FAIL (0.055s)\nStarting dynamic subtest: this-is-dynamic-2\nStack trace:\n #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n #1 ..\/runner\/testdata\/dynamic.c:5 __real_main3()\n #2 ..\/runner\/testdata\/dynamic.c:3 main()\n #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n #4 [_start+0x2a]\nDynamic subtest this-is-dynamic-2: FAIL (0.054s)\nSubtest debug-log-checking: FAIL (0.109s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"fail",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.109
+ },
+ "err":"Starting subtest: debug-log-checking\nStarting dynamic subtest: this-is-dynamic-1\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:8:\n(dynamic:20904) CRITICAL: Failed assertion: false\nDynamic subtest this-is-dynamic-1 failed.\n**** DEBUG ****\n(dynamic:20904) DEBUG: This print is from 1\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:8:\n(dynamic:20904) CRITICAL: Failed assertion: false\n(dynamic:20904) igt_core-INFO: Stack trace:\n(dynamic:20904) igt_core-INFO: #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n(dynamic:20904) igt_core-INFO: #1 ..\/runner\/testdata\/dynamic.c:11 __real_main3()\n(dynamic:20904) igt_core-INFO: #2 ..\/runner\/testdata\/dynamic.c:3 main()\n(dynamic:20904) igt_core-INFO: #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n(dynamic:20904) igt_core-INFO: #4 [_start+0x2a]\n**** END ****\nDynamic subtest this-is-dynamic-1: FAIL (0.055s)\nStarting dynamic subtest: this-is-dynamic-2\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:13:\n(dynamic:20904) CRITICAL: Failed assertion: false\nDynamic subtest this-is-dynamic-2 failed.\n**** DEBUG ****\n(dynamic:20904) DEBUG: This print is from 2\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:13:\n(dynamic:20904) CRITICAL: Failed assertion: false\n(dynamic:20904) igt_core-INFO: Stack trace:\n(dynamic:20904) igt_core-INFO: #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n(dynamic:20904) igt_core-INFO: #1 ..\/runner\/testdata\/dynamic.c:5 __real_main3()\n(dynamic:20904) igt_core-INFO: #2 ..\/runner\/testdata\/dynamic.c:3 main()\n(dynamic:20904) igt_core-INFO: #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n(dynamic:20904) igt_core-INFO: #4 [_start+0x2a]\n**** END ****\nDynamic subtest this-is-dynamic-2: FAIL (0.054s)\nSubtest debug-log-checking failed.\nNo log.\nSubtest debug-log-checking: FAIL (0.109s)\n",
+ "dmesg":"<6> [23426155.175691] Console: switching to colour dummy device 80x25\n<6> [23426155.175708] [IGT] dynamic: executing\n<6> [23426155.184875] [IGT] dynamic: starting subtest debug-log-checking\n<6> [23426155.184895] [IGT] dynamic: starting dynamic subtest this-is-dynamic-1\n<6> [23426155.240164] [IGT] dynamic: starting dynamic subtest this-is-dynamic-2\n<6> [23426155.293846] [IGT] dynamic: exiting, ret=98\n<6> [23426155.294003] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@dynamic@debug-log-checking@this-is-dynamic-1":{
+ "out":"Starting dynamic subtest: this-is-dynamic-1\nStack trace:\n #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n #1 ..\/runner\/testdata\/dynamic.c:11 __real_main3()\n #2 ..\/runner\/testdata\/dynamic.c:3 main()\n #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n #4 [_start+0x2a]\nDynamic subtest this-is-dynamic-1: FAIL (0.055s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"fail",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.055
+ },
+ "err":"Starting dynamic subtest: this-is-dynamic-1\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:8:\n(dynamic:20904) CRITICAL: Failed assertion: false\nDynamic subtest this-is-dynamic-1 failed.\n**** DEBUG ****\n(dynamic:20904) DEBUG: This print is from 1\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:8:\n(dynamic:20904) CRITICAL: Failed assertion: false\n(dynamic:20904) igt_core-INFO: Stack trace:\n(dynamic:20904) igt_core-INFO: #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n(dynamic:20904) igt_core-INFO: #1 ..\/runner\/testdata\/dynamic.c:11 __real_main3()\n(dynamic:20904) igt_core-INFO: #2 ..\/runner\/testdata\/dynamic.c:3 main()\n(dynamic:20904) igt_core-INFO: #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n(dynamic:20904) igt_core-INFO: #4 [_start+0x2a]\n**** END ****\nDynamic subtest this-is-dynamic-1: FAIL (0.055s)\n",
+ "dmesg":"<6> [23426155.184875] [IGT] dynamic: starting subtest debug-log-checking\n<6> [23426155.184895] [IGT] dynamic: starting dynamic subtest this-is-dynamic-1\n"
+ },
+ "igt@dynamic@debug-log-checking@this-is-dynamic-2":{
+ "out":"Starting dynamic subtest: this-is-dynamic-2\nStack trace:\n #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n #1 ..\/runner\/testdata\/dynamic.c:5 __real_main3()\n #2 ..\/runner\/testdata\/dynamic.c:3 main()\n #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n #4 [_start+0x2a]\nDynamic subtest this-is-dynamic-2: FAIL (0.054s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"fail",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.054
+ },
+ "err":"Starting dynamic subtest: this-is-dynamic-2\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:13:\n(dynamic:20904) CRITICAL: Failed assertion: false\nDynamic subtest this-is-dynamic-2 failed.\n**** DEBUG ****\n(dynamic:20904) DEBUG: This print is from 2\n(dynamic:20904) CRITICAL: Test assertion failure function __real_main3, file ..\/runner\/testdata\/dynamic.c:13:\n(dynamic:20904) CRITICAL: Failed assertion: false\n(dynamic:20904) igt_core-INFO: Stack trace:\n(dynamic:20904) igt_core-INFO: #0 ..\/lib\/igt_core.c:1607 __igt_fail_assert()\n(dynamic:20904) igt_core-INFO: #1 ..\/runner\/testdata\/dynamic.c:5 __real_main3()\n(dynamic:20904) igt_core-INFO: #2 ..\/runner\/testdata\/dynamic.c:3 main()\n(dynamic:20904) igt_core-INFO: #3 ..\/csu\/libc-start.c:342 __libc_start_main()\n(dynamic:20904) igt_core-INFO: #4 [_start+0x2a]\n**** END ****\nDynamic subtest this-is-dynamic-2: FAIL (0.054s)\n",
+ "dmesg":"<6> [23426155.240164] [IGT] dynamic: starting dynamic subtest this-is-dynamic-2\n<6> [23426155.293846] [IGT] dynamic: exiting, ret=98\n<6> [23426155.294003] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@dynamic@empty-container":{
+ "out":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: empty-container\nThis should skip\nNo dynamic tests executed.\nSubtest empty-container: SKIP (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: empty-container\nSubtest empty-container: SKIP (0.000s)\n",
+ "dmesg":"<6> [23426155.304955] Console: switching to colour dummy device 80x25\n<6> [23426155.304968] [IGT] dynamic: executing\n<6> [23426155.308644] [IGT] dynamic: starting subtest empty-container\n<6> [23426155.308671] [IGT] dynamic: exiting, ret=77\n<6> [23426155.308822] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@dynamic@normal":{
+ "out":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nStarting subtest: normal\nStarting dynamic subtest: normal-dynamic-subtest\nDynamic subtest normal-dynamic-subtest: SUCCESS (0.055s)\nSubtest normal: SUCCESS (0.100s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.100
+ },
+ "err":"Starting subtest: normal\nStarting dynamic subtest: normal-dynamic-subtest\nDynamic subtest normal-dynamic-subtest: SUCCESS (0.055s)\nSubtest normal: SUCCESS (0.100s)\n",
+ "dmesg":"<6> [23426155.175691] Console: switching to colour dummy device 80x25\n<6> [23426155.175708] [IGT] dynamic: executing\n<6> [23426155.184875] [IGT] dynamic: starting subtest normal\n<6> [23426155.184895] Dmesg output for normal\n<6> [23426155.184895] [IGT] dynamic: starting dynamic subtest normal-dynamic-subtest\n"
+ },
+ "igt@dynamic@normal@normal-dynamic-subtest":{
+ "out":"Starting dynamic subtest: normal-dynamic-subtest\nDynamic subtest normal-dynamic-subtest: SUCCESS (0.055s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.055
+ },
+ "err":"Starting dynamic subtest: normal-dynamic-subtest\nDynamic subtest normal-dynamic-subtest: SUCCESS (0.055s)\n",
+ "dmesg":"<6> [23426155.184875] [IGT] dynamic: starting subtest normal\n<6> [23426155.184895] Dmesg output for normal\n<6> [23426155.184895] [IGT] dynamic: starting dynamic subtest normal-dynamic-subtest\n"
+ },
+ "igt@dynamic@incomplete":{
+ "out":"Starting subtest: incomplete\nStarting dynamic subtest: this-is-incomplete\nThis is some output\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"incomplete",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: incomplete\nStarting dynamic subtest: this-is-incomplete\n",
+ "dmesg":"<6> [23426155.184875] [IGT] dynamic: starting subtest incomplete\n<6> [23426155.184895] [IGT] dynamic: starting dynamic subtest this-is-incomplete\n<6> [23426155.184895] Dmesg output for incomplete\n"
+ },
+ "igt@dynamic@incomplete@this-is-incomplete":{
+ "out":"Starting dynamic subtest: this-is-incomplete\nThis is some output\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"incomplete",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting dynamic subtest: this-is-incomplete\n",
+ "dmesg":"<6> [23426155.184875] [IGT] dynamic: starting subtest incomplete\n<6> [23426155.184895] [IGT] dynamic: starting dynamic subtest this-is-incomplete\n<6> [23426155.184895] Dmesg output for incomplete\n"
+ },
+ "igt@dynamic@resume":{
+ "out":"This is some output\nStarting subtest: resume\nSubtest resume: SUCCESS (0.109s)\n",
+ "igt-version":"IGT-Version: 1.23-g9e957acd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.109
+ },
+ "err":"Starting subtest: resume\nSubtest resume: SUCCESS (0.109s)\n",
+ "dmesg":"<6> [23426155.240164] [IGT] dynamic: starting subtest resume\n<6> [23426155.293846] [IGT] dynamic: exiting, ret=0\n<6> [23426155.294003] Console: switching to colour frame buffer device 240x75\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":3,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":1,
+ "incomplete":2,
+ "timeout":0,
+ "notrun":0,
+ "fail":3,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":3,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":1,
+ "incomplete":2,
+ "timeout":0,
+ "notrun":0,
+ "fail":3,
+ "warn":0
+ },
+ "igt@dynamic":{
+ "crash":0,
+ "pass":3,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":1,
+ "incomplete":2,
+ "timeout":0,
+ "notrun":0,
+ "fail":3,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt@dynamic":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.27400000000000002
+ }
+ }
+ }
+}
diff --git a/runner/json_tests_data/dynamic-subtests/starttime.txt b/runner/json_tests_data/dynamic-subtests/starttime.txt
new file mode 100644
index 00000000..b30b32ee
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/starttime.txt
@@ -0,0 +1 @@
+1560163492.266377
diff --git a/runner/json_tests_data/dynamic-subtests/uname.txt b/runner/json_tests_data/dynamic-subtests/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/dynamic-subtests/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
index 7c9540d1..3443bd2b 100644
--- a/runner/runner_json_tests.c
+++ b/runner/runner_json_tests.c
@@ -161,7 +161,8 @@ static const char *dirnames[] = {
"notrun-results-multiple-mode",
"dmesg-warn-level",
"dmesg-warn-level-piglit-style",
- "dmesg-warn-level-one-piglit-style"
+ "dmesg-warn-level-one-piglit-style",
+ "dynamic-subtests",
};
igt_main
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 08/10] kms_plane_cursor: Use dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (6 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 07/10] runner/json_tests: Test dynamic subtests Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 09/10] perf_pmu: " Petri Latvala
` (3 subsequent siblings)
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Instead of for_each_pipe_static, use for_each_pipe with a dynamic
subtest per pipe.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
tests/kms_plane_cursor.c | 34 ++++++++++++++++++----------------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/tests/kms_plane_cursor.c b/tests/kms_plane_cursor.c
index cbad0041..ed19a9b0 100644
--- a/tests/kms_plane_cursor.c
+++ b/tests/kms_plane_cursor.c
@@ -318,22 +318,24 @@ igt_main
igt_display_require_output(&data.display);
}
- for_each_pipe_static(pipe)
- for (i = 0; i < ARRAY_SIZE(cursor_sizes); ++i) {
- int size = cursor_sizes[i];
-
- igt_subtest_f("pipe-%s-overlay-size-%d",
- kmstest_pipe_name(pipe), size)
- test_cursor_overlay(&data, size, pipe);
-
- igt_subtest_f("pipe-%s-primary-size-%d",
- kmstest_pipe_name(pipe), size)
- test_cursor_primary(&data, size, pipe);
-
- igt_subtest_f("pipe-%s-viewport-size-%d",
- kmstest_pipe_name(pipe), size)
- test_cursor_viewport(&data, size, pipe);
- }
+ for (i = 0; i < ARRAY_SIZE(cursor_sizes); ++i) {
+ int size = cursor_sizes[i];
+
+ igt_dynamic_subtest_container_f("overlay-size-%d", size)
+ for_each_pipe(&data.display, pipe)
+ igt_dynamic_subtest_f("pipe-%s", kmstest_pipe_name(pipe))
+ test_cursor_overlay(&data, size, pipe);
+
+ igt_dynamic_subtest_container_f("primary-size-%d", size)
+ for_each_pipe(&data.display, pipe)
+ igt_dynamic_subtest_f("pipe-%s", kmstest_pipe_name(pipe))
+ test_cursor_primary(&data, size, pipe);
+
+ igt_dynamic_subtest_container_f("viewport-size-%d", size)
+ for_each_pipe(&data.display, pipe)
+ igt_dynamic_subtest_f("pipe-%s", kmstest_pipe_name(pipe))
+ test_cursor_viewport(&data, size, pipe);
+ }
igt_fixture {
igt_display_fini(&data.display);
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 09/10] perf_pmu: Use dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (7 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 08/10] kms_plane_cursor: Use " Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 10/10] i915/gem_exec_basic: " Petri Latvala
` (2 subsequent siblings)
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Instead of generating a subtest for each engine in a static list,
convert to dynamic subtests, with one dynamic subtest per actually
present physical engine.
In addition, convert comments to igt_describe calls.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
tests/perf_pmu.c | 239 ++++++++++++++++++++---------------------------
1 file changed, 103 insertions(+), 136 deletions(-)
diff --git a/tests/perf_pmu.c b/tests/perf_pmu.c
index d392a67d..9c38e675 100644
--- a/tests/perf_pmu.c
+++ b/tests/perf_pmu.c
@@ -267,7 +267,7 @@ static void end_spin(int fd, igt_spin_t *spin, unsigned int flags)
}
static void
-single(int gem_fd, const struct intel_execution_engine2 *e, unsigned int flags)
+single(int gem_fd, const struct intel_execution_engine2 *e, unsigned int num_engines, unsigned int flags)
{
unsigned long slept;
igt_spin_t *spin;
@@ -603,7 +603,7 @@ all_busy_check_all(int gem_fd, const unsigned int num_engines,
}
static void
-no_sema(int gem_fd, const struct intel_execution_engine2 *e, unsigned int flags)
+no_sema(int gem_fd, const struct intel_execution_engine2 *e, unsigned int num_engines, unsigned int flags)
{
igt_spin_t *spin;
uint64_t val[2][2];
@@ -643,7 +643,7 @@ no_sema(int gem_fd, const struct intel_execution_engine2 *e, unsigned int flags)
static void
sema_wait(int gem_fd, const struct intel_execution_engine2 *e,
- unsigned int flags)
+ unsigned int num_engines, unsigned int flags)
{
struct drm_i915_gem_relocation_entry reloc[2] = {};
struct drm_i915_gem_exec_object2 obj[2] = {};
@@ -1679,6 +1679,26 @@ igt_main
int fd = -1;
struct intel_execution_engine2 *e;
unsigned int i;
+ const unsigned int pct[] = { 2, 50, 98 };
+ struct {
+ void (*func)(int, const struct intel_execution_engine2 *, unsigned int, unsigned int);
+ const char *name;
+ unsigned int flags;
+ const char *desc;
+ } busyidle_tests[] = {
+ { single, "idle", 0, "Test that engines show no load when idle." },
+ { single, "busy", TEST_BUSY, "Test that a single engine reports load correctly." },
+ { single, "busy-idle", TEST_BUSY | TEST_TRAILING_IDLE, "Test that a single engine reports load correctly." },
+ { busy_check_all, "busy-check-all", TEST_BUSY, "Test that when one engine is loaded, others report no load." },
+ { busy_check_all, "busy-idle-check-all", TEST_BUSY | TEST_TRAILING_IDLE, "Test that one one engine is loaded, others report no load." },
+ { most_busy_check_all, "most-busy-check-all", TEST_BUSY, "Test that when all except one engine are loaded all loads are correctly reported." },
+ { most_busy_check_all, "most-busy-idle-check-all", TEST_BUSY | TEST_TRAILING_IDLE, "Test that when all except one engine are loaded all loads are correctly reported." },
+ { no_sema, "idle-no-semaphores", 0, "Test that semaphore counters report no activity on idle engines." },
+ { no_sema, "busy-no-semaphores", TEST_BUSY, "Test that semaphore counters report no activity on busy engines." },
+ { no_sema, "busy-idle-no-semaphores", TEST_BUSY | TEST_TRAILING_IDLE, "Test that semaphore counters report no activity on idle or busy engines." },
+ { sema_wait, "semaphore-wait", TEST_BUSY, "Test that semaphore waits are correctly reported." },
+ { sema_wait, "semaphore-wait-idle", TEST_BUSY | TEST_TRAILING_IDLE, "Test that semaphore waits are correctly reported." },
+ };
igt_fixture {
fd = drm_open_driver_master(DRIVER_INTEL);
@@ -1690,141 +1710,89 @@ igt_main
num_engines++;
}
- /**
- * Test invalid access via perf API is rejected.
- */
+ igt_describe("Test that invalid access via perf API is rejected.");
igt_subtest("invalid-init")
invalid_init();
- __for_each_physical_engine(fd, e) {
- const unsigned int pct[] = { 2, 50, 98 };
-
- /**
- * Test that a single engine metric can be initialized or it
- * is correctly rejected.
- */
- igt_subtest_f("init-busy-%s", e->name)
- init(fd, e, I915_SAMPLE_BUSY);
-
- igt_subtest_f("init-wait-%s", e->name)
- init(fd, e, I915_SAMPLE_WAIT);
-
- igt_subtest_f("init-sema-%s", e->name)
- init(fd, e, I915_SAMPLE_SEMA);
-
- /**
- * Test that engines show no load when idle.
- */
- igt_subtest_f("idle-%s", e->name)
- single(fd, e, 0);
-
- /**
- * Test that a single engine reports load correctly.
- */
- igt_subtest_f("busy-%s", e->name)
- single(fd, e, TEST_BUSY);
- igt_subtest_f("busy-idle-%s", e->name)
- single(fd, e, TEST_BUSY | TEST_TRAILING_IDLE);
-
- /**
- * Test that when one engine is loaded other report no
- * load.
- */
- igt_subtest_f("busy-check-all-%s", e->name)
- busy_check_all(fd, e, num_engines, TEST_BUSY);
- igt_subtest_f("busy-idle-check-all-%s", e->name)
- busy_check_all(fd, e, num_engines,
- TEST_BUSY | TEST_TRAILING_IDLE);
-
- /**
- * Test that when all except one engine are loaded all
- * loads are correctly reported.
- */
- igt_subtest_f("most-busy-check-all-%s", e->name)
- most_busy_check_all(fd, e, num_engines,
- TEST_BUSY);
- igt_subtest_f("most-busy-idle-check-all-%s", e->name)
- most_busy_check_all(fd, e, num_engines,
- TEST_BUSY |
- TEST_TRAILING_IDLE);
-
- /**
- * Test that semphore counters report no activity on
- * idle or busy engines.
- */
- igt_subtest_f("idle-no-semaphores-%s", e->name)
- no_sema(fd, e, 0);
-
- igt_subtest_f("busy-no-semaphores-%s", e->name)
- no_sema(fd, e, TEST_BUSY);
-
- igt_subtest_f("busy-idle-no-semaphores-%s", e->name)
- no_sema(fd, e, TEST_BUSY | TEST_TRAILING_IDLE);
-
- /**
- * Test that semaphore waits are correctly reported.
- */
- igt_subtest_f("semaphore-wait-%s", e->name)
- sema_wait(fd, e, TEST_BUSY);
+ igt_describe("Test that a single engine metric can be initialized "
+ "or it is correctly rejected.");
+ igt_dynamic_subtest_container("init-busy")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ init(fd, e, I915_SAMPLE_BUSY);
- igt_subtest_f("semaphore-wait-idle-%s", e->name)
- sema_wait(fd, e,
- TEST_BUSY | TEST_TRAILING_IDLE);
+ igt_describe("Test that a single engine metric can be initialized "
+ "or it is correctly rejected.");
+ igt_dynamic_subtest_container("init-wait")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ init(fd, e, I915_SAMPLE_WAIT);
- /**
- * Check that two perf clients do not influence each
- * others observations.
- */
- igt_subtest_f("multi-client-%s", e->name)
- multi_client(fd, e);
+ igt_describe("Test that a single engine metric can be initialized "
+ "or it is correctly rejected.");
+ igt_dynamic_subtest_container("init-sema")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ init(fd, e, I915_SAMPLE_SEMA);
+
+ for (i = 0; i < ARRAY_SIZE(busyidle_tests); i++) {
+ igt_describe(busyidle_tests[i].desc);
+ igt_dynamic_subtest_container(busyidle_tests[i].name)
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ busyidle_tests[i].func(fd, e, num_engines, busyidle_tests[i].flags);
+ }
- /**
- * Check that reported usage is correct when PMU is
- * enabled after the batch is running.
- */
- igt_subtest_f("busy-start-%s", e->name)
- busy_start(fd, e);
+ igt_describe("Check that two perf clients do not influence each other's observations.");
+ igt_dynamic_subtest_container("multi-client")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ multi_client(fd, e);
- /**
- * Check that reported usage is correct when PMU is
- * enabled after two batches are running.
- */
- igt_subtest_f("busy-double-start-%s", e->name) {
- gem_require_contexts(fd);
- busy_double_start(fd, e);
- }
+ igt_describe("Check that reported usage is correct when PMU is enabled after the batch is running.");
+ igt_dynamic_subtest_container("busy-start")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ busy_start(fd, e);
- /**
- * Check that the PMU can be safely enabled in face of
- * interrupt-heavy engine load.
- */
- igt_subtest_f("enable-race-%s", e->name)
- test_enable_race(fd, e);
+ igt_describe("Check that reported usage is correct when PMU is enabled after two batches are running.");
+ igt_dynamic_subtest_container("busy-double-start") {
+ gem_require_contexts(fd);
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ busy_double_start(fd, e);
+ }
- /**
- * Check engine busyness accuracy is as expected.
- */
- for (i = 0; i < ARRAY_SIZE(pct); i++) {
- igt_subtest_f("busy-accuracy-%u-%s",
- pct[i], e->name)
- accuracy(fd, e, pct[i], 10);
- }
+ igt_describe("Check that the PMU can be safely enabled in face of interrupt-heavy engine load.");
+ igt_dynamic_subtest_container("enable-race")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ test_enable_race(fd, e);
+
+ for (i = 0; i < ARRAY_SIZE(pct); i++) {
+ igt_describe("Check engine busyness accuracy is as expected.");
+ igt_dynamic_subtest_container_f("busy-accuracy-%u", pct[i])
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ accuracy(fd, e, pct[i], 10);
+ }
- igt_subtest_f("busy-hang-%s", e->name) {
- igt_hang_t hang = igt_allow_hang(fd, 0, 0);
+ igt_dynamic_subtest_container("busy-hang")
+ __for_each_physical_engine(fd, e)
+ igt_dynamic_subtest_f("%s", e->name) {
+ igt_hang_t hang = igt_allow_hang(fd, 0, 0);
- single(fd, e, TEST_BUSY | FLAG_HANG);
+ single(fd, e, 0, TEST_BUSY | FLAG_HANG);
- igt_disallow_hang(fd, hang);
- }
+ igt_disallow_hang(fd, hang);
+ }
- /**
- * Test that event waits are correctly reported.
- */
- if (e->class == I915_ENGINE_CLASS_RENDER)
- igt_subtest_f("event-wait-%s", e->name)
- event_wait(fd, e);
- }
+ igt_describe("Test that event waits are correctly reported.");
+ igt_dynamic_subtest_container("event-wait")
+ __for_each_physical_engine(fd, e)
+ if (e->class == I915_ENGINE_CLASS_RENDER)
+ igt_dynamic_subtest_f("%s", e->name)
+ event_wait(fd, e);
/**
* Test that when all engines are loaded all loads are
@@ -1881,9 +1849,7 @@ igt_main
igt_subtest("rc6-runtime-pm-long")
test_rc6(fd, TEST_RUNTIME_PM | FLAG_LONG);
- /**
- * Check render nodes are counted.
- */
+ igt_describe("Check that render nodes are counted.");
igt_subtest_group {
int render_fd = -1;
@@ -1894,14 +1860,15 @@ igt_main
gem_quiescent_gpu(fd);
}
- __for_each_physical_engine(render_fd, e) {
- igt_subtest_f("render-node-busy-%s", e->name)
- single(render_fd, e, TEST_BUSY);
- igt_subtest_f("render-node-busy-idle-%s",
- e->name)
- single(render_fd, e,
- TEST_BUSY | TEST_TRAILING_IDLE);
- }
+ igt_dynamic_subtest_container("render-node-busy")
+ __for_each_physical_engine(render_fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ single(render_fd, e, 0, TEST_BUSY);
+
+ igt_dynamic_subtest_container("render-node-busy-idle")
+ __for_each_physical_engine(render_fd, e)
+ igt_dynamic_subtest_f("%s", e->name)
+ single(render_fd, e, 0, TEST_BUSY | TEST_TRAILING_IDLE);
igt_fixture {
close(render_fd);
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] [PATCH i-g-t 10/10] i915/gem_exec_basic: Use dynamic subtests
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (8 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 09/10] perf_pmu: " Petri Latvala
@ 2019-08-16 9:34 ` Petri Latvala
2019-08-16 10:17 ` [igt-dev] ✓ Fi.CI.BAT: success for Dynamic subtests (rev2) Patchwork
2019-08-16 22:13 ` [igt-dev] ✓ Fi.CI.IGT: " Patchwork
11 siblings, 0 replies; 20+ messages in thread
From: Petri Latvala @ 2019-08-16 9:34 UTC (permalink / raw)
To: igt-dev; +Cc: Petri Latvala
Instead of generating a subtest for each engine in a static list,
convert to dynamic subtests, with one dynamic subtest per actually
present physical engine.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
---
tests/i915/gem_exec_basic.c | 26 +++++++++++++++++++-------
1 file changed, 19 insertions(+), 7 deletions(-)
diff --git a/tests/i915/gem_exec_basic.c b/tests/i915/gem_exec_basic.c
index 1287860b..a1d1922b 100644
--- a/tests/i915/gem_exec_basic.c
+++ b/tests/i915/gem_exec_basic.c
@@ -159,13 +159,25 @@ igt_main
igt_subtest("gtt-all")
gtt_all(fd);
- __for_each_physical_engine(fd, e) {
- igt_subtest_f("basic-%s", e->name)
- noop(fd, e->flags);
- igt_subtest_f("readonly-%s", e->name)
- readonly(fd, e->flags);
- igt_subtest_f("gtt-%s", e->name)
- gtt(fd, e->flags);
+ igt_dynamic_subtest_container("basic") {
+ __for_each_physical_engine(fd, e) {
+ igt_dynamic_subtest_f("%s", e->name)
+ noop(fd, e->flags);
+ }
+ }
+
+ igt_dynamic_subtest_container("readonly") {
+ __for_each_physical_engine(fd, e) {
+ igt_dynamic_subtest_f("%s", e->name)
+ readonly(fd, e->flags);
+ }
+ }
+
+ igt_dynamic_subtest_container("gtt") {
+ __for_each_physical_engine(fd, e) {
+ igt_dynamic_subtest_f("%s", e->name)
+ gtt(fd, e->flags);
+ }
}
igt_fixture {
--
2.19.1
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply related [flat|nested] 20+ messages in thread* [igt-dev] ✓ Fi.CI.BAT: success for Dynamic subtests (rev2)
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (9 preceding siblings ...)
2019-08-16 9:34 ` [igt-dev] [PATCH i-g-t 10/10] i915/gem_exec_basic: " Petri Latvala
@ 2019-08-16 10:17 ` Patchwork
2019-08-16 22:13 ` [igt-dev] ✓ Fi.CI.IGT: " Patchwork
11 siblings, 0 replies; 20+ messages in thread
From: Patchwork @ 2019-08-16 10:17 UTC (permalink / raw)
To: Petri Latvala; +Cc: igt-dev
== Series Details ==
Series: Dynamic subtests (rev2)
URL : https://patchwork.freedesktop.org/series/62384/
State : success
== Summary ==
CI Bug Log - changes from CI_DRM_6716 -> IGTPW_3355
====================================================
Summary
-------
**SUCCESS**
No regressions found.
External URL: https://patchwork.freedesktop.org/api/1.0/series/62384/revisions/2/mbox/
Known issues
------------
Here are the changes found in IGTPW_3355 that come from known issues:
### IGT changes ###
#### Issues hit ####
* igt@kms_busy@basic-flip-a:
- fi-kbl-7567u: [PASS][1] -> [SKIP][2] ([fdo#109271] / [fdo#109278]) +2 similar issues
[1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-kbl-7567u/igt@kms_busy@basic-flip-a.html
[2]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-kbl-7567u/igt@kms_busy@basic-flip-a.html
#### Possible fixes ####
* igt@gem_exec_reloc@basic-write-gtt:
- fi-icl-u3: [DMESG-WARN][3] ([fdo#107724]) -> [PASS][4]
[3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-icl-u3/igt@gem_exec_reloc@basic-write-gtt.html
[4]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-icl-u3/igt@gem_exec_reloc@basic-write-gtt.html
* igt@i915_selftest@live_execlists:
- fi-skl-gvtdvm: [DMESG-FAIL][5] ([fdo#111108]) -> [PASS][6]
[5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-skl-gvtdvm/igt@i915_selftest@live_execlists.html
[6]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-skl-gvtdvm/igt@i915_selftest@live_execlists.html
- fi-bwr-2160: [DMESG-WARN][7] ([fdo#111115]) -> [PASS][8]
[7]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-bwr-2160/igt@i915_selftest@live_execlists.html
[8]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-bwr-2160/igt@i915_selftest@live_execlists.html
* igt@i915_selftest@live_hangcheck:
- fi-bwr-2160: [DMESG-FAIL][9] ([fdo#111115]) -> [PASS][10]
[9]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-bwr-2160/igt@i915_selftest@live_hangcheck.html
[10]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-bwr-2160/igt@i915_selftest@live_hangcheck.html
* igt@i915_selftest@live_requests:
- fi-byt-j1900: [INCOMPLETE][11] ([fdo#102657]) -> [PASS][12]
[11]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-byt-j1900/igt@i915_selftest@live_requests.html
[12]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-byt-j1900/igt@i915_selftest@live_requests.html
* igt@kms_chamelium@common-hpd-after-suspend:
- fi-kbl-7567u: [WARN][13] ([fdo#109380]) -> [PASS][14]
[13]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-kbl-7567u/igt@kms_chamelium@common-hpd-after-suspend.html
[14]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-kbl-7567u/igt@kms_chamelium@common-hpd-after-suspend.html
* igt@kms_frontbuffer_tracking@basic:
- fi-icl-u2: [FAIL][15] ([fdo#103167]) -> [PASS][16]
[15]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_6716/fi-icl-u2/igt@kms_frontbuffer_tracking@basic.html
[16]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/fi-icl-u2/igt@kms_frontbuffer_tracking@basic.html
[fdo#102657]: https://bugs.freedesktop.org/show_bug.cgi?id=102657
[fdo#103167]: https://bugs.freedesktop.org/show_bug.cgi?id=103167
[fdo#107724]: https://bugs.freedesktop.org/show_bug.cgi?id=107724
[fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
[fdo#109278]: https://bugs.freedesktop.org/show_bug.cgi?id=109278
[fdo#109380]: https://bugs.freedesktop.org/show_bug.cgi?id=109380
[fdo#111108]: https://bugs.freedesktop.org/show_bug.cgi?id=111108
[fdo#111115]: https://bugs.freedesktop.org/show_bug.cgi?id=111115
Participating hosts (54 -> 45)
------------------------------
Additional (1): fi-cfl-8700k
Missing (10): fi-kbl-soraka fi-ilk-m540 fi-bdw-5557u fi-hsw-4200u fi-byt-squawks fi-bsw-cyan fi-ctg-p8600 fi-icl-y fi-byt-clapper fi-bdw-samus
Build changes
-------------
* CI: CI-20190529 -> None
* IGT: IGT_5138 -> IGTPW_3355
CI-20190529: 20190529
CI_DRM_6716: 64ecd8f88d7b55de82ff414784ae4daca93d0577 @ git://anongit.freedesktop.org/gfx-ci/linux
IGTPW_3355: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/
IGT_5138: b9abe0bf6c478c4f6cac56bff286d6926ad8c0ab @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
== Testlist changes ==
+++ 38 lines
--- 223 lines
== Logs ==
For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread* [igt-dev] ✓ Fi.CI.IGT: success for Dynamic subtests (rev2)
2019-08-16 9:34 [igt-dev] [PATCH i-g-t 00/10] Dynamic subtests, v2 Petri Latvala
` (10 preceding siblings ...)
2019-08-16 10:17 ` [igt-dev] ✓ Fi.CI.BAT: success for Dynamic subtests (rev2) Patchwork
@ 2019-08-16 22:13 ` Patchwork
11 siblings, 0 replies; 20+ messages in thread
From: Patchwork @ 2019-08-16 22:13 UTC (permalink / raw)
To: Petri Latvala; +Cc: igt-dev
== Series Details ==
Series: Dynamic subtests (rev2)
URL : https://patchwork.freedesktop.org/series/62384/
State : success
== Summary ==
CI Bug Log - changes from CI_DRM_6716_full -> IGTPW_3355_full
====================================================
Summary
-------
**SUCCESS**
No regressions found.
External URL: https://patchwork.freedesktop.org/api/1.0/series/62384/revisions/2/mbox/
Possible new issues
-------------------
Here are the unknown changes that may have been introduced in IGTPW_3355_full:
### IGT changes ###
#### Possible regressions ####
* {igt@perf_pmu@busy} (NEW):
- shard-iclb: NOTRUN -> [WARN][1]
[1]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/shard-iclb1/igt@perf_pmu@busy.html
* {igt@perf_pmu@busy-check-all} (NEW):
- shard-apl: NOTRUN -> [WARN][2]
[2]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/shard-apl4/igt@perf_pmu@busy-check-all.html
* {igt@perf_pmu@init-sema} (NEW):
- shard-kbl: NOTRUN -> [WARN][3] +3 similar issues
[3]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/shard-kbl3/igt@perf_pmu@init-sema.html
New tests
---------
New tests have been introduced between CI_DRM_6716_full and IGTPW_3355_full:
### New IGT tests (204) ###
* igt@gem_exec_basic@basic:
- Statuses : 6 pass(s)
- Exec time: [0.00, 0.01] s
* igt@gem_exec_basic@basic@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@basic@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.00] s
* igt@gem_exec_basic@basic@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@basic@vcs1:
- Statuses : 2 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@basic@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@gtt:
- Statuses : 6 pass(s)
- Exec time: [0.00, 0.01] s
* igt@gem_exec_basic@gtt@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@gtt@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.00] s
* igt@gem_exec_basic@gtt@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@gtt@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.0] s
* igt@gem_exec_basic@gtt@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@readonly:
- Statuses : 6 pass(s)
- Exec time: [0.00, 0.01] s
* igt@gem_exec_basic@readonly@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@readonly@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.00] s
* igt@gem_exec_basic@readonly@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@gem_exec_basic@readonly@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.0] s
* igt@gem_exec_basic@readonly@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@kms_plane_cursor@overlay-size-128:
- Statuses : 6 pass(s)
- Exec time: [5.52, 9.62] s
* igt@kms_plane_cursor@overlay-size-128@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [2.19, 2.72] s
* igt@kms_plane_cursor@overlay-size-128@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.24, 3.59] s
* igt@kms_plane_cursor@overlay-size-128@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.23, 3.58] s
* igt@kms_plane_cursor@overlay-size-256:
- Statuses : 6 pass(s)
- Exec time: [5.53, 9.63] s
* igt@kms_plane_cursor@overlay-size-256@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [2.20, 2.73] s
* igt@kms_plane_cursor@overlay-size-256@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.23, 3.58] s
* igt@kms_plane_cursor@overlay-size-256@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.24, 3.59] s
* igt@kms_plane_cursor@overlay-size-64:
- Statuses : 6 pass(s)
- Exec time: [5.51, 10.10] s
* igt@kms_plane_cursor@overlay-size-64@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [2.20, 2.71] s
* igt@kms_plane_cursor@overlay-size-64@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.23, 3.84] s
* igt@kms_plane_cursor@overlay-size-64@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.24, 3.79] s
* igt@kms_plane_cursor@primary-size-128:
- Statuses : 5 pass(s) 1 warn(s)
- Exec time: [4.95, 9.56] s
* igt@kms_plane_cursor@primary-size-128@pipe-a:
- Statuses : 5 pass(s)
- Exec time: [2.19, 2.56] s
* igt@kms_plane_cursor@primary-size-128@pipe-b:
- Statuses : 5 pass(s)
- Exec time: [2.20, 3.55] s
* igt@kms_plane_cursor@primary-size-128@pipe-c:
- Statuses : 4 pass(s)
- Exec time: [2.22, 3.58] s
* igt@kms_plane_cursor@primary-size-256:
- Statuses : 6 pass(s)
- Exec time: [4.95, 10.08] s
* igt@kms_plane_cursor@primary-size-256@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [2.19, 2.58] s
* igt@kms_plane_cursor@primary-size-256@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.20, 3.82] s
* igt@kms_plane_cursor@primary-size-256@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.22, 3.78] s
* igt@kms_plane_cursor@primary-size-64:
- Statuses : 6 pass(s)
- Exec time: [4.95, 7.72] s
* igt@kms_plane_cursor@primary-size-64@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [1.64, 2.57] s
* igt@kms_plane_cursor@primary-size-64@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.20, 2.74] s
* igt@kms_plane_cursor@primary-size-64@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.22, 2.75] s
* igt@kms_plane_cursor@viewport-size-128:
- Statuses : 6 pass(s)
- Exec time: [5.53, 9.65] s
* igt@kms_plane_cursor@viewport-size-128@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [2.19, 2.73] s
* igt@kms_plane_cursor@viewport-size-128@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.22, 3.61] s
* igt@kms_plane_cursor@viewport-size-128@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.24, 3.58] s
* igt@kms_plane_cursor@viewport-size-256:
- Statuses : 5 pass(s)
- Exec time: [5.53, 7.79] s
* igt@kms_plane_cursor@viewport-size-256@pipe-a:
- Statuses : 5 pass(s)
- Exec time: [2.20, 2.73] s
* igt@kms_plane_cursor@viewport-size-256@pipe-b:
- Statuses : 5 pass(s)
- Exec time: [2.24, 2.80] s
* igt@kms_plane_cursor@viewport-size-256@pipe-c:
- Statuses : 4 pass(s)
- Exec time: [2.25, 2.61] s
* igt@kms_plane_cursor@viewport-size-64:
- Statuses : 6 pass(s)
- Exec time: [5.52, 9.60] s
* igt@kms_plane_cursor@viewport-size-64@pipe-a:
- Statuses : 6 pass(s)
- Exec time: [2.22, 2.72] s
* igt@kms_plane_cursor@viewport-size-64@pipe-b:
- Statuses : 6 pass(s)
- Exec time: [2.27, 3.56] s
* igt@kms_plane_cursor@viewport-size-64@pipe-c:
- Statuses : 5 pass(s)
- Exec time: [2.27, 3.58] s
* igt@perf_pmu@busy:
- Statuses : 5 pass(s) 1 warn(s)
- Exec time: [2.02, 2.57] s
* igt@perf_pmu@busy-accuracy-2:
- Statuses : 6 pass(s)
- Exec time: [0.00, 15.18] s
* igt@perf_pmu@busy-accuracy-2@bcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 3.05] s
* igt@perf_pmu@busy-accuracy-2@rcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 3.05] s
* igt@perf_pmu@busy-accuracy-2@vcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 3.05] s
* igt@perf_pmu@busy-accuracy-2@vcs1:
- Statuses : 2 pass(s)
- Exec time: [3.03, 3.04] s
* igt@perf_pmu@busy-accuracy-2@vecs0:
- Statuses : 4 pass(s) 1 skip(s)
- Exec time: [0.0, 3.05] s
* igt@perf_pmu@busy-accuracy-50:
- Statuses : 5 pass(s) 1 warn(s)
- Exec time: [0.00, 7.61] s
* igt@perf_pmu@busy-accuracy-50@bcs0:
- Statuses : 3 pass(s) 2 skip(s)
- Exec time: [0.0, 1.54] s
* igt@perf_pmu@busy-accuracy-50@rcs0:
- Statuses : 3 pass(s) 2 skip(s)
- Exec time: [0.0, 1.54] s
* igt@perf_pmu@busy-accuracy-50@vcs0:
- Statuses : 3 pass(s) 2 skip(s)
- Exec time: [0.0, 1.54] s
* igt@perf_pmu@busy-accuracy-50@vecs0:
- Statuses : 3 pass(s) 1 skip(s)
- Exec time: [0.0, 1.54] s
* igt@perf_pmu@busy-accuracy-98:
- Statuses : 6 pass(s)
- Exec time: [0.00, 15.11] s
* igt@perf_pmu@busy-accuracy-98@bcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 3.04] s
* igt@perf_pmu@busy-accuracy-98@rcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 3.04] s
* igt@perf_pmu@busy-accuracy-98@vcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 3.04] s
* igt@perf_pmu@busy-accuracy-98@vcs1:
- Statuses : 1 pass(s)
- Exec time: [3.02] s
* igt@perf_pmu@busy-accuracy-98@vecs0:
- Statuses : 4 pass(s) 1 skip(s)
- Exec time: [0.0, 3.04] s
* igt@perf_pmu@busy-check-all:
- Statuses : 5 pass(s) 1 warn(s)
- Exec time: [2.03, 2.58] s
* igt@perf_pmu@busy-check-all@bcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@busy-check-all@rcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@busy-check-all@vcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 1.01] s
* igt@perf_pmu@busy-check-all@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.52] s
* igt@perf_pmu@busy-check-all@vecs0:
- Statuses : 4 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@busy-double-start:
- Statuses : 6 pass(s)
- Exec time: [11.03, 17.59] s
* igt@perf_pmu@busy-double-start@bcs0:
- Statuses : 6 pass(s)
- Exec time: [3.51, 3.52] s
* igt@perf_pmu@busy-double-start@rcs0:
- Statuses : 6 pass(s)
- Exec time: [3.51, 3.52] s
* igt@perf_pmu@busy-double-start@vcs0:
- Statuses : 6 pass(s)
- Exec time: [3.51, 4.01] s
* igt@perf_pmu@busy-double-start@vcs1:
- Statuses : 1 pass(s)
- Exec time: [3.52] s
* igt@perf_pmu@busy-double-start@vecs0:
- Statuses : 5 pass(s)
- Exec time: [3.51, 3.52] s
* igt@perf_pmu@busy-hang:
- Statuses : 6 pass(s)
- Exec time: [3.53, 5.09] s
* igt@perf_pmu@busy-hang@bcs0:
- Statuses : 6 pass(s)
- Exec time: [1.01, 1.02] s
* igt@perf_pmu@busy-hang@rcs0:
- Statuses : 6 pass(s)
- Exec time: [1.01, 1.02] s
* igt@perf_pmu@busy-hang@vcs0:
- Statuses : 6 pass(s)
- Exec time: [1.01, 1.51] s
* igt@perf_pmu@busy-hang@vcs1:
- Statuses : 2 pass(s)
- Exec time: [1.02] s
* igt@perf_pmu@busy-hang@vecs0:
- Statuses : 5 pass(s)
- Exec time: [1.01, 1.02] s
* igt@perf_pmu@busy-idle:
- Statuses : 6 pass(s)
- Exec time: [2.33, 3.09] s
* igt@perf_pmu@busy-idle-check-all:
- Statuses : 6 pass(s)
- Exec time: [2.34, 3.09] s
* igt@perf_pmu@busy-idle-check-all@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@busy-idle-check-all@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@busy-idle-check-all@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 1.11] s
* igt@perf_pmu@busy-idle-check-all@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.62] s
* igt@perf_pmu@busy-idle-check-all@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@busy-idle-no-semaphores:
- Statuses : 6 pass(s)
- Exec time: [2.31, 3.04] s
* igt@perf_pmu@busy-idle-no-semaphores@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.60, 0.61] s
* igt@perf_pmu@busy-idle-no-semaphores@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.60, 0.61] s
* igt@perf_pmu@busy-idle-no-semaphores@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.60, 1.10] s
* igt@perf_pmu@busy-idle-no-semaphores@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.61] s
* igt@perf_pmu@busy-idle-no-semaphores@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.60, 0.61] s
* igt@perf_pmu@busy-idle@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@busy-idle@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61] s
* igt@perf_pmu@busy-idle@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 1.11] s
* igt@perf_pmu@busy-idle@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.62] s
* igt@perf_pmu@busy-idle@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@busy-no-semaphores:
- Statuses : 6 pass(s)
- Exec time: [2.00, 2.52] s
* igt@perf_pmu@busy-no-semaphores@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.50, 0.51] s
* igt@perf_pmu@busy-no-semaphores@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.50, 0.51] s
* igt@perf_pmu@busy-no-semaphores@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.50, 1.00] s
* igt@perf_pmu@busy-no-semaphores@vcs1:
- Statuses : 2 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@busy-no-semaphores@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.50, 0.51] s
* igt@perf_pmu@busy-start:
- Statuses : 6 pass(s)
- Exec time: [8.02, 12.57] s
* igt@perf_pmu@busy-start@bcs0:
- Statuses : 6 pass(s)
- Exec time: [2.51] s
* igt@perf_pmu@busy-start@rcs0:
- Statuses : 6 pass(s)
- Exec time: [2.51, 2.52] s
* igt@perf_pmu@busy-start@vcs0:
- Statuses : 6 pass(s)
- Exec time: [2.51, 3.01] s
* igt@perf_pmu@busy-start@vcs1:
- Statuses : 1 pass(s)
- Exec time: [2.51] s
* igt@perf_pmu@busy-start@vecs0:
- Statuses : 5 pass(s)
- Exec time: [2.51, 2.52] s
* igt@perf_pmu@busy@bcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@busy@rcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@busy@vcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 1.01] s
* igt@perf_pmu@busy@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.51] s
* igt@perf_pmu@busy@vecs0:
- Statuses : 4 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@enable-race:
- Statuses : 6 pass(s)
- Exec time: [0.00, 62.82] s
* igt@perf_pmu@enable-race@bcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 12.62] s
* igt@perf_pmu@enable-race@rcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 12.62] s
* igt@perf_pmu@enable-race@vcs0:
- Statuses : 4 pass(s) 2 skip(s)
- Exec time: [0.0, 12.62] s
* igt@perf_pmu@enable-race@vcs1:
- Statuses : 1 pass(s)
- Exec time: [12.56] s
* igt@perf_pmu@enable-race@vecs0:
- Statuses : 4 pass(s) 1 skip(s)
- Exec time: [0.0, 12.62] s
* igt@perf_pmu@event-wait:
- Statuses : 5 pass(s)
- Exec time: [0.0, 1.83] s
* igt@perf_pmu@event-wait@rcs0:
- Statuses : 4 pass(s) 1 skip(s)
- Exec time: [0.0, 1.83] s
* igt@perf_pmu@idle:
- Statuses : 6 pass(s)
- Exec time: [1.50, 2.51] s
* igt@perf_pmu@idle-no-semaphores:
- Statuses : 5 pass(s)
- Exec time: [1.50, 2.51] s
* igt@perf_pmu@idle-no-semaphores@bcs0:
- Statuses : 5 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle-no-semaphores@rcs0:
- Statuses : 5 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle-no-semaphores@vcs0:
- Statuses : 5 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle-no-semaphores@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle-no-semaphores@vecs0:
- Statuses : 4 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@idle@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.50] s
* igt@perf_pmu@init-busy:
- Statuses : 6 pass(s)
- Exec time: [0.00, 0.01] s
* igt@perf_pmu@init-busy@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-busy@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.00] s
* igt@perf_pmu@init-busy@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-busy@vcs1:
- Statuses : 2 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-busy@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-sema:
- Statuses : 5 pass(s) 1 warn(s)
- Exec time: [0.00, 0.01] s
* igt@perf_pmu@init-sema@bcs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-sema@rcs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-sema@vcs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-sema@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.00] s
* igt@perf_pmu@init-sema@vecs0:
- Statuses : 4 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-wait:
- Statuses : 6 pass(s)
- Exec time: [0.00, 0.01] s
* igt@perf_pmu@init-wait@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-wait@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-wait@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@init-wait@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.0] s
* igt@perf_pmu@init-wait@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.0, 0.00] s
* igt@perf_pmu@most-busy-check-all:
- Statuses : 6 pass(s)
- Exec time: [1.53, 2.58] s
* igt@perf_pmu@most-busy-check-all@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@most-busy-check-all@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@most-busy-check-all@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@most-busy-check-all@vcs1:
- Statuses : 1 pass(s)
- Exec time: [0.52] s
* igt@perf_pmu@most-busy-check-all@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@most-busy-idle-check-all:
- Statuses : 5 pass(s)
- Exec time: [1.84, 3.09] s
* igt@perf_pmu@most-busy-idle-check-all@bcs0:
- Statuses : 5 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@most-busy-idle-check-all@rcs0:
- Statuses : 5 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@most-busy-idle-check-all@vcs0:
- Statuses : 5 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@most-busy-idle-check-all@vcs1:
- Statuses : 2 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@most-busy-idle-check-all@vecs0:
- Statuses : 4 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@multi-client:
- Statuses : 6 pass(s)
- Exec time: [3.52, 5.06] s
* igt@perf_pmu@multi-client@bcs0:
- Statuses : 6 pass(s)
- Exec time: [1.01, 1.02] s
* igt@perf_pmu@multi-client@rcs0:
- Statuses : 6 pass(s)
- Exec time: [1.00, 1.01] s
* igt@perf_pmu@multi-client@vcs0:
- Statuses : 6 pass(s)
- Exec time: [1.01, 1.51] s
* igt@perf_pmu@multi-client@vcs1:
- Statuses : 2 pass(s)
- Exec time: [1.01, 1.02] s
* igt@perf_pmu@multi-client@vecs0:
- Statuses : 5 pass(s)
- Exec time: [1.01, 1.02] s
* igt@perf_pmu@render-node-busy:
- Statuses : 5 pass(s) 1 warn(s)
- Exec time: [2.02, 2.58] s
* igt@perf_pmu@render-node-busy-idle:
- Statuses : 6 pass(s)
- Exec time: [2.33, 3.09] s
* igt@perf_pmu@render-node-busy-idle@bcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@render-node-busy-idle@rcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61] s
* igt@perf_pmu@render-node-busy-idle@vcs0:
- Statuses : 6 pass(s)
- Exec time: [0.61, 1.11] s
* igt@perf_pmu@render-node-busy-idle@vcs1:
- Statuses : 2 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@render-node-busy-idle@vecs0:
- Statuses : 5 pass(s)
- Exec time: [0.61, 0.62] s
* igt@perf_pmu@render-node-busy@bcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@render-node-busy@rcs0:
- Statuses : 5 pass(s)
- Exec time: [0.51, 0.52] s
* igt@perf_pmu@render-node-busy@vcs0:
-
== Logs ==
For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_3355/
_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev
^ permalink raw reply [flat|nested] 20+ messages in thread