[LTP] [PATCH] lib: tst_test: Add reproducible output.

[LTP] [PATCH] lib: tst_test: Add reproducible output.

[Cyril Hrubis]

This commit adds an environment variable LTP_REPRODUCIBLE_OUTPUT that
when set skips printing parts of the test messages that may contain data
that differ on subsequent runs (e.g. pids).

With this you can run a test twice under a different conditions and
check if the test codeflow was identical by simply doing diff of the
outputs from the two runs.

Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
Suggested-by: Martin Doucha <mdoucha@suse.cz>
CC: valgrind-developers@lists.sourceforge.net
---
 lib/tst_test.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/tst_test.c b/lib/tst_test.c
index 2bb4519dd..3f94a1607 100644
--- a/lib/tst_test.c
+++ b/lib/tst_test.c
@@ -64,6 +64,7 @@ static int mntpoint_mounted;
 static int ovl_mounted;
 static struct timespec tst_start_time; /* valid only for test pid */
 static int tdebug;
+static int reproducible_output;
 
 struct results {
 	int passed;
@@ -304,6 +305,9 @@ static void print_result(const char *file, const int lineno, int ttype,
 	str += ret;
 	size -= ret;
 
+	if (reproducible_output)
+		goto print;
+
 	if (tst_color_enabled(STDERR_FILENO))
 		ret = snprintf(str, size, "%s%s: %s", tst_ttype2color(ttype),
 			       res, ANSI_COLOR_RESET);
@@ -329,6 +333,7 @@ static void print_result(const char *file, const int lineno, int ttype,
 				"Next message is too long and truncated:");
 	}
 
+print:
 	snprintf(str, size, "\n");
 
 	/* we might be called from signal handler, so use write() */
@@ -1312,6 +1317,8 @@ static void do_setup(int argc, char *argv[])
 	if (tst_test->supported_archs && !tst_is_on_arch(tst_test->supported_archs))
 		tst_brk(TCONF, "This arch '%s' is not supported for test!", tst_arch.name);
 
+	reproducible_output = !!getenv("LTP_REPRODUCIBLE_OUTPUT");
+
 	assert_test_fn();
 
 	TCID = tid = get_tid(argv);
-- 
2.49.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH] lib: tst_test: Add reproducible output.

[Martin Doucha]

Hi,
one suggestion below.

On 09. 05. 25 10:34, Cyril Hrubis wrote:
> This commit adds an environment variable LTP_REPRODUCIBLE_OUTPUT that
> when set skips printing parts of the test messages that may contain data
> that differ on subsequent runs (e.g. pids).
> 
> With this you can run a test twice under a different conditions and
> check if the test codeflow was identical by simply doing diff of the
> outputs from the two runs.
> 
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
> Suggested-by: Martin Doucha <mdoucha@suse.cz>
> CC: valgrind-developers@lists.sourceforge.net
> ---
>   lib/tst_test.c | 7 +++++++
>   1 file changed, 7 insertions(+)
> 
> diff --git a/lib/tst_test.c b/lib/tst_test.c
> index 2bb4519dd..3f94a1607 100644
> --- a/lib/tst_test.c
> +++ b/lib/tst_test.c
> @@ -64,6 +64,7 @@ static int mntpoint_mounted;
>   static int ovl_mounted;
>   static struct timespec tst_start_time; /* valid only for test pid */
>   static int tdebug;
> +static int reproducible_output;
>   
>   struct results {
>   	int passed;
> @@ -304,6 +305,9 @@ static void print_result(const char *file, const int lineno, int ttype,
>   	str += ret;
>   	size -= ret;
>   
> +	if (reproducible_output)
> +		goto print;
> +

I'd move this goto one code chunk down (after the color if-else) so that 
we also get the res string in the output. Or at least add a strncpy(str, 
res, size) before the goto if you want to bypass the color logic.

>   	if (tst_color_enabled(STDERR_FILENO))
>   		ret = snprintf(str, size, "%s%s: %s", tst_ttype2color(ttype),
>   			       res, ANSI_COLOR_RESET);
> @@ -329,6 +333,7 @@ static void print_result(const char *file, const int lineno, int ttype,
>   				"Next message is too long and truncated:");
>   	}
>   
> +print:
>   	snprintf(str, size, "\n");
>   
>   	/* we might be called from signal handler, so use write() */
> @@ -1312,6 +1317,8 @@ static void do_setup(int argc, char *argv[])
>   	if (tst_test->supported_archs && !tst_is_on_arch(tst_test->supported_archs))
>   		tst_brk(TCONF, "This arch '%s' is not supported for test!", tst_arch.name);
>   
> +	reproducible_output = !!getenv("LTP_REPRODUCIBLE_OUTPUT");
> +
>   	assert_test_fn();
>   
>   	TCID = tid = get_tid(argv);


-- 
Martin Doucha   mdoucha@suse.cz
SW Quality Engineer
SUSE LINUX, s.r.o.
CORSO IIa
Krizikova 148/34
186 00 Prague 8
Czech Republic

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH] lib: tst_test: Add reproducible output.

[Cyril Hrubis]

Hi!
> >   struct results {
> >   	int passed;
> > @@ -304,6 +305,9 @@ static void print_result(const char *file, const int lineno, int ttype,
> >   	str += ret;
> >   	size -= ret;
> >   
> > +	if (reproducible_output)
> > +		goto print;
> > +
> 
> I'd move this goto one code chunk down (after the color if-else) so that 
> we also get the res string in the output. Or at least add a strncpy(str, 
> res, size) before the goto if you want to bypass the color logic.

The color output is disabled when output is redirected into a file so I
suppose that we can simply move the goto after we print the result. I
will send a v2.

-- 
Cyril Hrubis
chrubis@suse.cz

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp