[RFC PATCH 2/4] check: add --loop option to run each test multiple times

[Zorro Lang <zlang@kernel.org>]

From: Theodore Ts'o <tytso@mit.edu>

Teach the check script a new option --loop, which re-runs each test
multiple times.  This works very similarly to -L, which retries a test
after it first fails, except that the test is rerun unconditionally.

This differs from the "-i <n>" option, which iterates each set of
tests <n> times instead of each test.  The -i option is problematic in
two ways.  First, it doesn't save the test artifacts from each test
run.  This is unfortunate because when the developer is trying to debug
a flaky test failure, running "check -i 100" will run a test 100
times, but if only the 42nd test fails, the NNN.out.bad file for that
failing test run is not preserved.  The second difference between
--loop and -i is that the result.xml file is rewritten after each test
set, so we do not have the cumulative statistics of the 100 test runs
in the junit XML file.

Refactor the rerun-on-failure state machine so that it can support
multiple loop modes: the loop count is normalized to the total number
of runs at option parsing time, and the aggregate-results logic is
shared between the initial run and the reruns.  This also fixes an
off-by-one where --loop=1 would run the test twice.  A test run with
--loop fails if any of its runs fails, matching the behaviour of -i.
--loop and -L select different loop modes and cannot be combined.

Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Zorro Lang <zlang@kernel.org>
---
 check | 122 +++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 95 insertions(+), 27 deletions(-)

diff --git a/check b/check
index 538f22cf..b30532cf 100755
--- a/check
+++ b/check
@@ -26,7 +26,10 @@ do_report=false
 DUMP_OUTPUT=false
 iterations=1
 istop=false
-loop_on_fail=0
+# Selected by -L / --loop and its siblings: "", "always" or "on-fail"
+loop_mode=""
+# Total number of runs for the current loop mode
+loop_total=0
 exclude_tests=()
 
 # This is a global variable used to pass test failure text to reporting gunk
@@ -69,6 +72,8 @@ General options:
     -l                      line mode diff
     -L <n>                  loop tests <n> times following a failure, measuring
                               aggregate pass/fail metrics
+    --loop <n>              run each test <n> times, measuring aggregate
+                              pass/fail metrics (test fails if any run fails)
     -n                      show me, do not run tests
     -r                      randomize test order
     -s section              run only specified section from config file
@@ -386,10 +391,34 @@ process_remain_tests()
 	done
 }
 
+# Record the loop mode selected by -L / --loop / --loop-on-fail /
+# --loop-while-successful and normalize its count into the total number of
+# runs (including the first one).  The loop modes are mutually exclusive,
+# while repeating the same mode just overrides its count.
+_set_loop_mode()
+{
+	local mode="$1"
+	local count="$2"
+
+	[[ $count =~ ^[0-9]+$ ]] || usage
+	if [ -n "$loop_mode" ] && [ "$loop_mode" != "$mode" ]; then
+		echo "check: conflicting loop modes: -L and --loop are mutually exclusive" >&2
+		usage
+	fi
+	loop_mode="$mode"
+	if [ "$mode" = "on-fail" ]; then
+		# <n> reruns after the first failure, <n>+1 runs in total
+		loop_total=$((count + 1))
+	else
+		# <n> runs in total
+		loop_total=$count
+	fi
+}
+
 parse_check_args()
 {
 	local short_opts="g:x:X:e:E:s:S:lnri:I:TdbR:L:h"
-	local long_opts="fs:,exact-order,large-fs,extra-space:,udiff,help,iterate:,iterate-while-successful:"
+	local long_opts="fs:,exact-order,large-fs,extra-space:,udiff,help,iterate:,iterate-while-successful:,loop:"
 
 	compat_old_option "$@"
 
@@ -498,8 +527,11 @@ parse_check_args()
 			shift
 			;;
 		-L)
-			[[ $2 =~ ^[0-9]+$ ]] || usage
-			loop_on_fail=$2
+			_set_loop_mode on-fail "$2"
+			shift
+			;;
+		--loop)
+			_set_loop_mode always "$2"
 			shift
 			;;
 		-h|--help)
@@ -747,7 +779,7 @@ _expunge_test()
 }
 
 # retain files which would be overwritten in subsequent reruns of the same test
-_stash_fail_loop_files() {
+_stash_loop_files() {
 	local seq_prefix="${REPORT_DIR}/${1}"
 	local cp_suffix="$2"
 
@@ -759,6 +791,41 @@ _stash_fail_loop_files() {
 	done
 }
 
+# Record a failed test in @bad, avoiding duplicates for tests that already
+# failed in an earlier run of their loop.
+_add_bad() {
+	local test_seq="$1"
+	local b
+
+	for b in "${bad[@]}"; do
+		[ "$b" = "$test_seq" ] && return
+	done
+	bad+=("$test_seq")
+}
+
+# Print the aggregate results of the current loop once all loop_total runs
+# have been recorded, then reset the loop state so that the next test can
+# start.
+_try_loop_aggregate() {
+	local test_seq="$1"
+
+	if ((${#loop_status[*]} >= loop_total)); then
+		printf "%s aggregate results across %d runs: " \
+			"$test_seq" "${#loop_status[*]}"
+		awk "BEGIN {
+			n=split(\"${loop_status[*]}\", arr);"'
+			for (i = 1; i <= n; i++)
+				stats[arr[i]]++;
+			for (x in stats)
+				printf("%s=%d (%.1f%%)",
+				       (i-- > n ? x : ", " x),
+				       stats[x], 100 * stats[x] / n);
+			}'
+		echo
+		loop_status=()
+	fi
+}
+
 # Retain in @bad / @notrun the result of the just-run @test_seq. @try array
 # entries are added prior to execution.
 _stash_test_status() {
@@ -771,40 +838,41 @@ _stash_test_status() {
 	fi
 
 	if ((${#loop_status[*]} > 0)); then
-		# continuing or completing rerun-on-failure loop
-		_stash_fail_loop_files "$test_seq" ".rerun${#loop_status[*]}"
+		# continuing or completing a rerun loop
+		_stash_loop_files "$test_seq" ".rerun${#loop_status[*]}"
 		loop_status+=("$test_status")
-		if ((${#loop_status[*]} > loop_on_fail)); then
-			printf "%s aggregate results across %d runs: " \
-				"$test_seq" "${#loop_status[*]}"
-			awk "BEGIN {
-				n=split(\"${loop_status[*]}\", arr);"'
-				for (i = 1; i <= n; i++)
-					stats[arr[i]]++;
-				for (x in stats)
-					printf("%s=%d (%.1f%%)",
-					       (i-- > n ? x : ", " x),
-					       stats[x], 100 * stats[x] / n);
-				}'
-			echo
-			loop_status=()
+		if [ "$test_status" = "fail" ]; then
+			# any failed run fails the test
+			_add_bad "$test_seq"
 		fi
-		return	# only stash @bad result for initial failure in loop
+		_try_loop_aggregate "$test_seq"
+		# the rerun loop handles @bad/@notrun by itself
+		return
 	fi
 
 	case "$test_status" in
 	fail)
-		if ((loop_on_fail > 0)); then
-			# initial failure, start rerun-on-failure loop
-			_stash_fail_loop_files "$test_seq" ".rerun0"
+		bad+=("$test_seq")
+		if { [ "$loop_mode" = "always" ] || [ "$loop_mode" = "on-fail" ]; } \
+		   && ((loop_total > 1)); then
+			# initial failure, start a rerun loop
+			_stash_loop_files "$test_seq" ".rerun0"
 			loop_status+=("$test_status")
+			_try_loop_aggregate "$test_seq"
 		fi
-		bad+=("$test_seq")
 		;;
 	list|notrun)
 		notrun+=("$test_seq")
 		;;
-	pass|expunge)
+	pass)
+		if [ "$loop_mode" = "always" ] && ((loop_total > 1)); then
+			# start a rerun loop
+			_stash_loop_files "$test_seq" ".rerun0"
+			loop_status+=("$test_status")
+			_try_loop_aggregate "$test_seq"
+		fi
+		;;
+	expunge)
 		;;
 	*)
 		echo "Unexpected test $test_seq status: $test_status"
-- 
2.56.0