[RFC PATCH 2/2] check: add -L <n> parameter to rerun failed tests

[David Disseldorp <ddiss@suse.de>]

If check is run with -L <n>, then a failed test will be rerun <n> times
before proceeding to the next test. Following completion of the rerun
loop, aggregate pass/fail statistics are printed.

Caveats:
- rerun tests will be tracked as a single failure in @try and @bad
  + xunit reports do not include any rerun details
- .bad files generated on failure will be overwritten by test reruns

Suggested-by: Theodore Ts'o <tytso@mit.edu>
Link: https://lwn.net/Articles/897061/
Signed-off-by: David Disseldorp <ddiss@suse.de>
---
 check | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 75 insertions(+), 11 deletions(-)

diff --git a/check b/check
index 3e8a232c..97fa50ad 100755
--- a/check
+++ b/check
@@ -26,6 +26,7 @@ do_report=false
 DUMP_OUTPUT=false
 iterations=1
 istop=false
+loop_on_fail=0
 
 # This is a global variable used to pass test failure text to reporting gunk
 _err_msg=""
@@ -75,6 +76,7 @@ check options
     --large-fs		optimise scratch device for large filesystems
     -s section		run only specified section from config file
     -S section		exclude the specified section from the config file
+    -L <n>		loop tests <n> times following a failure, measuring aggregate pass/fail metrics
 
 testlist options
     -g group[,group...]	include tests from these groups
@@ -333,6 +335,7 @@ while [ $# -gt 0 ]; do
 		;;
 	--large-fs) export LARGE_SCRATCH_DEV=yes ;;
 	--extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
+	-L)	loop_on_fail=$2; shift ;;
 
 	-*)	usage ;;
 	*)	# not an argument, we've got tests now.
@@ -606,6 +609,40 @@ _run_seq() {
 	fi
 }
 
+# Check whether the last test should be rerun according to loop-on-error state
+# and return "0" if so, otherwise return "1".
+_ix_inc() {
+	local test_status="$1"
+	local loop_len="$2"
+
+	if ((!loop_on_fail)); then
+		echo 1
+		return
+	fi
+
+	if [ "$test_status" == "fail" ] && ((!loop_len)); then
+		echo 0	# initial failure of this test, start loop-on-fail
+	elif ((loop_len > 0)) && ((loop_len < loop_on_fail)); then
+		echo 0	# continue loop following initial failure
+	else
+		echo 1	# completed or not currently in a failure loop
+	fi
+}
+
+_failure_loop_dump_stats() {
+	local t="$1" && shift
+	awk "BEGIN {
+		n=split(\"$*\", arr);"'
+		for (i = 1; i <= n; i++)
+			stats[arr[i]]++;
+		printf("'"$t"' aggregate results across %d runs: ", n);
+		for (x in stats)
+			printf("%s=%d (%.1f%%)", (i-- > n ? x : ", " x),
+			       stats[x], 100 * stats[x] / n);
+		print;
+	     }'
+}
+
 _detect_kmemleak
 _prepare_test_list
 
@@ -746,13 +783,33 @@ function run_section()
 	seqres="$check"
 	_check_test_fs
 
-	local tc_status="init"
+	local tc_status="init" ix
 	prev_seq=""
-	for seq in $list ; do
-		# Run report for previous test!
-		_stash_test_status "$seqnum" "$tc_status"
-		if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
-			_make_testcase_report "$prev_seq" "$tc_status"
+	local -a _list=( $list ) loop_status=()
+	for ((ix = 0; ix < ${#_list[*]};
+	      ix += $(_ix_inc "$tc_status" "${#loop_status[*]}"))); do
+		seq="${_list[$ix]}"
+
+		if ((!loop_on_fail)); then
+			# Run report for previous test!
+			_stash_test_status "$seqnum" "$tc_status"
+			if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
+				_make_testcase_report "$prev_seq" "$tc_status"
+			fi
+		else
+			if [ "$seq" == "$prev_seq" ]; then
+				if ((!${#loop_status[*]})); then
+					_stash_test_status "$seqnum" "$tc_status"
+					if $do_report; then
+						_make_testcase_report "$prev_seq" "$tc_status"
+					fi
+				fi
+				loop_status+=("$tc_status")
+			elif ((${#loop_status[*]})); then
+				loop_status+=("$tc_status")
+				_failure_loop_dump_stats "$seqnum" "${loop_status[@]}"
+				loop_status=()
+			fi
 		fi
 
 		prev_seq="$seq"
@@ -822,7 +879,9 @@ function run_section()
 		fi
 
 		# record that we really tried to run this test.
-		try+=("$seqnum")
+		if ((!${#loop_status[*]})); then
+			try+=("$seqnum")
+		fi
 
 		awk 'BEGIN {lasttime="       "} \
 		     $1 == "'$seqnum'" {lasttime=" " $2 "s ... "; exit} \
@@ -949,10 +1008,15 @@ function run_section()
 		fi
 	done
 
-	# make sure we record the status of the last test we ran.
-	_stash_test_status "$seqnum" "$tc_status"
-	if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
-		_make_testcase_report "$prev_seq" "$tc_status"
+	if ((${#loop_status[*]})); then
+		loop_status+=("$tc_status")
+		_failure_loop_dump_stats "$seqnum" "${loop_status[@]}"
+	else
+		# make sure we record the status of the last test we ran.
+		_stash_test_status "$seqnum" "$tc_status"
+		if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
+			_make_testcase_report "$prev_seq" "$tc_status"
+		fi
 	fi
 
 	sect_stop=`_wallclock`
-- 
2.35.3