[RFC PATCH 4/4] check: add --loop-while-successful option

[Zorro Lang <zlang@kernel.org>]

Add one more member of the --loop family: --loop-while-successful runs
each test up to <n> times but stops rerunning it at the first failure.
This is faster than --loop for bisecting flaky tests, since the loop
ends as soon as the failure is reproduced.  The failing run is recorded
in @bad, so a reproduced failure is still reported as a failed test.

Signed-off-by: Zorro Lang <zlang@kernel.org>
---
 check | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/check b/check
index 96e4f2fd..6b7bd23e 100755
--- a/check
+++ b/check
@@ -26,7 +26,8 @@ do_report=false
 DUMP_OUTPUT=false
 iterations=1
 istop=false
-# Selected by -L / --loop and its siblings: "", "always" or "on-fail"
+# Selected by -L / --loop and its siblings: "", "always", "on-fail" or
+# "while-successful"
 loop_mode=""
 # Total number of runs for the current loop mode
 loop_total=0
@@ -74,6 +75,9 @@ General options:
                               aggregate pass/fail metrics
     --loop <n>              run each test <n> times, measuring aggregate
                               pass/fail metrics (test fails if any run fails)
+    --loop-while-successful <n>
+                              run each test up to <n> times, stopping at the
+                              first failure
     -n                      show me, do not run tests
     -r                      randomize test order
     -s section              run only specified section from config file
@@ -402,7 +406,7 @@ _set_loop_mode()
 
 	[[ $count =~ ^[0-9]+$ ]] || usage
 	if [ -n "$loop_mode" ] && [ "$loop_mode" != "$mode" ]; then
-		echo "check: conflicting loop modes: -L, --loop-on-fail and --loop are mutually exclusive" >&2
+		echo "check: conflicting loop modes: -L, --loop, --loop-on-fail and --loop-while-successful are mutually exclusive" >&2
 		usage
 	fi
 	loop_mode="$mode"
@@ -418,7 +422,9 @@ _set_loop_mode()
 parse_check_args()
 {
 	local short_opts="g:x:X:e:E:s:S:lnri:I:TdbR:L:h"
-	local long_opts="fs:,exact-order,large-fs,extra-space:,udiff,help,iterate:,iterate-while-successful:,loop:,loop-on-fail:"
+	local long_opts="fs:,exact-order,large-fs,extra-space:,udiff,help,"
+	long_opts+="iterate:,iterate-while-successful:,"
+	long_opts+="loop:,loop-on-fail:,loop-while-successful:"
 
 	compat_old_option "$@"
 
@@ -534,6 +540,10 @@ parse_check_args()
 			_set_loop_mode always "$2"
 			shift
 			;;
+		--loop-while-successful)
+			_set_loop_mode while-successful "$2"
+			shift
+			;;
 		-h|--help)
 			usage
 			;;
@@ -805,11 +815,13 @@ _add_bad() {
 
 # Print the aggregate results of the current loop once all loop_total runs
 # have been recorded, then reset the loop state so that the next test can
-# start.
+# start.  A non-empty @force argument ends the loop before loop_total runs,
+# which is used to stop a while-successful loop at the first failure.
 _try_loop_aggregate() {
 	local test_seq="$1"
+	local force="$2"
 
-	if ((${#loop_status[*]} >= loop_total)); then
+	if [ -n "$force" ] || ((${#loop_status[*]} >= loop_total)); then
 		printf "%s aggregate results across %d runs: " \
 			"$test_seq" "${#loop_status[*]}"
 		awk "BEGIN {
@@ -845,7 +857,12 @@ _stash_test_status() {
 			# any failed run fails the test
 			_add_bad "$test_seq"
 		fi
-		_try_loop_aggregate "$test_seq"
+		# a while-successful loop stops at the first failed run
+		if [ "$test_status" = "fail" ] && [ "$loop_mode" = "while-successful" ]; then
+			_try_loop_aggregate "$test_seq" force
+		else
+			_try_loop_aggregate "$test_seq"
+		fi
 		# the rerun loop handles @bad/@notrun by itself
 		return
 	fi
@@ -865,7 +882,8 @@ _stash_test_status() {
 		notrun+=("$test_seq")
 		;;
 	pass)
-		if [ "$loop_mode" = "always" ] && ((loop_total > 1)); then
+		if { [ "$loop_mode" = "always" ] || [ "$loop_mode" = "while-successful" ]; } \
+		   && ((loop_total > 1)); then
 			# start a rerun loop
 			_stash_loop_files "$test_seq" ".rerun0"
 			loop_status+=("$test_status")
-- 
2.56.0