[PATCH nft v3 11/11] tests/shell: support running tests in parallel

[Thomas Haller <thaller@redhat.com>]

Add option to enable running jobs in parallel. The purpose is to speed
up the run time of the tests.

The global cleanup (removal of kernel modules) interferes with parallel
jobs (or even with, unrelated jobs on the system). By setting
NFT_TEST_JOBS= to a positive number, that cleanup is skipped.

Signed-off-by: Thomas Haller <thaller@redhat.com>
---
 tests/shell/run-tests.sh | 83 +++++++++++++++++++++++++++++++++-------
 1 file changed, 69 insertions(+), 14 deletions(-)

diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh
index 1451d913bd3a..160524e889ed 100755
--- a/tests/shell/run-tests.sh
+++ b/tests/shell/run-tests.sh
@@ -41,6 +41,7 @@ usage() {
 	echo " -R|--without-realroot : sets NFT_TEST_HAVE_REALROOT=n"
 	echo " -U|--no-unshare  : sets NFT_TEST_NO_UNSHARE=y"
 	echo " -k|--keep-logs   : sets NFT_TEST_KEEP_LOGS=y"
+	echo " -j|--jobs        : sets NFT_TEST_JOBS=12"
 	echo
 	echo "ENVIRONMENT VARIABLES:"
 	echo " NFT=<PATH>   : Path to nft executable"
@@ -69,6 +70,10 @@ usage() {
 	echo "                Set to empty to not unshare. You may want to export NFT_TEST_IS_UNSHARED="
 	echo "                and NFT_TEST_HAVE_REALROOT= accordingly."
 	echo " NFT_TEST_KEEP_LOGS=*|y: Keep the temp directory. On success, it will be deleted by default."
+	echo " NFT_TEST_JOBS=<NUM}>: by default, run test sequentially. Set to an integer > 1 to"
+	echo "                run jobs in parallel. Leaving this unset or at zero means to run jobs sequentially"
+	echo "                and perform global cleanups between tests (remove kernel modules). Setting this"
+	echo "                to a positive number (including \"1\") means to disable such global cleanups."
 }
 
 NFT_TEST_BASEDIR="$(dirname "$0")"
@@ -83,6 +88,7 @@ KMEMLEAK="$KMEMLEAK"
 NFT_TEST_KEEP_LOGS="$NFT_TEST_KEEP_LOGS"
 NFT_TEST_HAVE_REALROOT="$NFT_TEST_HAVE_REALROOT"
 NFT_TEST_NO_UNSHARE="$NFT_TEST_NO_UNSHARE"
+NFT_TEST_JOBS="$NFT_TEST_JOBS"
 DO_LIST_TESTS=
 
 TESTS=()
@@ -119,6 +125,9 @@ while [ $# -gt 0 ] ; do
 		-U|--no-unshare)
 			NFT_TEST_NO_UNSHARE=y
 			;;
+		-j|--jobs)
+			NFT_TEST_JOBS=12
+			;;
 		--)
 			TESTS+=( "$@" )
 			shift $#
@@ -132,6 +141,11 @@ while [ $# -gt 0 ] ; do
 	esac
 done
 
+# normalize the jobs number to be an integer.
+case "$NFT_TEST_JOBS" in
+	''|*[!0-9]*) NFT_TEST_JOBS=0 ;;
+esac
+
 find_tests() {
 	find "$1" -type f -executable | sort
 }
@@ -199,13 +213,15 @@ ${NFT} > /dev/null 2>&1
 ret=$?
 if [ ${ret} -eq 126 ] || [ ${ret} -eq 127 ]; then
 	msg_error "cannot execute nft command: ${NFT}"
-else
-	msg_info "using nft command: ${NFT}"
 fi
+msg_info "using nft command: ${NFT}"
+msg_info "parallel job mode: NFT_TEST_JOBS=$NFT_TEST_JOBS"
 
-MODPROBE="$(which modprobe)"
-if [ ! -x "$MODPROBE" ] ; then
-	msg_error "no modprobe binary found"
+if [ "$NFT_TEST_JOBS" -eq 0 ] ; then
+	MODPROBE="$(which modprobe)"
+	if [ ! -x "$MODPROBE" ] ; then
+		msg_error "no modprobe binary found"
+	fi
 fi
 
 DIFF="$(which diff)"
@@ -231,6 +247,11 @@ export NFT_TEST_TMPDIR
 
 
 kernel_cleanup() {
+	if [ "$NFT_TEST_JOBS" -ne 0 ] ; then
+		# When we run jobs in parallel (even with only one "parallel"
+		# job via `NFT_TEST_JOBS=1`), we skip such global cleanups.
+		return
+	fi
 	if [ "$NFT_TEST_IS_UNSHARED" != y ] ; then
 		$NFT flush ruleset
 	fi
@@ -403,23 +424,57 @@ print_test_result() {
 	fi
 }
 
-for testfile in "${TESTS[@]}" ; do
-	kernel_cleanup
+declare -A JOBS_TEMPDIR
+declare -A JOBS_PIDLIST
 
-	# We also create and export a test-specific temporary directory.
-	NFT_TEST_TESTTMPDIR="$(mktemp -p "$NFT_TEST_TMPDIR" -d "test-${testfile//\//-}.XXXXXX")"
-	export NFT_TEST_TESTTMPDIR
+job_start() {
+	local testfile="$1"
 
-	print_test_header I "$testfile" "EXECUTING" ""
+	if [ "$NFT_TEST_JOBS" -le 1 ] ; then
+		print_test_header I "$testfile" "EXECUTING" ""
+	fi
+
+	NFT_TEST_TESTTMPDIR="${JOBS_TEMPDIR["$testfile"]}" \
 	NFT="$NFT" DIFF="$DIFF" DUMPGEN="$DUMPGEN" $NFT_TEST_UNSHARE_CMD "$NFT_TEST_BASEDIR/helpers/test-wrapper.sh" "$testfile"
 	rc_got=$?
-	echo -en "\033[1A\033[K" # clean the [EXECUTING] foobar line
 
-	print_test_result "$NFT_TEST_TESTTMPDIR" "$testfile" "$rc_got"
+	if [ "$NFT_TEST_JOBS" -le 1 ] ; then
+		echo -en "\033[1A\033[K" # clean the [EXECUTING] foobar line
+	fi
+
+	return "$rc_got"
+}
+
+job_wait()
+{
+	local num_jobs="$1"
+
+	while [ "$JOBS_N_RUNNING" -gt 0 -a "$JOBS_N_RUNNING" -ge "$num_jobs" ] ; do
+		wait -n -p JOBCOMPLETED
+		rc_got="$?"
+		testfile2="${JOBS_PIDLIST[$JOBCOMPLETED]}"
+		print_test_result "${JOBS_TEMPDIR["$testfile2"]}" "$testfile2" "$rc_got"
+		((JOBS_N_RUNNING--))
+		check_kmemleak
+	done
+}
+
+JOBS_N_RUNNING=0
+for testfile in "${TESTS[@]}" ; do
+	kernel_cleanup
+
+	job_wait "$NFT_TEST_JOBS"
 
-	check_kmemleak
+	NFT_TEST_TESTTMPDIR="$(mktemp -p "$NFT_TEST_TMPDIR" -d "test-${testfile//\//-}.XXXXXX")"
+	JOBS_TEMPDIR["$testfile"]="$NFT_TEST_TESTTMPDIR"
+
+	job_start "$testfile" &
+	JOBS_PIDLIST[$!]="$testfile"
+	((JOBS_N_RUNNING++))
 done
 
+job_wait 0
+
 echo ""
 
 # kmemleak may report suspected leaks
-- 
2.41.0