[PATCH] test-lib: save test counts across invocations

[Thomas Rast <trast@student.ethz.ch>]

Under 'prove' we can get progress status for tests running in
parallel.  However we never told it the total number of tests in a
file in advance, and thus the progress only showed the tests already
executed.

Save the number of tests run ($test_count) in a file under
test-counts/.  Then when sourcing test-lib.sh the next time, compare
the timestamps.  If the counts file is older than the test, discard.
Otherwise use the count that we saved and give prove the test plan
("1..N") up front.

This results in 'make prove' giving progress output like

  ===(    2884;121   8/12   24/147  1/8  1/3  )============================

if you have already run the tests before.

Prerequisite changes can mean that a whole test group is skipped
(e.g. NO_SVN_TESTS=1).  We thus need to be somewhat careful to only
emit the "full" plan once we know we're not going to $skip_all.

t9700 needs special treatment on top of $test_external_has_tap because
the latter can only be set once we know that the external test will
run.  If a prerequisite fails, we still need to emit the plan.

The Makefile changes are required because we want to keep that
test-count subdirectory unless the *user* invokes 'make clean', but we
previously ran the latter ourselves after every successful test run.

Signed-off-by: Thomas Rast <trast@student.ethz.ch>
---

Sparked by a discussion on G+.  I think this is the "simple" approach.
The "cute" approach would be to let test-lib.sh define test_* as
test-counting dummies once, source the test script itself (avoiding
the sourcing loop with test-lib) to count what it does, then do the
real work.


 t/.gitignore        |    1 +
 t/Makefile          |    9 ++++++---
 t/t9700-perl-git.sh |    1 +
 t/test-lib.sh       |   27 +++++++++++++++++++++++++--
 4 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/t/.gitignore b/t/.gitignore
index 4e731dc..7de845f 100644
--- a/t/.gitignore
+++ b/t/.gitignore
@@ -1,3 +1,4 @@
 /trash directory*
 /test-results
+/test-counts
 /.prove
diff --git a/t/Makefile b/t/Makefile
index 9046ec9..c70de07 100644
--- a/t/Makefile
+++ b/t/Makefile
@@ -36,11 +36,14 @@ $(T):
 pre-clean:
 	$(RM) -r test-results
 
-clean:
+post-clean:
 	$(RM) -r 'trash directory'.* test-results
 	$(RM) -r valgrind/bin
 	$(RM) .prove
 
+clean: post-clean
+	$(RM) -r test-counts
+
 test-lint: test-lint-duplicates test-lint-executable
 
 test-lint-duplicates:
@@ -55,7 +58,7 @@ test-lint-executable:
 
 aggregate-results-and-cleanup: $(T)
 	$(MAKE) aggregate-results
-	$(MAKE) clean
+	$(MAKE) post-clean
 
 aggregate-results:
 	for f in test-results/t*-*.counts; do \
@@ -111,4 +114,4 @@ smoke_report: smoke
 		http://smoke.git.nix.is/app/projects/process_add_report/1 \
 	| grep -v ^Redirecting
 
-.PHONY: pre-clean $(T) aggregate-results clean valgrind smoke smoke_report
+.PHONY: pre-clean $(T) aggregate-results clean valgrind smoke smoke_report post-clean
diff --git a/t/t9700-perl-git.sh b/t/t9700-perl-git.sh
index 3787186..20ec097 100755
--- a/t/t9700-perl-git.sh
+++ b/t/t9700-perl-git.sh
@@ -4,6 +4,7 @@
 #
 
 test_description='perl interface (Git.pm)'
+test_disable_saved_count=1
 . ./test-lib.sh
 
 if ! test_have_prereq PERL; then
diff --git a/t/test-lib.sh b/t/test-lib.sh
index bdd9513..374cdb2 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -522,11 +522,19 @@ test_skip () {
 	esac
 }
 
+test_emit_plan () {
+	if [ -z "$test_plan_emitted" -a -n "$test_count_saved" ]; then
+		say "1..$test_count_saved"
+		test_plan_emitted=y
+	fi
+}
+
 test_expect_failure () {
 	test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
 	test "$#" = 2 ||
 	error "bug in the test script: not 2 or 3 parameters to test-expect-failure"
 	export test_prereq
+	test_emit_plan
 	if ! test_skip "$@"
 	then
 		say >&3 "checking known breakage: $2"
@@ -545,6 +553,7 @@ test_expect_success () {
 	test "$#" = 2 ||
 	error "bug in the test script: not 2 or 3 parameters to test-expect-success"
 	export test_prereq
+	test_emit_plan
 	if ! test_skip "$@"
 	then
 		say >&3 "expecting success: $2"
@@ -860,12 +869,14 @@ test_done () {
 	fi
 	case "$test_failure" in
 	0)
+		[ -z "$skip_all" ] && echo "$test_count" > "$test_count_file"
+
 		# Maybe print SKIP message
 		[ -z "$skip_all" ] || skip_all=" # SKIP $skip_all"
 
 		if test $test_external_has_tap -eq 0; then
 			say_color pass "# passed all $msg"
-			say "1..$test_count$skip_all"
+			[ -z "$test_plan_emitted" ] && say "1..$test_count$skip_all"
 		fi
 
 		test -d "$remove_trash" &&
@@ -877,7 +888,7 @@ test_done () {
 	*)
 		if test $test_external_has_tap -eq 0; then
 			say_color error "# failed $test_failure among $msg"
-			say "1..$test_count"
+			[ -z "$test_plan_emitted" ] && say "1..$test_count"
 		fi
 
 		exit 1 ;;
@@ -896,6 +907,18 @@ then
 fi
 GIT_BUILD_DIR="$TEST_DIRECTORY"/..
 
+mkdir -p "$TEST_DIRECTORY"/test-counts
+test_count_file="$TEST_DIRECTORY"/test-counts/$(basename "$0" .sh)
+test_count_saved=$(
+	if [ -n "$test_disable_saved_count" ]; then
+		:
+	# the saved count is only valid if the file is newer than the test
+	elif [ -f "$test_count_file" -a "$test_count_file" -nt "$0" ]; then
+		cat "$test_count_file" 2>/dev/null
+	fi
+	# otherwise we leave the variable empty
+)
+
 if test -n "$valgrind"
 then
 	make_symlink () {
-- 
1.7.7.rc0.420.g468b7