[kvm-unit-tests PATCH 1/3] run_tests.sh: reduce return code ambiguity

[Andrew Jones <drjones@redhat.com>]

qemu/unittest exit codes are convoluted, causing codes 0 and 1
to be ambiguous. Here are the possible meanings

 .-----------------------------------------------------------------.
 |                | 0                                 | 1          |
 |-----------------------------------------------------------------|
 | QEMU           | did something successfully,       | FAILURE    |
 |                | but probably didn't run the       |            |
 |                | unittest, OR caught SIGINT,       |            |
 |                | SIGHUP, or SIGTERM                |            |
 |-----------------------------------------------------------------|
 | unittest       | for some reason exited using      | SUCCESS    |
 |                | ACPI/PSCI, not with debug-exit    |            |
 .-----------------------------------------------------------------.

As we can see above, an exit code of zero is even ambiguous for each
row, i.e. QEMU could exit with zero because it successfully completed,
or because it caught a signal. unittest could exit with zero because
it successfully powered-off, or because for some odd reason it powered-
off instead of calling debug-exit.

And, the most fun is that exit-code == 1 means QEMU failed, but the
unittest succeeded.

This patch attempts to reduce that ambiguity, by also looking at stderr.
With it, we have

 0      - unexpected exit from qemu, or the unittest not using debug-exit.
          Consider it a FAILURE
 1      - unittest SUCCESS
 < 128  - something failed (could be the unittest, qemu, or a run script)
          Check the logs.
 >= 128 - signal (signum = code - 128)

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 run_tests.sh            | 27 +++++++++++++++++++++++++--
 scripts/mkstandalone.sh | 27 ++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index fad22a935b007..f8de08cfb21b5 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -21,6 +21,7 @@ function run()
     local arch="$6"
     local check="$7"
     local accel="$8"
+    local errlog sig ret
 
     if [ -z "$testname" ]; then
         return
@@ -54,10 +55,32 @@ function run()
 
     # extra_params in the config file may contain backticks that need to be
     # expanded, so use eval to start qemu
-    eval $cmdline >> test.log
+    errlog=$(mktemp)
+    eval $cmdline >> test.log 2> $errlog
+    ret=$?
+
+    if [ "$(stat -c %s $errlog)" != "0" ]; then
+        # Some signals result in a zero return code, but the error log
+        # tells the truth.
+        sig="$(grep 'terminating on signal' $errlog | sed 's/.*terminating on signal \([0-9][0-9]*\).*/\1/')"
+        if [ $ret -eq 0 ] && [ "$sig" ]; then
+            ((ret=sig+128))
+        elif [ $ret -eq 1 ]; then
+            # We got the unittest SUCCESS code, but also error messages,
+            # let's assume qemu failed.
+            ret=2
+        fi
+        cat $errlog >> test.log
+    fi
+    rm -f $errlog
 
-    if [ $? -le 1 ]; then
+    if [ $ret -eq 0 ]; then
+        echo -e "\e[31mFAIL\e[0m $1 (debug-exit not called)"
+    elif [ $ret -eq 1 ]; then
         echo -e "\e[32mPASS\e[0m $1"
+    elif [ $ret -ge 128 ]; then
+        ((sig=ret-128))
+        echo -e "\e[31mFAIL\e[0m $1 (got signal $sig)"
     else
         echo -e "\e[31mFAIL\e[0m $1"
     fi
diff --git a/scripts/mkstandalone.sh b/scripts/mkstandalone.sh
index 3ce244aff67b9..94ea0467c5be6 100755
--- a/scripts/mkstandalone.sh
+++ b/scripts/mkstandalone.sh
@@ -83,8 +83,9 @@ exit 1
 EOF
 else
 	cat <<EOF >> $standalone
-trap 'rm -f \$bin; exit 1' HUP INT TERM
+trap 'rm -f \$bin \$errlog; exit 1' HUP INT TERM
 bin=\`mktemp\`
+errlog=\`mktemp\`
 base64 -d << 'BIN_EOF' | zcat > \$bin &&
 EOF
 gzip - < $kernel | base64 >> $standalone
@@ -109,16 +110,32 @@ else
 	done
 
 	cmdline="\`echo '$cmdline' | sed s%$kernel%\$bin%\`"
-	\$qemu \$cmdline -smp $smp $opts
+	\$qemu \$cmdline -smp $smp $opts 2> \$errlog
 	ret=\$?
+	echo Return value from qemu: \$ret
+
+	if [ "\`stat -c %s \$errlog\`" != "0" ]; then
+		sig="\`grep 'terminating on signal' \$errlog | sed 's/.*terminating on signal \([0-9][0-9]*\).*/\1/'\`"
+		if [ \$ret -eq 0 ] && [ "\$sig" ]; then
+			ret=\`expr \$sig + 128\`
+		elif [ \$ret -eq 1 ]; then
+			ret=2
+		fi
+		cat \$errlog
+	fi
 fi
-echo Return value from qemu: \$ret
-if [ \$ret -le 1 ]; then
+
+if [ \$ret -eq 0 ]; then
+	echo "FAIL $testname (debug-exit not called)" 1>&2
+elif [ \$ret -eq 1 ]; then
 	echo PASS $testname 1>&2
+elif [ \$ret -ge 128 ]; then
+	echo "FAIL $testname (got signal \`expr \$ret - 128\`)" 1>&2
 else
 	echo FAIL $testname 1>&2
 fi
-rm -f \$bin
+
+rm -f \$bin \$errlog
 exit 0
 EOF
 fi
-- 
2.4.3