[kvm-unit-tests PATCH v2 0/6] Generate standalone tests

[kvm-unit-tests PATCH v2 0/6] Generate standalone tests

[Andrew Jones]

Add support to convert unit tests to standalone scripts that
can be run outside the framework. This is almost an RFC, but
it doesn't impact the current framework (except for 'make install',
but was that ever used?). The scripting is ugly, but I see value
in having easily distributable unit tests.

Testing: if you run all standalone tests, concatenating all output
to a file, then that file will match test.log after running
run_tests.sh. Additionally, all prechecks are preserved, i.e.
specific arch and 'check' conditions from unittests.cfg.

Thanks,
drew

v2:
- dropped several changes to run_tests.sh [drew]
- pass DRYRUN through env, not config [drew]
- share "run_all", rather than duplicating it [drew]
- rewrote the script, improving it, and reducing ugliness
  (well, a bit) [drew]

Andrew Jones (6):
  run_tests.sh: share run_all as for_each_unittest
  run: check DRYRUN
  scripts: Introduce mkstandalone.sh
  arm/unittests.cfg: changes to be more standalone friendly
  Makefile: change 'make install' to install standalone tests
  standalone: add documentation to README

 .gitignore                   |   1 +
 Makefile                     |  10 +++-
 README                       |  10 ++++
 arm/run                      |  12 ++--
 arm/unittests.cfg            |  10 ++--
 config/config-arm-common.mak |   2 -
 config/config-x86-common.mak |   2 -
 run_tests.sh                 |  46 +--------------
 scripts/functions.bash       |  42 ++++++++++++++
 scripts/mkstandalone.sh      | 131 +++++++++++++++++++++++++++++++++++++++++++
 x86/run                      |  11 ++--
 11 files changed, 212 insertions(+), 65 deletions(-)
 create mode 100644 scripts/functions.bash
 create mode 100755 scripts/mkstandalone.sh

-- 
2.4.3

[kvm-unit-tests PATCH v2 1/6] run_tests.sh: share run_all as for_each_unittest

[Andrew Jones]

To avoid duplicating unittests.cfg parsing code in other scripts,
let's put it in a file where it can be shared.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 run_tests.sh           | 46 ++--------------------------------------------
 scripts/functions.bash | 42 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 44 deletions(-)
 create mode 100644 scripts/functions.bash

diff --git a/run_tests.sh b/run_tests.sh
index e48f1db049f81..ebb7e9fe6fdfc 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -5,6 +5,7 @@ if [ ! -f config.mak ]; then
     exit
 fi
 source config.mak
+source scripts/functions.bash
 
 config=$TEST_DIR/unittests.cfg
 qemu=${QEMU:-qemu-system-$ARCH}
@@ -61,49 +62,6 @@ function run()
     fi
 }
 
-function run_all()
-{
-    local config="$1"
-    local testname
-    local smp
-    local kernel
-    local opts
-    local groups
-    local arch
-    local check
-
-    exec {config_fd}<$config
-
-    while read -u $config_fd line; do
-        if [[ "$line" =~ ^\[(.*)\]$ ]]; then
-            run "$testname" "$groups" "$smp" "$kernel" "$opts" "$arch" "$check"
-            testname=${BASH_REMATCH[1]}
-            smp=1
-            kernel=""
-            opts=""
-            groups=""
-            arch=""
-            check=""
-        elif [[ $line =~ ^file\ *=\ *(.*)$ ]]; then
-            kernel=$TEST_DIR/${BASH_REMATCH[1]}
-        elif [[ $line =~ ^smp\ *=\ *(.*)$ ]]; then
-            smp=${BASH_REMATCH[1]}
-        elif [[ $line =~ ^extra_params\ *=\ *(.*)$ ]]; then
-            opts=${BASH_REMATCH[1]}
-        elif [[ $line =~ ^groups\ *=\ *(.*)$ ]]; then
-            groups=${BASH_REMATCH[1]}
-        elif [[ $line =~ ^arch\ *=\ *(.*)$ ]]; then
-            arch=${BASH_REMATCH[1]}
-        elif [[ $line =~ ^check\ *=\ *(.*)$ ]]; then
-            check=${BASH_REMATCH[1]}
-        fi
-    done
-
-    run "$testname" "$groups" "$smp" "$kernel" "$opts" "$arch" "$check"
-
-    exec {config_fd}<&-
-}
-
 function usage()
 {
 cat <<EOF
@@ -139,4 +97,4 @@ while getopts "g:hv" opt; do
     esac
 done
 
-run_all $config
+for_each_unittest $config run
diff --git a/scripts/functions.bash b/scripts/functions.bash
new file mode 100644
index 0000000000000..7ed5a517250bc
--- /dev/null
+++ b/scripts/functions.bash
@@ -0,0 +1,42 @@
+
+function for_each_unittest()
+{
+	local unittests="$1"
+	local cmd="$2"
+	local testname
+	local smp
+	local kernel
+	local opts
+	local groups
+	local arch
+	local check
+
+	exec {fd}<"$unittests"
+
+	while read -u $fd line; do
+		if [[ "$line" =~ ^\[(.*)\]$ ]]; then
+			"$cmd" "$testname" "$groups" "$smp" "$kernel" "$opts" "$arch" "$check"
+			testname=${BASH_REMATCH[1]}
+			smp=1
+			kernel=""
+			opts=""
+			groups=""
+			arch=""
+			check=""
+		elif [[ $line =~ ^file\ *=\ *(.*)$ ]]; then
+			kernel=$TEST_DIR/${BASH_REMATCH[1]}
+		elif [[ $line =~ ^smp\ *=\ *(.*)$ ]]; then
+			smp=${BASH_REMATCH[1]}
+		elif [[ $line =~ ^extra_params\ *=\ *(.*)$ ]]; then
+			opts=${BASH_REMATCH[1]}
+		elif [[ $line =~ ^groups\ *=\ *(.*)$ ]]; then
+			groups=${BASH_REMATCH[1]}
+		elif [[ $line =~ ^arch\ *=\ *(.*)$ ]]; then
+			arch=${BASH_REMATCH[1]}
+		elif [[ $line =~ ^check\ *=\ *(.*)$ ]]; then
+			check=${BASH_REMATCH[1]}
+		fi
+	done
+	"$cmd" "$testname" "$groups" "$smp" "$kernel" "$opts" "$arch" "$check"
+	exec {fd}<&-
+}
-- 
2.4.3

[kvm-unit-tests PATCH v2 2/6] run: check DRYRUN

[Andrew Jones]

Only execute the test if DRYRUN isn't set to yes. This is used
by mkstandalone.sh

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 arm/run | 12 +++++++-----
 x86/run | 11 +++++++----
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/arm/run b/arm/run
index 662a8564674a3..8cc2fa2571967 100755
--- a/arm/run
+++ b/arm/run
@@ -44,9 +44,11 @@ fi
 
 command="$qemu $M -cpu $processor $chr_testdev"
 command+=" -display none -serial stdio -kernel"
-
 echo $command "$@"
-$command "$@"
-ret=$?
-echo Return value from qemu: $ret
-exit $ret
+
+if [ "$DRYRUN" != "yes" ]; then
+	$command "$@"
+	ret=$?
+	echo Return value from qemu: $ret
+	exit $ret
+fi
diff --git a/x86/run b/x86/run
index 5281fca2125d8..b4a35b2332846 100755
--- a/x86/run
+++ b/x86/run
@@ -44,7 +44,10 @@ fi
 
 command="${qemu} -enable-kvm $pc_testdev -vnc none -serial stdio $pci_testdev -kernel"
 echo ${command} "$@"
-${command} "$@"
-ret=$?
-echo Return value from qemu: $ret
-exit $ret
+
+if [ "$DRYRUN" != "yes" ]; then
+	${command} "$@"
+	ret=$?
+	echo Return value from qemu: $ret
+	exit $ret
+fi
-- 
2.4.3

[kvm-unit-tests PATCH v2 3/6] scripts: Introduce mkstandalone.sh

[Andrew Jones]

This is a pretty ugly bash script, wrapped around a couple ugly
bash scripts, and it generates an ugly sh script. But, it gets the
job done. What's the job? Take a unit test and generate a standalone
script that can be run on any appropriate system with an appropriate
qemu. This makes distributing single, pre-compiled, unit tests easy,
even through email, which can be useful for getting test results from
a variety of users, or even for deploying the unit test as utility
(if it works that way) in a distribution. "Packaging" works like shar,
but doesn't use shar, as it's better to avoid the dependency. Can be
used to create just one unit test, or all of them ('make standalone').
The standalone test(s) are placed in ./tests.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 .gitignore              |   1 +
 Makefile                |   4 ++
 scripts/mkstandalone.sh | 131 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 136 insertions(+)
 create mode 100755 scripts/mkstandalone.sh

diff --git a/.gitignore b/.gitignore
index e21939a8771e9..242fae475094c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ cscope.*
 /*-run
 /test.log
 /msr.out
+/tests
diff --git a/Makefile b/Makefile
index 4f28f072ae3d7..e40735f45b131 100644
--- a/Makefile
+++ b/Makefile
@@ -64,6 +64,9 @@ $(LIBFDT_archive): $(addprefix $(LIBFDT_objdir)/,$(LIBFDT_OBJS))
 
 -include */.*.d */*/.*.d
 
+standalone: all
+	@scripts/mkstandalone.sh
+
 install:
 	mkdir -p $(DESTDIR)
 	install $(tests_and_config) $(DESTDIR)
@@ -78,6 +81,7 @@ libfdt_clean:
 
 distclean: clean libfdt_clean
 	$(RM) lib/asm config.mak $(TEST_DIR)-run test.log msr.out cscope.*
+	$(RM) -r tests
 
 cscope: common_dirs = lib lib/libfdt lib/asm lib/asm-generic
 cscope:
diff --git a/scripts/mkstandalone.sh b/scripts/mkstandalone.sh
new file mode 100755
index 0000000000000..4cf346ab87d24
--- /dev/null
+++ b/scripts/mkstandalone.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+
+if [ ! -f config.mak ]; then
+	echo "run ./configure && make first. See ./configure -h"
+	exit
+fi
+source config.mak
+source scripts/functions.bash
+
+one_kernel="$1"
+[ "$one_kernel" ] && one_kernel_base=$(basename $one_kernel)
+one_testname="$2"
+if [ -n "$one_kernel" ] && [ ! -f $one_kernel ]; then
+	echo "$one_kernel doesn't exist"
+	exit 1
+elif [ -n "$one_kernel" ] && [ -z "$one_testname" ]; then
+	one_testname="${one_kernel_base%.*}"
+fi
+
+unittests=$TEST_DIR/unittests.cfg
+mkdir -p tests
+
+function mkstandalone()
+{
+	local testname="$1"
+	local groups="$2"
+	local smp="$3"
+	local kernel="$4"
+	local opts="$5"
+	local arch="$6"
+	local check="$7"
+
+	if [ -z "$testname" ]; then
+		return 1
+	fi
+
+	if [ -n "$one_testname" ] && [ "$testname" != "$one_testname" ]; then
+		return 1
+	fi
+
+	standalone=tests/$testname
+	cmdline=$(DRYRUN=yes ./$TEST_DIR-run $kernel)
+	if [ $? -ne 0 ]; then
+		echo $cmdline
+		exit 1
+	fi
+	qemu=$(cut -d' ' -f1 <<< "$cmdline")
+	cmdline=$(cut -d' ' -f2- <<< "$cmdline")
+
+	cat <<EOF > $standalone
+#!/bin/sh
+
+EOF
+if [ "$arch" ]; then
+	cat <<EOF >> $standalone
+ARCH=\`uname -m | sed -e s/i.86/i386/ | sed -e 's/arm.*/arm/'\`
+[ "\$ARCH" = "aarch64" ] && ARCH="arm64"
+if [ "\$ARCH" != "$arch" ]; then
+	echo "skip $testname ($arch only)" 1>&2
+	exit 1
+fi
+
+EOF
+fi
+if [ "$check" ]; then
+	cat <<EOF >> $standalone
+for param in $check; do
+	path=\`echo \$param | cut -d= -f1\`
+	value=\`echo \$param | cut -d= -f2\`
+	if [ -f "\$path" ] && [ "\`cat \$path\`" != "\$value" ]; then
+		echo "skip $testname (\$path not equal to \$value)" 1>&2
+		exit 1
+	fi
+done
+
+EOF
+fi
+if [ ! -f $kernel ]; then
+	cat <<EOF >> $standalone
+echo "skip $testname (test kernel not present)" 1>&2
+exit 1
+EOF
+else
+	cat <<EOF >> $standalone
+trap 'rm -f \$bin; exit 1' HUP INT TERM
+bin=\`mktemp\`
+base64 -d << 'BIN_EOF' | zcat > \$bin &&
+EOF
+gzip - < $kernel | base64 >> $standalone
+	cat <<EOF >> $standalone
+BIN_EOF
+
+qemu="$qemu"
+if [ "\$QEMU" ]; then
+	qemu="\$QEMU"
+fi
+cmdline="\`echo '$cmdline' | sed s%$kernel%\$bin%\`"
+echo \$qemu $cmdline -smp $smp $opts
+\$qemu \$cmdline -smp $smp $opts
+ret=\$?
+echo Return value from qemu: \$ret
+if [ \$ret -le 1 ]; then
+	echo PASS $testname 1>&2
+else
+	echo FAIL $testname 1>&2
+fi
+rm -f \$bin
+exit 0
+EOF
+fi
+chmod +x $standalone
+return 0
+}
+
+trap 'rm -f $cfg; exit 1' HUP INT TERM
+trap 'rm -f $cfg' EXIT
+cfg=$(mktemp)
+
+if [ -n "$one_testname" ]; then
+	if grep -q "\[$one_testname\]" $unittests; then
+		sed -n "/\\[$one_testname\\]/,/^\\[/p" $unittests \
+			| awk '!/^\[/ || NR == 1' > $cfg
+	else
+		echo "[$one_testname]" > $cfg
+		echo "file = $one_kernel_base" >> $cfg
+	fi
+else
+	cp -f $unittests $cfg
+fi
+
+for_each_unittest $cfg mkstandalone
-- 
2.4.3

[kvm-unit-tests PATCH v2 4/6] arm/unittests.cfg: changes to be more standalone friendly

[Andrew Jones]

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 arm/unittests.cfg | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/arm/unittests.cfg b/arm/unittests.cfg
index ee655b2678a4e..e068a0cdd9c1f 100644
--- a/arm/unittests.cfg
+++ b/arm/unittests.cfg
@@ -11,27 +11,27 @@
 # that the configured amount of memory (-m <MB>) are correctly setup
 # by the framework.
 #
-[selftest::setup]
+[selftest-setup]
 file = selftest.flat
 smp = 2
 extra_params = -m 256 -append 'setup smp=2 mem=256'
 groups = selftest
 
 # Test vector setup and exception handling (kernel mode).
-[selftest::vectors-kernel]
+[selftest-vectors-kernel]
 file = selftest.flat
 extra_params = -append 'vectors-kernel'
 groups = selftest
 
 # Test vector setup and exception handling (user mode).
-[selftest::vectors-user]
+[selftest-vectors-user]
 file = selftest.flat
 extra_params = -append 'vectors-user'
 groups = selftest
 
 # Test SMP support
-[selftest::smp]
+[selftest-smp]
 file = selftest.flat
-smp = $(getconf _NPROCESSORS_CONF)
+smp = `getconf _NPROCESSORS_CONF`
 extra_params = -append 'smp'
 groups = selftest
-- 
2.4.3

[kvm-unit-tests PATCH v2 5/6] Makefile: change 'make install' to install standalone tests

[Andrew Jones]

make install wasn't very useful without also installing scripts for
the qemu command lines. I suspect 'make install' was never used.
Installing standalone tests could be useful though, so let's do
that instead.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 Makefile                     | 6 +++---
 config/config-arm-common.mak | 2 --
 config/config-x86-common.mak | 2 --
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index e40735f45b131..0d5933474cd8c 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ endif
 
 include config.mak
 
-DESTDIR := $(PREFIX)/share/qemu/tests
+DESTDIR := $(PREFIX)/share/kvm-unit-tests/
 
 .PHONY: arch_clean clean distclean cscope
 
@@ -67,9 +67,9 @@ $(LIBFDT_archive): $(addprefix $(LIBFDT_objdir)/,$(LIBFDT_OBJS))
 standalone: all
 	@scripts/mkstandalone.sh
 
-install:
+install: standalone
 	mkdir -p $(DESTDIR)
-	install $(tests_and_config) $(DESTDIR)
+	install tests/* $(DESTDIR)
 
 clean: arch_clean
 	$(RM) lib/.*.d $(libcflat) $(cflatobjs)
diff --git a/config/config-arm-common.mak b/config/config-arm-common.mak
index 0674daaa476d7..698555d6a676f 100644
--- a/config/config-arm-common.mak
+++ b/config/config-arm-common.mak
@@ -64,8 +64,6 @@ arm_clean: libfdt_clean asm_offsets_clean
 
 ##################################################################
 
-tests_and_config = $(TEST_DIR)/*.flat $(TEST_DIR)/unittests.cfg
-
 generated_files = $(asm-offsets)
 
 test_cases: $(generated_files) $(tests-common) $(tests)
diff --git a/config/config-x86-common.mak b/config/config-x86-common.mak
index d45c9a8e454d8..2ee7b7724bbaf 100644
--- a/config/config-x86-common.mak
+++ b/config/config-x86-common.mak
@@ -43,8 +43,6 @@ tests-common += api/dirty-log
 tests-common += api/dirty-log-perf
 endif
 
-tests_and_config = $(TEST_DIR)/*.flat $(TEST_DIR)/unittests.cfg
-
 test_cases: $(tests-common) $(tests)
 
 $(TEST_DIR)/%.o: CFLAGS += -std=gnu99 -ffreestanding -I lib -I lib/x86
-- 
2.4.3

[kvm-unit-tests PATCH v2 6/6] standalone: add documentation to README

[Andrew Jones]

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 README | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README b/README
index e9869d12bfa20..eab5ea28f7fab 100644
--- a/README
+++ b/README
@@ -16,6 +16,16 @@ To select a specific qemu binary, specify the QEMU=<path>
 environment variable, e.g.
   QEMU=/tmp/qemu/x86_64-softmmu/qemu-system-x86_64 ./x86-run ./x86/msr.flat
 
+To create and use standalone tests do
+  ./configure
+  make standalone
+  (send tests/some-test somewhere)
+  (go to somewhere)
+  ./some-test
+
+'make install' will install all tests in PREFIX/share/kvm-unit-tests/tests,
+each as a standalone test.
+
 Directory structure:
 .:		configure script, top-level Makefile, and run_tests.sh
 ./config:	collection of architecture dependent makefiles
-- 
2.4.3

Re: [kvm-unit-tests PATCH v2 0/6] Generate standalone tests

[Paolo Bonzini]

On 13/07/2015 19:02, Andrew Jones wrote:
> Add support to convert unit tests to standalone scripts that
> can be run outside the framework. This is almost an RFC, but
> it doesn't impact the current framework (except for 'make install',
> but was that ever used?). The scripting is ugly, but I see value
> in having easily distributable unit tests.
> 
> Testing: if you run all standalone tests, concatenating all output
> to a file, then that file will match test.log after running
> run_tests.sh. Additionally, all prechecks are preserved, i.e.
> specific arch and 'check' conditions from unittests.cfg.

This is pretty wicked, but it's indeed useful.  Applied. :)

Paolo

> Thanks,
> drew
> 
> v2:
> - dropped several changes to run_tests.sh [drew]
> - pass DRYRUN through env, not config [drew]
> - share "run_all", rather than duplicating it [drew]
> - rewrote the script, improving it, and reducing ugliness
>   (well, a bit) [drew]
> 
> Andrew Jones (6):
>   run_tests.sh: share run_all as for_each_unittest
>   run: check DRYRUN
>   scripts: Introduce mkstandalone.sh
>   arm/unittests.cfg: changes to be more standalone friendly
>   Makefile: change 'make install' to install standalone tests
>   standalone: add documentation to README
> 
>  .gitignore                   |   1 +
>  Makefile                     |  10 +++-
>  README                       |  10 ++++
>  arm/run                      |  12 ++--
>  arm/unittests.cfg            |  10 ++--
>  config/config-arm-common.mak |   2 -
>  config/config-x86-common.mak |   2 -
>  run_tests.sh                 |  46 +--------------
>  scripts/functions.bash       |  42 ++++++++++++++
>  scripts/mkstandalone.sh      | 131 +++++++++++++++++++++++++++++++++++++++++++
>  x86/run                      |  11 ++--
>  11 files changed, 212 insertions(+), 65 deletions(-)
>  create mode 100644 scripts/functions.bash
>  create mode 100755 scripts/mkstandalone.sh
>