From mboxrd@z Thu Jan 1 00:00:00 1970 From: Joerg Vehlow Date: Fri, 7 May 2021 12:32:57 +0200 Subject: [LTP] [PATCH v2 1/2] shell: Extend timeout tests In-Reply-To: <20210507103258.232174-1-lkml@jv-coder.de> References: <20210507103258.232174-1-lkml@jv-coder.de> Message-ID: <20210507103258.232174-2-lkml@jv-coder.de> List-Id: MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit To: ltp@lists.linux.it From: Joerg Vehlow - Add test, that verifies, that the timeout is working as expected. - Add test case to check if there are processes left, after a test finished without running into a timeout. - Add execution of timeout03.sh and verification of return code. Signed-off-by: Joerg Vehlow --- lib/newlib_tests/shell/test_timeout.sh | 178 ++++++++++++++++++++++--- lib/newlib_tests/shell/timeout04.sh | 22 +++ 2 files changed, 179 insertions(+), 21 deletions(-) create mode 100755 lib/newlib_tests/shell/timeout04.sh diff --git a/lib/newlib_tests/shell/test_timeout.sh b/lib/newlib_tests/shell/test_timeout.sh index 26477d6f2..06f3acf88 100755 --- a/lib/newlib_tests/shell/test_timeout.sh +++ b/lib/newlib_tests/shell/test_timeout.sh @@ -1,42 +1,178 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (c) 2019 Petr Vorel +# Copyright (c) 2021 Joerg Vehlow PATH="$(dirname $0):$(dirname $0)/../../../testcases/lib/:$PATH" +# Test cases are separated by newlines. +# Every test has the following fields in this order: +# file +# timeout_mul +# use_cat +# max_runtime +# expected_exit_code +# expected passes +# expected failed +# expected broken +# description +# Whole lines can be commented out using "#" DATA=" -timeout01.sh||0 -timeout02.sh||0 -timeout02.sh|foo|2 -timeout02.sh|2|0 -timeout01.sh|2|0 -timeout02.sh|1.1|0 -timeout02.sh|-10|2 -timeout02.sh|-0.1|0 -timeout02.sh|-1.1|2 -timeout02.sh|-10.1|2 +timeout01.sh| |0| |0 +timeout02.sh| |0| |0 +timeout02.sh| foo|0| |2 +timeout02.sh| 2|0| |0 +timeout01.sh| 2|0| |0 +timeout02.sh| 1.1|0| |0 +timeout02.sh| -10|0| |2 +timeout02.sh| -0.1|0| |0 +timeout02.sh| -1.1|0| |2 +timeout02.sh|-10.1|0| |2 +timeout03.sh| |0|12|137| | | |Test kill if test does not terminate by SIGINT +timeout04.sh| |0| | 2|0|0|1|Verify that timeout is enforced +timeout02.sh| 2|1| 2| |1|0|0|Test termination of timeout process " +# Executes a test +# Parameter: +# - test: The test to execute +# - timeout: The timeout multiplicator (optional) +# - use_cat: Pipe the output of the command through cat (optional) +# If this is used, the exit code is NOT returned! +# +# The function returns the following global variables: +# - test_exit: The exit code of the test +# - test_duration: The duration of the test in seconds +# - test_output: The full output of the test +# - test_passed: The number of passed tests parsed from the summary +# - test_failed: The number of failed tests parsed from the summary +# - test_broken: The number of broken tests parsed from the summary +run_test() +{ + local test=$1 + local timeout=$2 + local use_cat=$3 + local tmpfile start end; + + tmpfile=$(mktemp -t ltp_timeout_XXXXXXXX) + start=$(date +%s) + # We have to use set monitor mode (set -m) here. + # In most shells in most cases this creates a + # new process group for the next command. + # A process group is required for the timeout functionality, + # because it sends signals to the whole process group. + set -m + # The use_cat is for testing if any programm using stdout + # is still running, after the test finished. + # cat will wait for its stdin to be closed. + # + # In the pure shell implementation of the timeout handling, + # the sleep process was never killed, when a test finished + # before the timeout. + if [ "$use_cat" = "1" ]; then + LTP_TIMEOUT_MUL=$timeout $test 2>&1 | cat >$tmpfile + else + LTP_TIMEOUT_MUL=$timeout $test 1>$tmpfile 2>&1 + fi + test_exit=$? + set +m + end=$(date +%s) + + test_duration=$((end - start)) + test_output=$(cat $tmpfile) + rm $tmpfile + + eval $(echo "$test_output" | awk ' + BEGIN {sum=0} + $1 == "Summary:" { + sum=1; + } + sum && ( \ + $1 == "passed" \ + || $1 == "failed" \ + || $1 == "broken") { + print "test_" $1 "=" $2 + } + ') +} + echo "Testing timeout in shell API" echo failed=0 -for i in $DATA; do - file=$(echo $i | cut -d'|' -f1) - timeout=$(echo $i | cut -d'|' -f2) - exp_exit=$(echo $i | cut -d'|' -f3) - - echo "=== $file (LTP_TIMEOUT_MUL='$timeout') ===" - LTP_TIMEOUT_MUL=$timeout $file - ret=$? - if [ $ret -ne $exp_exit ]; then - echo "FAILED (exit code: $ret, expected $exp_exit)" - failed=$((failed+1)) +test_nr=0 + +old_ifs="$IFS" +IFS=$(printf "\n\b") + +# Remove comments and empty lines +CLEANED_DATA=$(echo "$DATA" | sed '/^\s*#/d;/^\s*$/d') +test_max=$(echo "$CLEANED_DATA" | wc -l) +for d in $CLEANED_DATA; do + IFS="$old_ifs" + + file=$(echo $d | cut -d'|' -f1 | xargs) + timeout=$(echo $d | cut -d'|' -f2 | xargs) + use_cat=$(echo $d | cut -d'|' -f3 | xargs) + max_runtime=$(echo $d | cut -d'|' -f4 | xargs) + max_runtime=${max_runtime:--1} + exp_exit=$(echo $d | cut -d'|' -f5 | xargs) + exp_exit=${exp_exit:--1} + exp_passed=$(echo $d | cut -d'|' -f6 | xargs) + exp_passed=${exp_passed:--1} + exp_failed=$(echo $d | cut -d'|' -f7 | xargs) + exp_failed=${exp_failed:--1} + exp_broken=$(echo $d | cut -d'|' -f8 | xargs) + exp_broken=${exp_broken:--1} + description=$(echo $d | cut -d'|' -f9) + + test_nr=$(($test_nr + 1)) + + cur_fails=0 + + if [ -z "$description" ]; then + description="$file (LTP_TIMEOUT_MUL='$timeout')" + fi + + echo "=== $test_nr/$test_max $description ===" + run_test "$file" "$timeout" "$use_cat" + + if [ $max_runtime -ne -1 ] && [ $test_duration -gt $max_runtime ]; then + echo "FAILED (runtime: $test_duration, expected less than $max_runtime)" + cur_fails=$((cur_fails + 1)) + fi + + if [ $exp_passed -ne -1 ] && [ $exp_passed -ne $test_passed ]; then + echo "FAILED (passes: $test_passed, expected $exp_passed)" + cur_fails=$((cur_fails + 1)) + fi + + if [ $exp_failed -ne -1 ] && [ $exp_failed -ne $test_failed ]; then + echo "FAILED (failed: $test_failed, expected $exp_failed)" + cur_fails=$((cur_fails + 1)) + fi + + if [ $exp_broken -ne -1 ] && [ $exp_broken -ne $test_broken ]; then + echo "FAILED (broken: $test_broken, expected $exp_broken)" + cur_fails=$((cur_fails + 1)) + fi + + if [ $exp_exit -ne -1 ] && [ $test_exit -ne $exp_exit ]; then + echo "FAILED (exit code: $test_exit, expected $exp_exit)" + cur_fails=$((cur_fails + 1)) + fi + + if [ $cur_fails -gt 0 ]; then + failed=$((failed + 1)) + echo "--------" + echo "$test_output" + echo "--------" else echo "PASSED" fi echo done +IFS="$old_ifs" echo "Failed tests: $failed" exit $failed diff --git a/lib/newlib_tests/shell/timeout04.sh b/lib/newlib_tests/shell/timeout04.sh new file mode 100755 index 000000000..0a6ba053c --- /dev/null +++ b/lib/newlib_tests/shell/timeout04.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (c) 2021 Joerg Vehlow + +TST_TESTFUNC=do_test + +TST_TIMEOUT=1 +. tst_test.sh + +do_test() +{ + tst_res TINFO "Start" + sleep 5 + tst_res TFAIL "End" +} + +do_cleanup() +{ + tst_res TINFO "cleanup" +} + +tst_run -- 2.25.1