[PATCH 1/3] ktest: save test output

[PATCH 1/3] ktest: save test output

[Rabin Vincent]

The test output may contain useful information; save it along with the
already-saved buildlog, dmesg, and .config.

Signed-off-by: Rabin Vincent <rabin@rab.in>
---
 tools/testing/ktest/ktest.pl |   30 +++++++++++++++++++-----------
 1 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 30e2bef..3185e0a 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -103,6 +103,7 @@ my $in_patchcheck = 0;
 my $run_test;
 my $redirect;
 my $buildlog;
+my $testlog;
 my $dmesg;
 my $monitor_fp;
 my $monitor_pid;
@@ -1005,17 +1006,19 @@ sub fail {
 	    mkpath($faildir) or
 		die "can't create $faildir";
 	}
-	if (-f "$output_config") {
-	    cp "$output_config", "$faildir/config" or
-		die "failed to copy .config";
-	}
-	if (-f $buildlog) {
-	    cp $buildlog, "$faildir/buildlog" or
-		die "failed to move $buildlog";
-	}
-	if (-f $dmesg) {
-	    cp $dmesg, "$faildir/dmesg" or
-		die "failed to move $dmesg";
+
+	my %files = (
+		"config" => $output_config,
+		"buildlog" => $buildlog,
+		"dmesg" => $dmesg,
+		"testlog" => $testlog,
+	);
+
+	while (my ($name, $source) = each(%files)) {
+		if (-f "$source") {
+			cp "$source", "$faildir/$name" or
+				die "failed to copy $source";
+		}
 	}
 
 	doprint "*** Saved info to $faildir ***\n";
@@ -1653,7 +1656,10 @@ sub child_run_test {
     $poweroff_on_error = 0;
     $die_on_failure = 1;
 
+    $redirect = "$testlog";
     run_command $run_test or $failed = 1;
+    undef $redirect;
+
     exit $failed;
 }
 
@@ -3153,6 +3159,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $target = "$ssh_user\@$machine";
 
     $buildlog = "$tmpdir/buildlog-$machine";
+    $testlog = "$tmpdir/testlog-$machine";
     $dmesg = "$tmpdir/dmesg-$machine";
     $make = "$makecmd O=$outputdir";
     $output_config = "$outputdir/.config";
@@ -3189,6 +3196,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
 
     unlink $dmesg;
     unlink $buildlog;
+    unlink $testlog;
 
     if (defined($addconfig)) {
 	my $min = $minconfig;
-- 
1.7.7.3

[PATCH 2/3] ktest: check parent options for a repeated test

[Rabin Vincent]

Let's say we have "OUTPUT_DIR = build/${TEST_NAME}", and we're iterating
a test.  In the second iteration of a test, the TEST_NAME of the test
we're repeating is not used.  Instead, ${TEST_NAME} appears literally:

   touch /home/rabin/kernel/test/build/${TEST_NAME}/.config ... SUCCESS

Fix this by making __eval_option() check the parent test options
for a repeated test.

Signed-off-by: Rabin Vincent <rabin@rab.in>
---
 tools/testing/ktest/ktest.pl |   16 ++++++++++++++++
 1 files changed, 16 insertions(+), 0 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 3185e0a..e93c21c 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -748,6 +748,18 @@ sub __eval_option {
     # Add space to evaluate the character before $
     $option = " $option";
     my $retval = "";
+    my $repeated = 0;
+    my $parent = 0;
+
+    foreach my $test (keys %repeat_tests) {
+	if ($i >= $test &&
+	    $i < $test + $repeat_tests{$test}) {
+
+	    $repeated = 1;
+	    $parent = $test;
+	    last;
+	}
+    }
 
     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 	my $start = $1;
@@ -761,10 +773,14 @@ sub __eval_option {
 	# otherwise see if the default OPT (without [$i]) exists.
 
 	my $o = "$var\[$i\]";
+	my $parento = "$var\[$parent\]";
 
 	if (defined($opt{$o})) {
 	    $o = $opt{$o};
 	    $retval = "$retval$o";
+	} elsif ($repeated && defined($opt{$parento})) {
+	    $o = $opt{$parento};
+	    $retval = "$retval$o";
 	} elsif (defined($opt{$var})) {
 	    $o = $opt{$var};
 	    $retval = "$retval$o";
-- 
1.7.7.3

Re: [PATCH 2/3] ktest: check parent options for a repeated test

[Steven Rostedt]

On Fri, 2011-11-18 at 17:05 +0530, Rabin Vincent wrote:
> Let's say we have "OUTPUT_DIR = build/${TEST_NAME}", and we're iterating
> a test.  In the second iteration of a test, the TEST_NAME of the test
> we're repeating is not used.  Instead, ${TEST_NAME} appears literally:
> 
>    touch /home/rabin/kernel/test/build/${TEST_NAME}/.config ... SUCCESS
> 
> Fix this by making __eval_option() check the parent test options
> for a repeated test.

Nice catch! I missed this. I'll give it a test run, and if all is fine,
I'll pull it in. Thanks.

This looks like a bug fix, so I may even push this to Linus now.

Thanks!

-- Steve

> Signed-off-by: Rabin Vincent <rabin@rab.in>
> ---
>  tools/testing/ktest/ktest.pl |   16 ++++++++++++++++
>  1 files changed, 16 insertions(+), 0 deletions(-)
> 
> diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
> index 3185e0a..e93c21c 100755
> --- a/tools/testing/ktest/ktest.pl
> +++ b/tools/testing/ktest/ktest.pl
> @@ -748,6 +748,18 @@ sub __eval_option {
>      # Add space to evaluate the character before $
>      $option = " $option";
>      my $retval = "";
> +    my $repeated = 0;
> +    my $parent = 0;
> +
> +    foreach my $test (keys %repeat_tests) {
> +	if ($i >= $test &&
> +	    $i < $test + $repeat_tests{$test}) {
> +
> +	    $repeated = 1;
> +	    $parent = $test;
> +	    last;
> +	}
> +    }
>  
>      while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
>  	my $start = $1;
> @@ -761,10 +773,14 @@ sub __eval_option {
>  	# otherwise see if the default OPT (without [$i]) exists.
>  
>  	my $o = "$var\[$i\]";
> +	my $parento = "$var\[$parent\]";
>  
>  	if (defined($opt{$o})) {
>  	    $o = $opt{$o};
>  	    $retval = "$retval$o";
> +	} elsif ($repeated && defined($opt{$parento})) {
> +	    $o = $opt{$parento};
> +	    $retval = "$retval$o";
>  	} elsif (defined($opt{$var})) {
>  	    $o = $opt{$var};
>  	    $retval = "$retval$o";

[PATCH 3/3] ktest: allow success logs to be stored

[Rabin Vincent]

Add a STORE_SUCCESSES option, to allow success logs to be stored, for
example to double-check or otherwise post-process the test logs.

Signed-off-by: Rabin Vincent <rabin@rab.in>
---
 tools/testing/ktest/ktest.pl    |   80 ++++++++++++++++++++++----------------
 tools/testing/ktest/sample.conf |    6 +++
 2 files changed, 52 insertions(+), 34 deletions(-)

diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index e93c21c..6ef104e 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -113,6 +113,7 @@ my $bisect_sleep_time;
 my $patchcheck_sleep_time;
 my $ignore_warnings;
 my $store_failures;
+my $store_successes;
 my $test_name;
 my $timeout;
 my $booted_timeout;
@@ -976,6 +977,43 @@ sub wait_for_monitor {
     print "** Monitor flushed **\n";
 }
 
+sub save_logs {
+	my ($result, $basedir) = @_;
+	my @t = localtime;
+	my $date = sprintf "%04d%02d%02d%02d%02d%02d",
+		1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
+
+	my $type = $build_type;
+	if ($type =~ /useconfig/) {
+	    $type = "useconfig";
+	}
+
+	my $dir = "$machine-$test_type-$type-$result-$date";
+
+	$dir = "$basedir/$dir";
+
+	if (!-d $dir) {
+	    mkpath($dir) or
+		die "can't create $dir";
+	}
+
+	my %files = (
+		"config" => $output_config,
+		"buildlog" => $buildlog,
+		"dmesg" => $dmesg,
+		"testlog" => $testlog,
+	);
+
+	while (my ($name, $source) = each(%files)) {
+		if (-f "$source") {
+			cp "$source", "$dir/$name" or
+				die "failed to copy $source";
+		}
+	}
+
+	doprint "*** Saved info to $dir ***\n";
+}
+
 sub fail {
 
 	if ($die_on_failure) {
@@ -1004,40 +1042,9 @@ sub fail {
 	doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 	doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 
-	return 1 if (!defined($store_failures));
-
-	my @t = localtime;
-	my $date = sprintf "%04d%02d%02d%02d%02d%02d",
-		1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
-
-	my $type = $build_type;
-	if ($type =~ /useconfig/) {
-	    $type = "useconfig";
-	}
-
-	my $dir = "$machine-$test_type-$type-fail-$date";
-	my $faildir = "$store_failures/$dir";
-
-	if (!-d $faildir) {
-	    mkpath($faildir) or
-		die "can't create $faildir";
-	}
-
-	my %files = (
-		"config" => $output_config,
-		"buildlog" => $buildlog,
-		"dmesg" => $dmesg,
-		"testlog" => $testlog,
-	);
-
-	while (my ($name, $source) = each(%files)) {
-		if (-f "$source") {
-			cp "$source", "$faildir/$name" or
-				die "failed to copy $source";
-		}
-	}
-
-	doprint "*** Saved info to $faildir ***\n";
+	if (defined($store_failures)) {
+	    save_logs "fail", $store_failures;
+        }
 
 	return 1;
 }
@@ -1643,6 +1650,10 @@ sub success {
     doprint     "*******************************************\n";
     doprint     "*******************************************\n";
 
+    if (defined($store_successes)) {
+        save_logs "success", $store_successes;
+    }
+
     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
 	doprint "Reboot and wait $sleep_time seconds\n";
 	reboot $sleep_time;
@@ -3137,6 +3148,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
     $bisect_skip = set_test_option("BISECT_SKIP", $i);
     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
     $store_failures = set_test_option("STORE_FAILURES", $i);
+    $store_successes = set_test_option("STORE_SUCCESSES", $i);
     $test_name = set_test_option("TEST_NAME", $i);
     $timeout = set_test_option("TIMEOUT", $i);
     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index dbedfa1..42e0eb9 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -589,6 +589,12 @@
 # (default undefined)
 #STORE_FAILURES = /home/test/failures
 
+# Directory to store success directories on success. If this is not
+# set, the .config, dmesg and bootlog will not be saved if a
+# test succeeds.
+# (default undefined)
+#STORE_SUCCESSES = /home/test/successes
+
 # Build without doing a make mrproper, or removing .config
 # (default 0)
 #BUILD_NOCLEAN = 0
-- 
1.7.7.3

Re: [PATCH 3/3] ktest: allow success logs to be stored

[Steven Rostedt]

On Fri, 2011-11-18 at 17:05 +0530, Rabin Vincent wrote:
> Add a STORE_SUCCESSES option, to allow success logs to be stored, for
> example to double-check or otherwise post-process the test logs.

Looks good. I'll give it a test run and put it into the 3.3 queue.

Thanks!

-- Steve

> 
> Signed-off-by: Rabin Vincent <rabin@rab.in>
> ---
>  tools/testing/ktest/ktest.pl    |   80 ++++++++++++++++++++++----------------
>  tools/testing/ktest/sample.conf |    6 +++
>  2 files changed, 52 insertions(+), 34 deletions(-)
> 
> diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
> index e93c21c..6ef104e 100755
> --- a/tools/testing/ktest/ktest.pl
> +++ b/tools/testing/ktest/ktest.pl
> @@ -113,6 +113,7 @@ my $bisect_sleep_time;
>  my $patchcheck_sleep_time;
>  my $ignore_warnings;
>  my $store_failures;
> +my $store_successes;
>  my $test_name;
>  my $timeout;
>  my $booted_timeout;
> @@ -976,6 +977,43 @@ sub wait_for_monitor {
>      print "** Monitor flushed **\n";
>  }
>  
> +sub save_logs {
> +	my ($result, $basedir) = @_;
> +	my @t = localtime;
> +	my $date = sprintf "%04d%02d%02d%02d%02d%02d",
> +		1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
> +
> +	my $type = $build_type;
> +	if ($type =~ /useconfig/) {
> +	    $type = "useconfig";
> +	}
> +
> +	my $dir = "$machine-$test_type-$type-$result-$date";
> +
> +	$dir = "$basedir/$dir";
> +
> +	if (!-d $dir) {
> +	    mkpath($dir) or
> +		die "can't create $dir";
> +	}
> +
> +	my %files = (
> +		"config" => $output_config,
> +		"buildlog" => $buildlog,
> +		"dmesg" => $dmesg,
> +		"testlog" => $testlog,
> +	);
> +
> +	while (my ($name, $source) = each(%files)) {
> +		if (-f "$source") {
> +			cp "$source", "$dir/$name" or
> +				die "failed to copy $source";
> +		}
> +	}
> +
> +	doprint "*** Saved info to $dir ***\n";
> +}
> +
>  sub fail {
>  
>  	if ($die_on_failure) {
> @@ -1004,40 +1042,9 @@ sub fail {
>  	doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
>  	doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
>  
> -	return 1 if (!defined($store_failures));
> -
> -	my @t = localtime;
> -	my $date = sprintf "%04d%02d%02d%02d%02d%02d",
> -		1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
> -
> -	my $type = $build_type;
> -	if ($type =~ /useconfig/) {
> -	    $type = "useconfig";
> -	}
> -
> -	my $dir = "$machine-$test_type-$type-fail-$date";
> -	my $faildir = "$store_failures/$dir";
> -
> -	if (!-d $faildir) {
> -	    mkpath($faildir) or
> -		die "can't create $faildir";
> -	}
> -
> -	my %files = (
> -		"config" => $output_config,
> -		"buildlog" => $buildlog,
> -		"dmesg" => $dmesg,
> -		"testlog" => $testlog,
> -	);
> -
> -	while (my ($name, $source) = each(%files)) {
> -		if (-f "$source") {
> -			cp "$source", "$faildir/$name" or
> -				die "failed to copy $source";
> -		}
> -	}
> -
> -	doprint "*** Saved info to $faildir ***\n";
> +	if (defined($store_failures)) {
> +	    save_logs "fail", $store_failures;
> +        }
>  
>  	return 1;
>  }
> @@ -1643,6 +1650,10 @@ sub success {
>      doprint     "*******************************************\n";
>      doprint     "*******************************************\n";
>  
> +    if (defined($store_successes)) {
> +        save_logs "success", $store_successes;
> +    }
> +
>      if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
>  	doprint "Reboot and wait $sleep_time seconds\n";
>  	reboot $sleep_time;
> @@ -3137,6 +3148,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
>      $bisect_skip = set_test_option("BISECT_SKIP", $i);
>      $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
>      $store_failures = set_test_option("STORE_FAILURES", $i);
> +    $store_successes = set_test_option("STORE_SUCCESSES", $i);
>      $test_name = set_test_option("TEST_NAME", $i);
>      $timeout = set_test_option("TIMEOUT", $i);
>      $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
> diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
> index dbedfa1..42e0eb9 100644
> --- a/tools/testing/ktest/sample.conf
> +++ b/tools/testing/ktest/sample.conf
> @@ -589,6 +589,12 @@
>  # (default undefined)
>  #STORE_FAILURES = /home/test/failures
>  
> +# Directory to store success directories on success. If this is not
> +# set, the .config, dmesg and bootlog will not be saved if a
> +# test succeeds.
> +# (default undefined)
> +#STORE_SUCCESSES = /home/test/successes
> +
>  # Build without doing a make mrproper, or removing .config
>  # (default 0)
>  #BUILD_NOCLEAN = 0

Re: [PATCH 3/3] ktest: allow success logs to be stored

[Steven Rostedt]

On Fri, 2011-11-18 at 17:05 +0530, Rabin Vincent wrote:
> Add a STORE_SUCCESSES option, to allow success logs to be stored, for
> example to double-check or otherwise post-process the test logs.
> 

This was actually on my todo list, so thanks for taking that off for me.
But I want to let you know that new features like these wont be pushed
to mainline until the next merge window. I already added it to my queue
for 3.3. I'll be pushing to my tree sometime in the later -rc cycles.

Thanks again,

-- Steve

Re: [PATCH 1/3] ktest: save test output

[Steven Rostedt]

On Fri, 2011-11-18 at 17:05 +0530, Rabin Vincent wrote:
> The test output may contain useful information; save it along with the
> already-saved buildlog, dmesg, and .config.

Makes sense.

> 
> Signed-off-by: Rabin Vincent <rabin@rab.in>
> ---
>  tools/testing/ktest/ktest.pl |   30 +++++++++++++++++++-----------
>  1 files changed, 19 insertions(+), 11 deletions(-)
> 
> diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
> index 30e2bef..3185e0a 100755
> --- a/tools/testing/ktest/ktest.pl
> +++ b/tools/testing/ktest/ktest.pl
> @@ -103,6 +103,7 @@ my $in_patchcheck = 0;
>  my $run_test;
>  my $redirect;
>  my $buildlog;
> +my $testlog;
>  my $dmesg;
>  my $monitor_fp;
>  my $monitor_pid;
> @@ -1005,17 +1006,19 @@ sub fail {
>  	    mkpath($faildir) or
>  		die "can't create $faildir";
>  	}
> -	if (-f "$output_config") {
> -	    cp "$output_config", "$faildir/config" or
> -		die "failed to copy .config";
> -	}
> -	if (-f $buildlog) {
> -	    cp $buildlog, "$faildir/buildlog" or
> -		die "failed to move $buildlog";
> -	}
> -	if (-f $dmesg) {
> -	    cp $dmesg, "$faildir/dmesg" or
> -		die "failed to move $dmesg";
> +
> +	my %files = (
> +		"config" => $output_config,
> +		"buildlog" => $buildlog,
> +		"dmesg" => $dmesg,
> +		"testlog" => $testlog,
> +	);
> +
> +	while (my ($name, $source) = each(%files)) {
> +		if (-f "$source") {
> +			cp "$source", "$faildir/$name" or
> +				die "failed to copy $source";
> +		}

Nice little clean up. You can see that I think more C like than Perl. I
usually don't like "perlisms" but this is a clean perl fix that isn't
too complex to understand by non perl folks.

>  	}
>  
>  	doprint "*** Saved info to $faildir ***\n";
> @@ -1653,7 +1656,10 @@ sub child_run_test {
>      $poweroff_on_error = 0;
>      $die_on_failure = 1;
>  
> +    $redirect = "$testlog";
>      run_command $run_test or $failed = 1;
> +    undef $redirect;
> +
>      exit $failed;
>  }
>  
> @@ -3153,6 +3159,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
>      $target = "$ssh_user\@$machine";
>  
>      $buildlog = "$tmpdir/buildlog-$machine";
> +    $testlog = "$tmpdir/testlog-$machine";
>      $dmesg = "$tmpdir/dmesg-$machine";
>      $make = "$makecmd O=$outputdir";
>      $output_config = "$outputdir/.config";
> @@ -3189,6 +3196,7 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
>  
>      unlink $dmesg;
>      unlink $buildlog;
> +    unlink $testlog;
>  
>      if (defined($addconfig)) {
>  	my $min = $minconfig;

I'll pull it in, thanks!

-- Steve