[PATCH v3 0/5] kselftest framework and test changes to use it

[PATCH v3 0/5] kselftest framework and test changes to use it

[Shuah Khan]

This patch v3 series: (revised to address v2 comments)
Dropped mount and ptrace patches.

Add kselftest framework for tests to use. This is a light
weight framework provides a set of interfaces to report test
results and test statistics on number of tests passed and failed.

Several tests are changed to use the framework to report results.

Shuah Khan (5):
  selftests: add kselftest framework for uniform test reporting
  selftests/breakpoints: change test to use ksft framework
  selftests/ipc: change test to use ksft framework
  selftests/kcmp: change test to use ksft framework
  selftests/timers: change test to use ksft framework

 .../selftests/breakpoints/breakpoint_test.c        | 10 ++--
 tools/testing/selftests/ipc/msgque.c               | 26 ++++-----
 tools/testing/selftests/kcmp/kcmp_test.c           | 27 +++++++---
 tools/testing/selftests/kselftest.h                | 62 ++++++++++++++++++++++
 tools/testing/selftests/timers/posix_timers.c      | 14 ++---
 5 files changed, 110 insertions(+), 29 deletions(-)
 create mode 100644 tools/testing/selftests/kselftest.h

-- 
1.9.1

[PATCH v3 1/5] selftests: add kselftest framework for uniform test reporting

[Shuah Khan]

Add kselftest framework for tests to use. This is a light
weight framework provides a set of interfaces to report test
results. Tests can use these interfaces to report pass, and
fail cases as well as when failure is due to configuration
problems such as missing modules, or when a test that is should
fail, fails as expected, and a test that should fail, passes.
The framework uses POSIX standard return codes for reporting
results to address the needs of users that want to run the kernel
selftests from their user-space test suites and want to know why a
test failed. In addition, the framework includes interfaces to use
to report test statistics on number of tests passed and failed.

Signed-off-by: Shuah Khan <shuahkh-JPH+aEBZ4P+UEJcrhfAQsw@public.gmane.org>
---
 tools/testing/selftests/kselftest.h | 62 +++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 tools/testing/selftests/kselftest.h

diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h
new file mode 100644
index 0000000..572c888
--- /dev/null
+++ b/tools/testing/selftests/kselftest.h
@@ -0,0 +1,62 @@
+/*
+ * kselftest.h:	kselftest framework return codes to include from
+ *		selftests.
+ *
+ * Copyright (c) 2014 Shuah Khan <shuahkh-JPH+aEBZ4P+UEJcrhfAQsw@public.gmane.org>
+ * Copyright (c) 2014 Samsung Electronics Co., Ltd.
+ *
+ * This file is released under the GPLv2.
+ */
+#ifndef __KSELFTEST_H
+#define __KSELFTEST_H
+
+#include <stdlib.h>
+#include <unistd.h>
+
+/* counters */
+struct ksft_count {
+	unsigned int ksft_pass;
+	unsigned int ksft_fail;
+	unsigned int ksft_xfail;
+	unsigned int ksft_xpass;
+	unsigned int ksft_xskip;
+};
+
+static struct ksft_count ksft_cnt;
+
+static inline void ksft_inc_pass_cnt(void) { ksft_cnt.ksft_pass++; }
+static inline void ksft_inc_fail_cnt(void) { ksft_cnt.ksft_fail++; }
+static inline void ksft_inc_xfail_cnt(void) { ksft_cnt.ksft_xfail++; }
+static inline void ksft_inc_xpass_cnt(void) { ksft_cnt.ksft_xpass++; }
+static inline void ksft_inc_xskip_cnt(void) { ksft_cnt.ksft_xskip++; }
+
+static inline void ksft_print_cnts(void)
+{
+	printf("Pass: %d Fail: %d Xfail: %d Xpass: %d, Xskip: %d\n",
+		ksft_cnt.ksft_pass, ksft_cnt.ksft_fail,
+		ksft_cnt.ksft_xfail, ksft_cnt.ksft_xpass,
+		ksft_cnt.ksft_xskip);
+}
+
+static inline int ksft_exit_pass(void)
+{
+	exit(0);
+}
+static inline int ksft_exit_fail(void)
+{
+	exit(1);
+}
+static inline int ksft_exit_xfail(void)
+{
+	exit(2);
+}
+static inline int ksft_exit_xpass(void)
+{
+	exit(3);
+}
+static inline int ksft_exit_skip(void)
+{
+	exit(4);
+}
+
+#endif /* __KSELFTEST_H */
-- 
1.9.1

[PATCH v3 2/5] selftests/breakpoints: change test to use ksft framework

[Shuah Khan]

Change breakpoints test to use kselftest framework to report
test results.

Signed-off-by: Shuah Khan <shuahkh-JPH+aEBZ4P+UEJcrhfAQsw@public.gmane.org>
---
 tools/testing/selftests/breakpoints/breakpoint_test.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tools/testing/selftests/breakpoints/breakpoint_test.c b/tools/testing/selftests/breakpoints/breakpoint_test.c
index a0743f3..120895a 100644
--- a/tools/testing/selftests/breakpoints/breakpoint_test.c
+++ b/tools/testing/selftests/breakpoints/breakpoint_test.c
@@ -17,6 +17,8 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 
+#include "../kselftest.h"
+
 
 /* Breakpoint access modes */
 enum {
@@ -42,7 +44,7 @@ static void set_breakpoint_addr(void *addr, int n)
 		     offsetof(struct user, u_debugreg[n]), addr);
 	if (ret) {
 		perror("Can't set breakpoint addr\n");
-		exit(-1);
+		ksft_exit_fail();
 	}
 }
 
@@ -105,7 +107,7 @@ static void toggle_breakpoint(int n, int type, int len,
 		     offsetof(struct user, u_debugreg[7]), dr7);
 	if (ret) {
 		perror("Can't set dr7");
-		exit(-1);
+		ksft_exit_fail();
 	}
 }
 
@@ -275,7 +277,7 @@ static void check_success(const char *msg)
 			msg2 = "Ok";
 		if (ptrace(PTRACE_POKEDATA, child_pid, &trapped, 1)) {
 			perror("Can't poke\n");
-			exit(-1);
+			ksft_exit_fail();
 		}
 	}
 
@@ -390,5 +392,5 @@ int main(int argc, char **argv)
 
 	wait(NULL);
 
-	return 0;
+	return ksft_exit_pass();
 }
-- 
1.9.1

[PATCH v3 3/5] selftests/ipc: change test to use ksft framework

[Shuah Khan]

Change ipc test to use kselftest framework to report
test results. With this change this test exits with
EXIT_FAIL instead of -errno. Changed print errno in
test fail messages to not loose that information.

Signed-off-by: Shuah Khan <shuahkh@osg.samsung.com>
---
 tools/testing/selftests/ipc/msgque.c | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/tools/testing/selftests/ipc/msgque.c b/tools/testing/selftests/ipc/msgque.c
index 552f081..1b2ce33 100644
--- a/tools/testing/selftests/ipc/msgque.c
+++ b/tools/testing/selftests/ipc/msgque.c
@@ -5,6 +5,8 @@
 #include <linux/msg.h>
 #include <fcntl.h>
 
+#include "../kselftest.h"
+
 #define MAX_MSG_SIZE		32
 
 struct msg1 {
@@ -195,58 +197,58 @@ int main(int argc, char **argv)
 
 	if (getuid() != 0) {
 		printf("Please run the test as root - Exiting.\n");
-		exit(1);
+		return ksft_exit_fail();
 	}
 
 	msgque.key = ftok(argv[0], 822155650);
 	if (msgque.key == -1) {
-		printf("Can't make key\n");
-		return -errno;
+		printf("Can't make key: %d\n", -errno);
+		return ksft_exit_fail();
 	}
 
 	msgque.msq_id = msgget(msgque.key, IPC_CREAT | IPC_EXCL | 0666);
 	if (msgque.msq_id == -1) {
 		err = -errno;
-		printf("Can't create queue\n");
+		printf("Can't create queue: %d\n", err);
 		goto err_out;
 	}
 
 	err = fill_msgque(&msgque);
 	if (err) {
-		printf("Failed to fill queue\n");
+		printf("Failed to fill queue: %d\n", err);
 		goto err_destroy;
 	}
 
 	err = dump_queue(&msgque);
 	if (err) {
-		printf("Failed to dump queue\n");
+		printf("Failed to dump queue: %d\n", err);
 		goto err_destroy;
 	}
 
 	err = check_and_destroy_queue(&msgque);
 	if (err) {
-		printf("Failed to check and destroy queue\n");
+		printf("Failed to check and destroy queue: %d\n", err);
 		goto err_out;
 	}
 
 	err = restore_queue(&msgque);
 	if (err) {
-		printf("Failed to restore queue\n");
+		printf("Failed to restore queue: %d\n", err);
 		goto err_destroy;
 	}
 
 	err = check_and_destroy_queue(&msgque);
 	if (err) {
-		printf("Failed to test queue\n");
+		printf("Failed to test queue: %d\n", err);
 		goto err_out;
 	}
-	return 0;
+	return ksft_exit_pass();
 
 err_destroy:
 	if (msgctl(msgque.msq_id, IPC_RMID, 0)) {
 		printf("Failed to destroy queue: %d\n", -errno);
-		return -errno;
+		return ksft_exit_fail();
 	}
 err_out:
-	return err;
+	return ksft_exit_fail();
 }
-- 
1.9.1

Re: [PATCH v3 3/5] selftests/ipc: change test to use ksft framework

[Davidlohr Bueso]

On Fri, 2014-10-03 at 09:36 -0600, Shuah Khan wrote:
>  	msgque.key = ftok(argv[0], 822155650);
>  	if (msgque.key == -1) {
> -		printf("Can't make key\n");
> -		return -errno;
> +		printf("Can't make key: %d\n", -errno);

So printing a numeric value is quite useless when users actually run
into these errors -- which is why I like err() so much. How about using
strerror() instead?

Thanks,
Davidlohr

Re: [PATCH v3 3/5] selftests/ipc: change test to use ksft framework

[Shuah Khan]

On 10/03/2014 11:39 AM, Davidlohr Bueso wrote:
> On Fri, 2014-10-03 at 09:36 -0600, Shuah Khan wrote:
>>  	msgque.key = ftok(argv[0], 822155650);
>>  	if (msgque.key == -1) {
>> -		printf("Can't make key\n");
>> -		return -errno;
>> +		printf("Can't make key: %d\n", -errno);
> 
> So printing a numeric value is quite useless when users actually run
> into these errors -- which is why I like err() so much. How about using
> strerror() instead?
> 

Yes. using perror() does give better information. There are other
places in this file that use errno. How about I make that a separate
patch and catch all of them at once to use perror() as a follow-up
change? That way I fix all at once without adding more changes to
this patch.

thanks,
-- Shuah

-- 
Shuah Khan
Sr. Linux Kernel Developer
Samsung Research America (Silicon Valley)
shuahkh-JPH+aEBZ4P+UEJcrhfAQsw@public.gmane.org | (970) 217-8978

Re: [PATCH v3 3/5] selftests/ipc: change test to use ksft framework

[Davidlohr Bueso]

On Fri, 2014-10-03 at 13:42 -0600, Shuah Khan wrote:
> On 10/03/2014 11:39 AM, Davidlohr Bueso wrote:
> > On Fri, 2014-10-03 at 09:36 -0600, Shuah Khan wrote:
> >>  	msgque.key = ftok(argv[0], 822155650);
> >>  	if (msgque.key == -1) {
> >> -		printf("Can't make key\n");
> >> -		return -errno;
> >> +		printf("Can't make key: %d\n", -errno);
> > 
> > So printing a numeric value is quite useless when users actually run
> > into these errors -- which is why I like err() so much. How about using
> > strerror() instead?
> > 
> 
> Yes. using perror() does give better information. There are other
> places in this file that use errno. How about I make that a separate
> patch and catch all of them at once to use perror() as a follow-up
> change? That way I fix all at once without adding more changes to
> this patch.

Sounds good.

[PATCH v3 4/5] selftests/kcmp: change test to use ksft framework

[Shuah Khan]

Change kcmp test to use kselftest framework to report
test results and test statistics.

Signed-off-by: Shuah Khan <shuahkh@osg.samsung.com>
---
 tools/testing/selftests/kcmp/kcmp_test.c | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/tools/testing/selftests/kcmp/kcmp_test.c b/tools/testing/selftests/kcmp/kcmp_test.c
index dbba408..a5a4da8 100644
--- a/tools/testing/selftests/kcmp/kcmp_test.c
+++ b/tools/testing/selftests/kcmp/kcmp_test.c
@@ -17,6 +17,8 @@
 #include <sys/stat.h>
 #include <sys/wait.h>
 
+#include "../kselftest.h"
+
 static long sys_kcmp(int pid1, int pid2, int type, int fd1, int fd2)
 {
 	return syscall(__NR_kcmp, pid1, pid2, type, fd1, fd2);
@@ -34,13 +36,13 @@ int main(int argc, char **argv)
 
 	if (fd1 < 0) {
 		perror("Can't create file");
-		exit(1);
+		ksft_exit_fail();
 	}
 
 	pid2 = fork();
 	if (pid2 < 0) {
 		perror("fork failed");
-		exit(1);
+		ksft_exit_fail();
 	}
 
 	if (!pid2) {
@@ -50,7 +52,7 @@ int main(int argc, char **argv)
 		fd2 = open(kpath, O_RDWR, 0644);
 		if (fd2 < 0) {
 			perror("Can't open file");
-			exit(1);
+			ksft_exit_fail();
 		}
 
 		/* An example of output and arguments */
@@ -74,23 +76,34 @@ int main(int argc, char **argv)
 		if (ret) {
 			printf("FAIL: 0 expected but %d returned (%s)\n",
 				ret, strerror(errno));
+			ksft_inc_fail_cnt();
 			ret = -1;
-		} else
+		} else {
 			printf("PASS: 0 returned as expected\n");
+			ksft_inc_pass_cnt();
+		}
 
 		/* Compare with self */
 		ret = sys_kcmp(pid1, pid1, KCMP_VM, 0, 0);
 		if (ret) {
 			printf("FAIL: 0 expected but %d returned (%s)\n",
 				ret, strerror(errno));
+			ksft_inc_fail_cnt();
 			ret = -1;
-		} else
+		} else {
 			printf("PASS: 0 returned as expected\n");
+			ksft_inc_pass_cnt();
+		}
+
+		ksft_print_cnts();
 
-		exit(ret);
+		if (ret)
+			ksft_exit_fail();
+		else
+			ksft_exit_pass();
 	}
 
 	waitpid(pid2, &status, P_ALL);
 
-	return 0;
+	return ksft_exit_pass();
 }
-- 
1.9.1

[PATCH v3 5/5] selftests/timers: change test to use ksft framework

[Shuah Khan]

Change timers test to use kselftest framework to report
test results.

Signed-off-by: Shuah Khan <shuahkh@osg.samsung.com>
---
 tools/testing/selftests/timers/posix_timers.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c
index 41bd855..f87d970 100644
--- a/tools/testing/selftests/timers/posix_timers.c
+++ b/tools/testing/selftests/timers/posix_timers.c
@@ -15,6 +15,8 @@
 #include <time.h>
 #include <pthread.h>
 
+#include "../kselftest.h"
+
 #define DELAY 2
 #define USECS_PER_SEC 1000000
 
@@ -194,16 +196,16 @@ int main(int argc, char **argv)
 	printf("based timers if other threads run on the CPU...\n");
 
 	if (check_itimer(ITIMER_VIRTUAL) < 0)
-		return -1;
+		return ksft_exit_fail();
 
 	if (check_itimer(ITIMER_PROF) < 0)
-		return -1;
+		return ksft_exit_fail();
 
 	if (check_itimer(ITIMER_REAL) < 0)
-		return -1;
+		return ksft_exit_fail();
 
 	if (check_timer_create(CLOCK_THREAD_CPUTIME_ID) < 0)
-		return -1;
+		return ksft_exit_fail();
 
 	/*
 	 * It's unfortunately hard to reliably test a timer expiration
@@ -215,7 +217,7 @@ int main(int argc, char **argv)
 	 * find a better solution.
 	 */
 	if (check_timer_create(CLOCK_PROCESS_CPUTIME_ID) < 0)
-		return -1;
+		return ksft_exit_fail();
 
-	return 0;
+	return ksft_exit_pass();
 }
-- 
1.9.1