* [RFC v1 1/6] kunit: Add test attributes API structure
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
@ 2023-06-10 0:51 ` Rae Moar
2023-06-10 8:29 ` David Gow
2023-06-10 0:51 ` [RFC v1 2/6] kunit: Add speed attribute Rae Moar
` (5 subsequent siblings)
6 siblings, 1 reply; 21+ messages in thread
From: Rae Moar @ 2023-06-10 0:51 UTC (permalink / raw)
To: shuah, davidgow, dlatypov, brendan.higgins
Cc: linux-kselftest, kunit-dev, linux-kernel, keescook,
linux-hardening, jstultz, tglx, sboyd, Rae Moar
Add the basic structure of the test attribute API to KUnit, which can be
used to save and access test associated data.
Add attributes.c and attributes.h to hold associated structs and functions
for the API.
Create a struct that holds a variety of associated helper functions for
each test attribute. These helper functions will be used to get the
attribute value, convert the value to a string, and filter based on the
value. This struct is flexible by design to allow for attributes of
numerous types and contexts.
Add a method to print test attributes in the format of "# [<test_name if
not suite>.]<attribute_name>: <attribute_value>".
Example for a suite: "# speed: slow"
Example for a test case: "# test_case.speed: very_slow"
Use this method to report attributes in the KTAP output and _list_tests
output.
In test.h, add fields and associated helper functions to test cases and
suites to hold user-inputted test attributes.
Signed-off-by: Rae Moar <rmoar@google.com>
---
include/kunit/attributes.h | 19 +++++++++++
include/kunit/test.h | 33 +++++++++++++++++++
lib/kunit/Makefile | 3 +-
lib/kunit/attributes.c | 65 ++++++++++++++++++++++++++++++++++++++
lib/kunit/executor.c | 10 +++++-
lib/kunit/test.c | 17 ++++++----
6 files changed, 138 insertions(+), 9 deletions(-)
create mode 100644 include/kunit/attributes.h
create mode 100644 lib/kunit/attributes.c
diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
new file mode 100644
index 000000000000..9fcd184cce36
--- /dev/null
+++ b/include/kunit/attributes.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KUnit API to save and access test attributes
+ *
+ * Copyright (C) 2023, Google LLC.
+ * Author: Rae Moar <rmoar@google.com>
+ */
+
+#ifndef _KUNIT_ATTRIBUTES_H
+#define _KUNIT_ATTRIBUTES_H
+
+/*
+ * Print all test attributes for a test case or suite.
+ * Output format for test cases: "# <test_name>.<attribute>: <value>"
+ * Output format for test suites: "# <attribute>: <value>"
+ */
+void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
+
+#endif /* _KUNIT_ATTRIBUTES_H */
diff --git a/include/kunit/test.h b/include/kunit/test.h
index 23120d50499e..1fc9155988e9 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -63,12 +63,16 @@ enum kunit_status {
KUNIT_SKIPPED,
};
+/* Holds attributes for each test case and suite */
+struct kunit_attributes {};
+
/**
* struct kunit_case - represents an individual test case.
*
* @run_case: the function representing the actual test case.
* @name: the name of the test case.
* @generate_params: the generator function for parameterized tests.
+ * @attr: the attributes associated with the test
*
* A test case is a function with the signature,
* ``void (*)(struct kunit *)``
@@ -104,6 +108,7 @@ struct kunit_case {
void (*run_case)(struct kunit *test);
const char *name;
const void* (*generate_params)(const void *prev, char *desc);
+ struct kunit_attributes attr;
/* private: internal use only. */
enum kunit_status status;
@@ -133,6 +138,18 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
*/
#define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
+/**
+ * KUNIT_CASE_ATTR - A helper for creating a &struct kunit_case
+ * with attributes
+ *
+ * @test_name: a reference to a test case function.
+ * @attributes: a reference to a struct kunit_attributes object containing
+ * test attributes
+ */
+#define KUNIT_CASE_ATTR(test_name, attributes) \
+ { .run_case = test_name, .name = #test_name, \
+ .attr = attributes }
+
/**
* KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
*
@@ -154,6 +171,20 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params }
+/**
+ * KUNIT_CASE_PARAM_ATTR - A helper for creating a parameterized &struct
+ * kunit_case with attributes
+ *
+ * @test_name: a reference to a test case function.
+ * @gen_params: a reference to a parameter generator function.
+ * @attributes: a reference to a struct kunit_attributes object containing
+ * test attributes
+ */
+#define KUNIT_CASE_PARAM_ATTR(test_name, gen_params, attributes) \
+ { .run_case = test_name, .name = #test_name, \
+ .generate_params = gen_params, \
+ .attr = attributes }
+
/**
* struct kunit_suite - describes a related collection of &struct kunit_case
*
@@ -163,6 +194,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
* @init: called before every test case.
* @exit: called after every test case.
* @test_cases: a null terminated array of test cases.
+ * @attr: the attributes associated with the test suite
*
* A kunit_suite is a collection of related &struct kunit_case s, such that
* @init is called before every test case and @exit is called after every
@@ -182,6 +214,7 @@ struct kunit_suite {
int (*init)(struct kunit *test);
void (*exit)(struct kunit *test);
struct kunit_case *test_cases;
+ struct kunit_attributes attr;
/* private: internal use only */
char status_comment[KUNIT_STATUS_COMMENT_SIZE];
diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile
index cb417f504996..46f75f23dfe4 100644
--- a/lib/kunit/Makefile
+++ b/lib/kunit/Makefile
@@ -6,7 +6,8 @@ kunit-objs += test.o \
string-stream.o \
assert.o \
try-catch.o \
- executor.o
+ executor.o \
+ attributes.o
ifeq ($(CONFIG_KUNIT_DEBUGFS),y)
kunit-objs += debugfs.o
diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
new file mode 100644
index 000000000000..0ea641be795f
--- /dev/null
+++ b/lib/kunit/attributes.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit API to save and access test attributes
+ *
+ * Copyright (C) 2023, Google LLC.
+ * Author: Rae Moar <rmoar@google.com>
+ */
+
+#include <kunit/test.h>
+#include <kunit/attributes.h>
+
+/**
+ * struct kunit_attr - represents a test attribute and holds flexible
+ * helper functions to interact with attribute.
+ *
+ * @name: name of test attribute, eg. speed
+ * @get_attr: function to return attribute value given a test
+ * @to_string: function to return string representation of given
+ * attribute value
+ * @filter: function to indicate whether a given attribute value passes a
+ * filter
+ */
+struct kunit_attr {
+ const char *name;
+ void *(*get_attr)(void *test_or_suite, bool is_test);
+ const char *(*to_string)(void *attr, bool *to_free);
+ int (*filter)(void *attr, const char *input, int *err);
+ void *attr_default;
+};
+
+/* List of all Test Attributes */
+
+static struct kunit_attr kunit_attr_list[1] = {};
+
+/* Helper Functions to Access Attributes */
+
+void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level)
+{
+ int i;
+ bool to_free;
+ void *attr;
+ const char *attr_name, *attr_str;
+ struct kunit_suite *suite = is_test ? NULL : test_or_suite;
+ struct kunit_case *test = is_test ? test_or_suite : NULL;
+
+ for (i = 0; i < ARRAY_SIZE(kunit_attr_list); i++) {
+ attr = kunit_attr_list[i].get_attr(test_or_suite, is_test);
+ if (attr) {
+ attr_name = kunit_attr_list[i].name;
+ attr_str = kunit_attr_list[i].to_string(attr, &to_free);
+ if (test) {
+ kunit_log(KERN_INFO, test, "%*s# %s.%s: %s",
+ KUNIT_INDENT_LEN * test_level, "", test->name,
+ attr_name, attr_str);
+ } else {
+ kunit_log(KERN_INFO, suite, "%*s# %s: %s",
+ KUNIT_INDENT_LEN * test_level, "", attr_name, attr_str);
+ }
+
+ /* Free to_string of attribute if needed */
+ if (to_free)
+ kfree(attr_str);
+ }
+ }
+}
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index 74982b83707c..767a84e32f06 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -2,6 +2,7 @@
#include <linux/reboot.h>
#include <kunit/test.h>
+#include <kunit/attributes.h>
#include <linux/glob.h>
#include <linux/moduleparam.h>
@@ -180,10 +181,17 @@ static void kunit_exec_list_tests(struct suite_set *suite_set)
/* Hack: print a ktap header so kunit.py can find the start of KUnit output. */
pr_info("KTAP version 1\n");
- for (suites = suite_set->start; suites < suite_set->end; suites++)
+ for (suites = suite_set->start; suites < suite_set->end; suites++) {
+ /* Print suite name and suite attributes */
+ pr_info("%s\n", (*suites)->name);
+ kunit_print_attr((void *)(*suites), false, 0);
+
+ /* Print test case name and attributes in suite */
kunit_suite_for_each_test_case((*suites), test_case) {
pr_info("%s.%s\n", (*suites)->name, test_case->name);
+ kunit_print_attr((void *)test_case, true, 0);
}
+ }
}
int kunit_run_all_tests(void)
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 84e4666555c9..9ee55139ecd1 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -9,6 +9,7 @@
#include <kunit/resource.h>
#include <kunit/test.h>
#include <kunit/test-bug.h>
+#include <kunit/attributes.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
@@ -168,6 +169,13 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite)
}
EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases);
+/* Currently supported test levels */
+enum {
+ KUNIT_LEVEL_SUITE = 0,
+ KUNIT_LEVEL_CASE,
+ KUNIT_LEVEL_CASE_PARAM,
+};
+
static void kunit_print_suite_start(struct kunit_suite *suite)
{
/*
@@ -181,17 +189,11 @@ static void kunit_print_suite_start(struct kunit_suite *suite)
pr_info(KUNIT_SUBTEST_INDENT "KTAP version 1\n");
pr_info(KUNIT_SUBTEST_INDENT "# Subtest: %s\n",
suite->name);
+ kunit_print_attr((void *)suite, false, KUNIT_LEVEL_CASE);
pr_info(KUNIT_SUBTEST_INDENT "1..%zd\n",
kunit_suite_num_test_cases(suite));
}
-/* Currently supported test levels */
-enum {
- KUNIT_LEVEL_SUITE = 0,
- KUNIT_LEVEL_CASE,
- KUNIT_LEVEL_CASE_PARAM,
-};
-
static void kunit_print_ok_not_ok(struct kunit *test,
unsigned int test_level,
enum kunit_status status,
@@ -651,6 +653,7 @@ int kunit_run_tests(struct kunit_suite *suite)
}
}
+ kunit_print_attr((void *)test_case, true, KUNIT_LEVEL_CASE);
kunit_print_test_stats(&test, param_stats);
--
2.41.0.162.gfafddb0af9-goog
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [RFC v1 1/6] kunit: Add test attributes API structure
2023-06-10 0:51 ` [RFC v1 1/6] kunit: Add test attributes API structure Rae Moar
@ 2023-06-10 8:29 ` David Gow
2023-06-13 20:36 ` Rae Moar
0 siblings, 1 reply; 21+ messages in thread
From: David Gow @ 2023-06-10 8:29 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
[-- Attachment #1: Type: text/plain, Size: 13111 bytes --]
On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
>
> Add the basic structure of the test attribute API to KUnit, which can be
> used to save and access test associated data.
>
> Add attributes.c and attributes.h to hold associated structs and functions
> for the API.
>
> Create a struct that holds a variety of associated helper functions for
> each test attribute. These helper functions will be used to get the
> attribute value, convert the value to a string, and filter based on the
> value. This struct is flexible by design to allow for attributes of
> numerous types and contexts.
>
> Add a method to print test attributes in the format of "# [<test_name if
> not suite>.]<attribute_name>: <attribute_value>".
>
> Example for a suite: "# speed: slow"
>
> Example for a test case: "# test_case.speed: very_slow"
>
> Use this method to report attributes in the KTAP output and _list_tests
> output.
Can we have a link to the draft KTAP spec for this?
Also, by _list_tests, I assume we're talking about the
kunit.action=list option. That's been an "internal" feature for the
kunit script thus far, but kunit.py doesn't use this attribute info
anywhere yet, apart from to print it out via --list_tests. Maybe we
should make including the attributes optional, as the raw list of
tests is currently more useful.
>
> In test.h, add fields and associated helper functions to test cases and
> suites to hold user-inputted test attributes.
>
> Signed-off-by: Rae Moar <rmoar@google.com>
> ---
> include/kunit/attributes.h | 19 +++++++++++
> include/kunit/test.h | 33 +++++++++++++++++++
> lib/kunit/Makefile | 3 +-
> lib/kunit/attributes.c | 65 ++++++++++++++++++++++++++++++++++++++
> lib/kunit/executor.c | 10 +++++-
> lib/kunit/test.c | 17 ++++++----
> 6 files changed, 138 insertions(+), 9 deletions(-)
> create mode 100644 include/kunit/attributes.h
> create mode 100644 lib/kunit/attributes.c
>
> diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
> new file mode 100644
> index 000000000000..9fcd184cce36
> --- /dev/null
> +++ b/include/kunit/attributes.h
> @@ -0,0 +1,19 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * KUnit API to save and access test attributes
> + *
> + * Copyright (C) 2023, Google LLC.
> + * Author: Rae Moar <rmoar@google.com>
> + */
> +
> +#ifndef _KUNIT_ATTRIBUTES_H
> +#define _KUNIT_ATTRIBUTES_H
> +
> +/*
> + * Print all test attributes for a test case or suite.
> + * Output format for test cases: "# <test_name>.<attribute>: <value>"
> + * Output format for test suites: "# <attribute>: <value>"
> + */
> +void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
> +
> +#endif /* _KUNIT_ATTRIBUTES_H */
> diff --git a/include/kunit/test.h b/include/kunit/test.h
> index 23120d50499e..1fc9155988e9 100644
> --- a/include/kunit/test.h
> +++ b/include/kunit/test.h
> @@ -63,12 +63,16 @@ enum kunit_status {
> KUNIT_SKIPPED,
> };
>
> +/* Holds attributes for each test case and suite */
> +struct kunit_attributes {};
This is a placeholder as attributes won't be included until patch 2, right?
> +
> /**
> * struct kunit_case - represents an individual test case.
> *
> * @run_case: the function representing the actual test case.
> * @name: the name of the test case.
> * @generate_params: the generator function for parameterized tests.
> + * @attr: the attributes associated with the test
> *
> * A test case is a function with the signature,
> * ``void (*)(struct kunit *)``
> @@ -104,6 +108,7 @@ struct kunit_case {
> void (*run_case)(struct kunit *test);
> const char *name;
> const void* (*generate_params)(const void *prev, char *desc);
> + struct kunit_attributes attr;
>
> /* private: internal use only. */
> enum kunit_status status;
> @@ -133,6 +138,18 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> */
> #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
>
> +/**
> + * KUNIT_CASE_ATTR - A helper for creating a &struct kunit_case
> + * with attributes
> + *
> + * @test_name: a reference to a test case function.
> + * @attributes: a reference to a struct kunit_attributes object containing
> + * test attributes
> + */
> +#define KUNIT_CASE_ATTR(test_name, attributes) \
> + { .run_case = test_name, .name = #test_name, \
> + .attr = attributes }
> +
This is a bit awkward, as the attributes are either entirely unset
(i.e., zero-initialised), or have to be specified here. I assume the
plan is to use designated initialisers, e.g.:
KUNIT_CASE_ATTR(foo_test, {.speed = KUNIT_SPEED_SLOW, .flaky = true}).
That at least makes it reasonably user-friendly, though the whole need
to make sure zero-initialised values are the defaults is a bit icky.
> /**
> * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
> *
> @@ -154,6 +171,20 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> { .run_case = test_name, .name = #test_name, \
> .generate_params = gen_params }
>
> +/**
> + * KUNIT_CASE_PARAM_ATTR - A helper for creating a parameterized &struct
> + * kunit_case with attributes
> + *
> + * @test_name: a reference to a test case function.
> + * @gen_params: a reference to a parameter generator function.
> + * @attributes: a reference to a struct kunit_attributes object containing
> + * test attributes
> + */
> +#define KUNIT_CASE_PARAM_ATTR(test_name, gen_params, attributes) \
> + { .run_case = test_name, .name = #test_name, \
> + .generate_params = gen_params, \
> + .attr = attributes }
> +
> /**
> * struct kunit_suite - describes a related collection of &struct kunit_case
> *
> @@ -163,6 +194,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> * @init: called before every test case.
> * @exit: called after every test case.
> * @test_cases: a null terminated array of test cases.
> + * @attr: the attributes associated with the test suite
> *
> * A kunit_suite is a collection of related &struct kunit_case s, such that
> * @init is called before every test case and @exit is called after every
> @@ -182,6 +214,7 @@ struct kunit_suite {
> int (*init)(struct kunit *test);
> void (*exit)(struct kunit *test);
> struct kunit_case *test_cases;
> + struct kunit_attributes attr;
>
> /* private: internal use only */
> char status_comment[KUNIT_STATUS_COMMENT_SIZE];
> diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile
> index cb417f504996..46f75f23dfe4 100644
> --- a/lib/kunit/Makefile
> +++ b/lib/kunit/Makefile
> @@ -6,7 +6,8 @@ kunit-objs += test.o \
> string-stream.o \
> assert.o \
> try-catch.o \
> - executor.o
> + executor.o \
> + attributes.o
>
> ifeq ($(CONFIG_KUNIT_DEBUGFS),y)
> kunit-objs += debugfs.o
> diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
> new file mode 100644
> index 000000000000..0ea641be795f
> --- /dev/null
> +++ b/lib/kunit/attributes.c
> @@ -0,0 +1,65 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * KUnit API to save and access test attributes
> + *
> + * Copyright (C) 2023, Google LLC.
> + * Author: Rae Moar <rmoar@google.com>
> + */
> +
> +#include <kunit/test.h>
> +#include <kunit/attributes.h>
> +
> +/**
> + * struct kunit_attr - represents a test attribute and holds flexible
> + * helper functions to interact with attribute.
> + *
> + * @name: name of test attribute, eg. speed
> + * @get_attr: function to return attribute value given a test
> + * @to_string: function to return string representation of given
> + * attribute value
> + * @filter: function to indicate whether a given attribute value passes a
> + * filter
> + */
> +struct kunit_attr {
> + const char *name;
> + void *(*get_attr)(void *test_or_suite, bool is_test);
> + const char *(*to_string)(void *attr, bool *to_free);
> + int (*filter)(void *attr, const char *input, int *err);
> + void *attr_default;
> +};
> +
> +/* List of all Test Attributes */
> +
> +static struct kunit_attr kunit_attr_list[1] = {};
> +
> +/* Helper Functions to Access Attributes */
> +
> +void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level)
> +{
> + int i;
> + bool to_free;
> + void *attr;
> + const char *attr_name, *attr_str;
> + struct kunit_suite *suite = is_test ? NULL : test_or_suite;
> + struct kunit_case *test = is_test ? test_or_suite : NULL;
> +
> + for (i = 0; i < ARRAY_SIZE(kunit_attr_list); i++) {
> + attr = kunit_attr_list[i].get_attr(test_or_suite, is_test);
> + if (attr) {
> + attr_name = kunit_attr_list[i].name;
> + attr_str = kunit_attr_list[i].to_string(attr, &to_free);
> + if (test) {
> + kunit_log(KERN_INFO, test, "%*s# %s.%s: %s",
> + KUNIT_INDENT_LEN * test_level, "", test->name,
> + attr_name, attr_str);
> + } else {
> + kunit_log(KERN_INFO, suite, "%*s# %s: %s",
> + KUNIT_INDENT_LEN * test_level, "", attr_name, attr_str);
> + }
> +
> + /* Free to_string of attribute if needed */
> + if (to_free)
> + kfree(attr_str);
> + }
> + }
> +}
> diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
> index 74982b83707c..767a84e32f06 100644
> --- a/lib/kunit/executor.c
> +++ b/lib/kunit/executor.c
> @@ -2,6 +2,7 @@
>
> #include <linux/reboot.h>
> #include <kunit/test.h>
> +#include <kunit/attributes.h>
> #include <linux/glob.h>
> #include <linux/moduleparam.h>
>
> @@ -180,10 +181,17 @@ static void kunit_exec_list_tests(struct suite_set *suite_set)
> /* Hack: print a ktap header so kunit.py can find the start of KUnit output. */
> pr_info("KTAP version 1\n");
>
> - for (suites = suite_set->start; suites < suite_set->end; suites++)
> + for (suites = suite_set->start; suites < suite_set->end; suites++) {
> + /* Print suite name and suite attributes */
> + pr_info("%s\n", (*suites)->name);
> + kunit_print_attr((void *)(*suites), false, 0);
> +
> + /* Print test case name and attributes in suite */
> kunit_suite_for_each_test_case((*suites), test_case) {
> pr_info("%s.%s\n", (*suites)->name, test_case->name);
> + kunit_print_attr((void *)test_case, true, 0);
> }
> + }
> }
>
> int kunit_run_all_tests(void)
> diff --git a/lib/kunit/test.c b/lib/kunit/test.c
> index 84e4666555c9..9ee55139ecd1 100644
> --- a/lib/kunit/test.c
> +++ b/lib/kunit/test.c
> @@ -9,6 +9,7 @@
> #include <kunit/resource.h>
> #include <kunit/test.h>
> #include <kunit/test-bug.h>
> +#include <kunit/attributes.h>
> #include <linux/kernel.h>
> #include <linux/module.h>
> #include <linux/moduleparam.h>
> @@ -168,6 +169,13 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite)
> }
> EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases);
>
> +/* Currently supported test levels */
> +enum {
> + KUNIT_LEVEL_SUITE = 0,
> + KUNIT_LEVEL_CASE,
> + KUNIT_LEVEL_CASE_PARAM,
> +};
> +
> static void kunit_print_suite_start(struct kunit_suite *suite)
> {
> /*
> @@ -181,17 +189,11 @@ static void kunit_print_suite_start(struct kunit_suite *suite)
> pr_info(KUNIT_SUBTEST_INDENT "KTAP version 1\n");
> pr_info(KUNIT_SUBTEST_INDENT "# Subtest: %s\n",
> suite->name);
> + kunit_print_attr((void *)suite, false, KUNIT_LEVEL_CASE);
> pr_info(KUNIT_SUBTEST_INDENT "1..%zd\n",
> kunit_suite_num_test_cases(suite));
> }
>
> -/* Currently supported test levels */
> -enum {
> - KUNIT_LEVEL_SUITE = 0,
> - KUNIT_LEVEL_CASE,
> - KUNIT_LEVEL_CASE_PARAM,
> -};
> -
> static void kunit_print_ok_not_ok(struct kunit *test,
> unsigned int test_level,
> enum kunit_status status,
> @@ -651,6 +653,7 @@ int kunit_run_tests(struct kunit_suite *suite)
> }
> }
>
> + kunit_print_attr((void *)test_case, true, KUNIT_LEVEL_CASE);
>
> kunit_print_test_stats(&test, param_stats);
>
> --
> 2.41.0.162.gfafddb0af9-goog
>
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4003 bytes --]
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [RFC v1 1/6] kunit: Add test attributes API structure
2023-06-10 8:29 ` David Gow
@ 2023-06-13 20:36 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:36 UTC (permalink / raw)
To: David Gow
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 4:29 AM David Gow <davidgow@google.com> wrote:
>
> On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
> >
> > Add the basic structure of the test attribute API to KUnit, which can be
> > used to save and access test associated data.
> >
> > Add attributes.c and attributes.h to hold associated structs and functions
> > for the API.
> >
> > Create a struct that holds a variety of associated helper functions for
> > each test attribute. These helper functions will be used to get the
> > attribute value, convert the value to a string, and filter based on the
> > value. This struct is flexible by design to allow for attributes of
> > numerous types and contexts.
> >
> > Add a method to print test attributes in the format of "# [<test_name if
> > not suite>.]<attribute_name>: <attribute_value>".
> >
> > Example for a suite: "# speed: slow"
> >
> > Example for a test case: "# test_case.speed: very_slow"
> >
> > Use this method to report attributes in the KTAP output and _list_tests
> > output.
>
> Can we have a link to the draft KTAP spec for this?
>
Here is the link: https://docs.kernel.org/dev-tools/ktap.html. I will
also add a link in the next version.
> Also, by _list_tests, I assume we're talking about the
> kunit.action=list option. That's been an "internal" feature for the
> kunit script thus far, but kunit.py doesn't use this attribute info
> anywhere yet, apart from to print it out via --list_tests. Maybe we
> should make including the attributes optional, as the raw list of
> tests is currently more useful.
>
Oh this is an interesting idea. I could pass in the option to include
or not include attributes. Maybe the flag could then be
--list_tests_attrs? I could also remove the plain test information in
this new option. However, I do like seeing tests that have no set
attributes in the output. I'll work on this implementation.
> >
> > In test.h, add fields and associated helper functions to test cases and
> > suites to hold user-inputted test attributes.
> >
> > Signed-off-by: Rae Moar <rmoar@google.com>
> > ---
> > include/kunit/attributes.h | 19 +++++++++++
> > include/kunit/test.h | 33 +++++++++++++++++++
> > lib/kunit/Makefile | 3 +-
> > lib/kunit/attributes.c | 65 ++++++++++++++++++++++++++++++++++++++
> > lib/kunit/executor.c | 10 +++++-
> > lib/kunit/test.c | 17 ++++++----
> > 6 files changed, 138 insertions(+), 9 deletions(-)
> > create mode 100644 include/kunit/attributes.h
> > create mode 100644 lib/kunit/attributes.c
> >
> > diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
> > new file mode 100644
> > index 000000000000..9fcd184cce36
> > --- /dev/null
> > +++ b/include/kunit/attributes.h
> > @@ -0,0 +1,19 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * KUnit API to save and access test attributes
> > + *
> > + * Copyright (C) 2023, Google LLC.
> > + * Author: Rae Moar <rmoar@google.com>
> > + */
> > +
> > +#ifndef _KUNIT_ATTRIBUTES_H
> > +#define _KUNIT_ATTRIBUTES_H
> > +
> > +/*
> > + * Print all test attributes for a test case or suite.
> > + * Output format for test cases: "# <test_name>.<attribute>: <value>"
> > + * Output format for test suites: "# <attribute>: <value>"
> > + */
> > +void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
> > +
> > +#endif /* _KUNIT_ATTRIBUTES_H */
> > diff --git a/include/kunit/test.h b/include/kunit/test.h
> > index 23120d50499e..1fc9155988e9 100644
> > --- a/include/kunit/test.h
> > +++ b/include/kunit/test.h
> > @@ -63,12 +63,16 @@ enum kunit_status {
> > KUNIT_SKIPPED,
> > };
> >
> > +/* Holds attributes for each test case and suite */
> > +struct kunit_attributes {};
>
> This is a placeholder as attributes won't be included until patch 2, right?
Yes, this is just a placeholder until the next patch. Should I add a
temporary comment?
>
> > +
> > /**
> > * struct kunit_case - represents an individual test case.
> > *
> > * @run_case: the function representing the actual test case.
> > * @name: the name of the test case.
> > * @generate_params: the generator function for parameterized tests.
> > + * @attr: the attributes associated with the test
> > *
> > * A test case is a function with the signature,
> > * ``void (*)(struct kunit *)``
> > @@ -104,6 +108,7 @@ struct kunit_case {
> > void (*run_case)(struct kunit *test);
> > const char *name;
> > const void* (*generate_params)(const void *prev, char *desc);
> > + struct kunit_attributes attr;
> >
> > /* private: internal use only. */
> > enum kunit_status status;
> > @@ -133,6 +138,18 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> > */
> > #define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
> >
> > +/**
> > + * KUNIT_CASE_ATTR - A helper for creating a &struct kunit_case
> > + * with attributes
> > + *
> > + * @test_name: a reference to a test case function.
> > + * @attributes: a reference to a struct kunit_attributes object containing
> > + * test attributes
> > + */
> > +#define KUNIT_CASE_ATTR(test_name, attributes) \
> > + { .run_case = test_name, .name = #test_name, \
> > + .attr = attributes }
> > +
>
> This is a bit awkward, as the attributes are either entirely unset
> (i.e., zero-initialised), or have to be specified here. I assume the
> plan is to use designated initialisers, e.g.:
> KUNIT_CASE_ATTR(foo_test, {.speed = KUNIT_SPEED_SLOW, .flaky = true}).
>
> That at least makes it reasonably user-friendly, though the whole need
> to make sure zero-initialised values are the defaults is a bit icky.
>
Yeah there seem to be positives and negatives to this approach. If we
allow attributes to be uninitialized, the user experience is slightly
better and it is evident if the attribute is set or not.
However, this results in the negative implication that attributes with
a zero-equivalent value are treated as if they are unset. This results
in requiring enum values to have an unset option at the zero-index
(which I think is a fine solution) and it requires boolean attributes
to allow false values to act as the default (a little bit messier, but
a solution to this is switching booleans to enums when necessary).
>
>
> > /**
> > * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
> > *
> > @@ -154,6 +171,20 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> > { .run_case = test_name, .name = #test_name, \
> > .generate_params = gen_params }
> >
> > +/**
> > + * KUNIT_CASE_PARAM_ATTR - A helper for creating a parameterized &struct
> > + * kunit_case with attributes
> > + *
> > + * @test_name: a reference to a test case function.
> > + * @gen_params: a reference to a parameter generator function.
> > + * @attributes: a reference to a struct kunit_attributes object containing
> > + * test attributes
> > + */
> > +#define KUNIT_CASE_PARAM_ATTR(test_name, gen_params, attributes) \
> > + { .run_case = test_name, .name = #test_name, \
> > + .generate_params = gen_params, \
> > + .attr = attributes }
> > +
> > /**
> > * struct kunit_suite - describes a related collection of &struct kunit_case
> > *
> > @@ -163,6 +194,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> > * @init: called before every test case.
> > * @exit: called after every test case.
> > * @test_cases: a null terminated array of test cases.
> > + * @attr: the attributes associated with the test suite
> > *
> > * A kunit_suite is a collection of related &struct kunit_case s, such that
> > * @init is called before every test case and @exit is called after every
> > @@ -182,6 +214,7 @@ struct kunit_suite {
> > int (*init)(struct kunit *test);
> > void (*exit)(struct kunit *test);
> > struct kunit_case *test_cases;
> > + struct kunit_attributes attr;
> >
> > /* private: internal use only */
> > char status_comment[KUNIT_STATUS_COMMENT_SIZE];
> > diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile
> > index cb417f504996..46f75f23dfe4 100644
> > --- a/lib/kunit/Makefile
> > +++ b/lib/kunit/Makefile
> > @@ -6,7 +6,8 @@ kunit-objs += test.o \
> > string-stream.o \
> > assert.o \
> > try-catch.o \
> > - executor.o
> > + executor.o \
> > + attributes.o
> >
> > ifeq ($(CONFIG_KUNIT_DEBUGFS),y)
> > kunit-objs += debugfs.o
> > diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
> > new file mode 100644
> > index 000000000000..0ea641be795f
> > --- /dev/null
> > +++ b/lib/kunit/attributes.c
> > @@ -0,0 +1,65 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * KUnit API to save and access test attributes
> > + *
> > + * Copyright (C) 2023, Google LLC.
> > + * Author: Rae Moar <rmoar@google.com>
> > + */
> > +
> > +#include <kunit/test.h>
> > +#include <kunit/attributes.h>
> > +
> > +/**
> > + * struct kunit_attr - represents a test attribute and holds flexible
> > + * helper functions to interact with attribute.
> > + *
> > + * @name: name of test attribute, eg. speed
> > + * @get_attr: function to return attribute value given a test
> > + * @to_string: function to return string representation of given
> > + * attribute value
> > + * @filter: function to indicate whether a given attribute value passes a
> > + * filter
> > + */
> > +struct kunit_attr {
> > + const char *name;
> > + void *(*get_attr)(void *test_or_suite, bool is_test);
> > + const char *(*to_string)(void *attr, bool *to_free);
> > + int (*filter)(void *attr, const char *input, int *err);
> > + void *attr_default;
> > +};
> > +
> > +/* List of all Test Attributes */
> > +
> > +static struct kunit_attr kunit_attr_list[1] = {};
> > +
> > +/* Helper Functions to Access Attributes */
> > +
> > +void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level)
> > +{
> > + int i;
> > + bool to_free;
> > + void *attr;
> > + const char *attr_name, *attr_str;
> > + struct kunit_suite *suite = is_test ? NULL : test_or_suite;
> > + struct kunit_case *test = is_test ? test_or_suite : NULL;
> > +
> > + for (i = 0; i < ARRAY_SIZE(kunit_attr_list); i++) {
> > + attr = kunit_attr_list[i].get_attr(test_or_suite, is_test);
> > + if (attr) {
> > + attr_name = kunit_attr_list[i].name;
> > + attr_str = kunit_attr_list[i].to_string(attr, &to_free);
> > + if (test) {
> > + kunit_log(KERN_INFO, test, "%*s# %s.%s: %s",
> > + KUNIT_INDENT_LEN * test_level, "", test->name,
> > + attr_name, attr_str);
> > + } else {
> > + kunit_log(KERN_INFO, suite, "%*s# %s: %s",
> > + KUNIT_INDENT_LEN * test_level, "", attr_name, attr_str);
> > + }
> > +
> > + /* Free to_string of attribute if needed */
> > + if (to_free)
> > + kfree(attr_str);
> > + }
> > + }
> > +}
> > diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
> > index 74982b83707c..767a84e32f06 100644
> > --- a/lib/kunit/executor.c
> > +++ b/lib/kunit/executor.c
> > @@ -2,6 +2,7 @@
> >
> > #include <linux/reboot.h>
> > #include <kunit/test.h>
> > +#include <kunit/attributes.h>
> > #include <linux/glob.h>
> > #include <linux/moduleparam.h>
> >
> > @@ -180,10 +181,17 @@ static void kunit_exec_list_tests(struct suite_set *suite_set)
> > /* Hack: print a ktap header so kunit.py can find the start of KUnit output. */
> > pr_info("KTAP version 1\n");
> >
> > - for (suites = suite_set->start; suites < suite_set->end; suites++)
> > + for (suites = suite_set->start; suites < suite_set->end; suites++) {
> > + /* Print suite name and suite attributes */
> > + pr_info("%s\n", (*suites)->name);
> > + kunit_print_attr((void *)(*suites), false, 0);
> > +
> > + /* Print test case name and attributes in suite */
> > kunit_suite_for_each_test_case((*suites), test_case) {
> > pr_info("%s.%s\n", (*suites)->name, test_case->name);
> > + kunit_print_attr((void *)test_case, true, 0);
> > }
> > + }
> > }
> >
> > int kunit_run_all_tests(void)
> > diff --git a/lib/kunit/test.c b/lib/kunit/test.c
> > index 84e4666555c9..9ee55139ecd1 100644
> > --- a/lib/kunit/test.c
> > +++ b/lib/kunit/test.c
> > @@ -9,6 +9,7 @@
> > #include <kunit/resource.h>
> > #include <kunit/test.h>
> > #include <kunit/test-bug.h>
> > +#include <kunit/attributes.h>
> > #include <linux/kernel.h>
> > #include <linux/module.h>
> > #include <linux/moduleparam.h>
> > @@ -168,6 +169,13 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite)
> > }
> > EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases);
> >
> > +/* Currently supported test levels */
> > +enum {
> > + KUNIT_LEVEL_SUITE = 0,
> > + KUNIT_LEVEL_CASE,
> > + KUNIT_LEVEL_CASE_PARAM,
> > +};
> > +
> > static void kunit_print_suite_start(struct kunit_suite *suite)
> > {
> > /*
> > @@ -181,17 +189,11 @@ static void kunit_print_suite_start(struct kunit_suite *suite)
> > pr_info(KUNIT_SUBTEST_INDENT "KTAP version 1\n");
> > pr_info(KUNIT_SUBTEST_INDENT "# Subtest: %s\n",
> > suite->name);
> > + kunit_print_attr((void *)suite, false, KUNIT_LEVEL_CASE);
> > pr_info(KUNIT_SUBTEST_INDENT "1..%zd\n",
> > kunit_suite_num_test_cases(suite));
> > }
> >
> > -/* Currently supported test levels */
> > -enum {
> > - KUNIT_LEVEL_SUITE = 0,
> > - KUNIT_LEVEL_CASE,
> > - KUNIT_LEVEL_CASE_PARAM,
> > -};
> > -
> > static void kunit_print_ok_not_ok(struct kunit *test,
> > unsigned int test_level,
> > enum kunit_status status,
> > @@ -651,6 +653,7 @@ int kunit_run_tests(struct kunit_suite *suite)
> > }
> > }
> >
> > + kunit_print_attr((void *)test_case, true, KUNIT_LEVEL_CASE);
> >
> > kunit_print_test_stats(&test, param_stats);
> >
> > --
> > 2.41.0.162.gfafddb0af9-goog
> >
^ permalink raw reply [flat|nested] 21+ messages in thread
* [RFC v1 2/6] kunit: Add speed attribute
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
2023-06-10 0:51 ` [RFC v1 1/6] kunit: Add test attributes API structure Rae Moar
@ 2023-06-10 0:51 ` Rae Moar
2023-06-10 8:29 ` David Gow
2023-06-10 0:51 ` [RFC v1 3/6] kunit: Add ability to filter attributes Rae Moar
` (4 subsequent siblings)
6 siblings, 1 reply; 21+ messages in thread
From: Rae Moar @ 2023-06-10 0:51 UTC (permalink / raw)
To: shuah, davidgow, dlatypov, brendan.higgins
Cc: linux-kselftest, kunit-dev, linux-kernel, keescook,
linux-hardening, jstultz, tglx, sboyd, Rae Moar
Add speed attribute to the test attribute API. This attribute will allow
users to mark tests with a category of speed.
Currently the categories of speed proposed are: fast, normal, slow, and
very_slow. These are outlined in the enum kunit_speed. Note the speed
attribute can also be left as unset and then, will act as the default which
is "normal", during filtering.
Note speed is intended to be marked based on relative speeds rather than
quantitative speeds of KUnit tests. This is because tests may run on
various architectures at different speeds.
Add the macro KUNIT_CASE_SLOW to set a test as slow, as this is likely a
common use of the attributes API.
Add an example of marking a slow test to kunit-example-test.c.
Signed-off-by: Rae Moar <rmoar@google.com>
---
include/kunit/test.h | 31 ++++++++++++++++++++++-
lib/kunit/attributes.c | 45 +++++++++++++++++++++++++++++++++-
lib/kunit/kunit-example-test.c | 9 +++++++
3 files changed, 83 insertions(+), 2 deletions(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h
index 1fc9155988e9..3d684723ae57 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -63,8 +63,26 @@ enum kunit_status {
KUNIT_SKIPPED,
};
+/* Attribute struct/enum definitions */
+
+/*
+ * Speed Attribute is stored as an enum and separated into categories of
+ * speed: very_slowm, slow, normal, and fast. These speeds are relative
+ * to other KUnit tests.
+ */
+enum kunit_speed {
+ KUNIT_SPEED_UNSET,
+ KUNIT_SPEED_VERY_SLOW,
+ KUNIT_SPEED_SLOW,
+ KUNIT_SPEED_NORMAL,
+ KUNIT_SPEED_FAST,
+ KUNIT_SPEED_MAX = KUNIT_SPEED_FAST,
+};
+
/* Holds attributes for each test case and suite */
-struct kunit_attributes {};
+struct kunit_attributes {
+ enum kunit_speed speed;
+};
/**
* struct kunit_case - represents an individual test case.
@@ -150,6 +168,17 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
{ .run_case = test_name, .name = #test_name, \
.attr = attributes }
+/**
+ * KUNIT_CASE_SLOW - A helper for creating a &struct kunit_case
+ * with the slow attribute
+ *
+ * @test_name: a reference to a test case function.
+ */
+
+#define KUNIT_CASE_SLOW(test_name) \
+ { .run_case = test_name, .name = #test_name, \
+ .attr.speed = KUNIT_SPEED_SLOW }
+
/**
* KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
*
diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
index 0ea641be795f..e17889f94693 100644
--- a/lib/kunit/attributes.c
+++ b/lib/kunit/attributes.c
@@ -28,9 +28,52 @@ struct kunit_attr {
void *attr_default;
};
+/* String Lists for enum Attributes */
+
+static const char * const speed_str_list[] = {"unset", "very_slow", "slow", "normal", "fast"};
+
+/* To String Methods */
+
+static const char *attr_enum_to_string(void *attr, const char * const str_list[], bool *to_free)
+{
+ long val = (long)attr;
+
+ *to_free = false;
+ if (!val)
+ return NULL;
+ return str_list[val];
+}
+
+static const char *attr_speed_to_string(void *attr, bool *to_free)
+{
+ return attr_enum_to_string(attr, speed_str_list, to_free);
+}
+
+/* Get Attribute Methods */
+
+static void *attr_speed_get(void *test_or_suite, bool is_test)
+{
+ struct kunit_suite *suite = is_test ? NULL : test_or_suite;
+ struct kunit_case *test = is_test ? test_or_suite : NULL;
+
+ if (test)
+ return ((void *) test->attr.speed);
+ else
+ return ((void *) suite->attr.speed);
+}
+
+/* Attribute Struct Definitions */
+
+static const struct kunit_attr speed_attr = {
+ .name = "speed",
+ .get_attr = attr_speed_get,
+ .to_string = attr_speed_to_string,
+ .attr_default = (void *)KUNIT_SPEED_NORMAL,
+};
+
/* List of all Test Attributes */
-static struct kunit_attr kunit_attr_list[1] = {};
+static struct kunit_attr kunit_attr_list[1] = {speed_attr};
/* Helper Functions to Access Attributes */
diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c
index b69b689ea850..01a769f35e1d 100644
--- a/lib/kunit/kunit-example-test.c
+++ b/lib/kunit/kunit-example-test.c
@@ -220,6 +220,14 @@ static void example_params_test(struct kunit *test)
KUNIT_EXPECT_EQ(test, param->value % param->value, 0);
}
+/*
+ * This test should always pass. Can be used to practice filtering attributes.
+ */
+static void example_slow_test(struct kunit *test)
+{
+ KUNIT_EXPECT_EQ(test, 1 + 1, 2);
+}
+
/*
* Here we make a list of all the test cases we want to add to the test suite
* below.
@@ -237,6 +245,7 @@ static struct kunit_case example_test_cases[] = {
KUNIT_CASE(example_all_expect_macros_test),
KUNIT_CASE(example_static_stub_test),
KUNIT_CASE_PARAM(example_params_test, example_gen_params),
+ KUNIT_CASE_SLOW(example_slow_test),
{}
};
--
2.41.0.162.gfafddb0af9-goog
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [RFC v1 2/6] kunit: Add speed attribute
2023-06-10 0:51 ` [RFC v1 2/6] kunit: Add speed attribute Rae Moar
@ 2023-06-10 8:29 ` David Gow
2023-06-13 20:37 ` Rae Moar
0 siblings, 1 reply; 21+ messages in thread
From: David Gow @ 2023-06-10 8:29 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
[-- Attachment #1: Type: text/plain, Size: 7003 bytes --]
On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
>
> Add speed attribute to the test attribute API. This attribute will allow
> users to mark tests with a category of speed.
>
> Currently the categories of speed proposed are: fast, normal, slow, and
> very_slow. These are outlined in the enum kunit_speed. Note the speed
> attribute can also be left as unset and then, will act as the default which
> is "normal", during filtering.
Do we need both "fast" and "normal". KUnit tests are normally very
fast: I'm not sure there's a way to make them faster enough to make a
separate category here useful.
>
> Note speed is intended to be marked based on relative speeds rather than
> quantitative speeds of KUnit tests. This is because tests may run on
> various architectures at different speeds.
My rule of thumb here is that a test is slow if it takes more than a
"trivial" amount of time (<1s), regardless of the machine it's running
on.
While the actual speed taken varies a lot (the time_test_cases take ~3
seconds on most fast, modern machines, even under something like qemu,
but ~15 minutes on an old 486), it's the idea that a test is doing
some significant amount of work (loops over many thousands or millions
of entries, etc) that pretty comfortably divides these into "normal"
and "slow".
Most tests run very, very quickly on even very slow systems, as all
they're doing is checking the result of one or two trivial
calculations or functions.
> Add the macro KUNIT_CASE_SLOW to set a test as slow, as this is likely a
> common use of the attributes API.
I'd ask if we need a KUNIT_CASE_VERY_SLOW() as well, but let's leave
that until we have something which uses it.
> Add an example of marking a slow test to kunit-example-test.c.
>
> Signed-off-by: Rae Moar <rmoar@google.com>
> ---
> include/kunit/test.h | 31 ++++++++++++++++++++++-
> lib/kunit/attributes.c | 45 +++++++++++++++++++++++++++++++++-
> lib/kunit/kunit-example-test.c | 9 +++++++
> 3 files changed, 83 insertions(+), 2 deletions(-)
>
> diff --git a/include/kunit/test.h b/include/kunit/test.h
> index 1fc9155988e9..3d684723ae57 100644
> --- a/include/kunit/test.h
> +++ b/include/kunit/test.h
> @@ -63,8 +63,26 @@ enum kunit_status {
> KUNIT_SKIPPED,
> };
>
> +/* Attribute struct/enum definitions */
> +
> +/*
> + * Speed Attribute is stored as an enum and separated into categories of
> + * speed: very_slowm, slow, normal, and fast. These speeds are relative
> + * to other KUnit tests.
> + */
> +enum kunit_speed {
> + KUNIT_SPEED_UNSET,
> + KUNIT_SPEED_VERY_SLOW,
> + KUNIT_SPEED_SLOW,
> + KUNIT_SPEED_NORMAL,
> + KUNIT_SPEED_FAST,
> + KUNIT_SPEED_MAX = KUNIT_SPEED_FAST,
> +};
Question: Does it make sense to have these in this order: slow ->
fast? I think it does ("speed>slow" seems more correct than
"speed<slow"), but it'd be the other way round if we wanted to call
this, e.g., size instead of speed.
That being said, if it went the other way, we could rely on the fact
that the default is fast, and not need a separate "unset" default...
> +
> /* Holds attributes for each test case and suite */
> -struct kunit_attributes {};
> +struct kunit_attributes {
> + enum kunit_speed speed;
> +};
>
> /**
> * struct kunit_case - represents an individual test case.
> @@ -150,6 +168,17 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> { .run_case = test_name, .name = #test_name, \
> .attr = attributes }
>
> +/**
> + * KUNIT_CASE_SLOW - A helper for creating a &struct kunit_case
> + * with the slow attribute
> + *
> + * @test_name: a reference to a test case function.
> + */
> +
> +#define KUNIT_CASE_SLOW(test_name) \
> + { .run_case = test_name, .name = #test_name, \
> + .attr.speed = KUNIT_SPEED_SLOW }
> +
> /**
> * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
> *
> diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
> index 0ea641be795f..e17889f94693 100644
> --- a/lib/kunit/attributes.c
> +++ b/lib/kunit/attributes.c
> @@ -28,9 +28,52 @@ struct kunit_attr {
> void *attr_default;
> };
>
> +/* String Lists for enum Attributes */
> +
> +static const char * const speed_str_list[] = {"unset", "very_slow", "slow", "normal", "fast"};
> +
> +/* To String Methods */
> +
> +static const char *attr_enum_to_string(void *attr, const char * const str_list[], bool *to_free)
> +{
> + long val = (long)attr;
> +
> + *to_free = false;
> + if (!val)
> + return NULL;
> + return str_list[val];
> +}
> +
> +static const char *attr_speed_to_string(void *attr, bool *to_free)
> +{
> + return attr_enum_to_string(attr, speed_str_list, to_free);
> +}
> +
> +/* Get Attribute Methods */
> +
> +static void *attr_speed_get(void *test_or_suite, bool is_test)
> +{
> + struct kunit_suite *suite = is_test ? NULL : test_or_suite;
> + struct kunit_case *test = is_test ? test_or_suite : NULL;
> +
> + if (test)
> + return ((void *) test->attr.speed);
> + else
> + return ((void *) suite->attr.speed);
> +}
> +
> +/* Attribute Struct Definitions */
> +
> +static const struct kunit_attr speed_attr = {
> + .name = "speed",
> + .get_attr = attr_speed_get,
> + .to_string = attr_speed_to_string,
> + .attr_default = (void *)KUNIT_SPEED_NORMAL,
> +};
> +
> /* List of all Test Attributes */
>
> -static struct kunit_attr kunit_attr_list[1] = {};
> +static struct kunit_attr kunit_attr_list[1] = {speed_attr};
Nit: Can we remove the hardcoded [1] here, and let the compiler do this for us?
>
> /* Helper Functions to Access Attributes */
>
> diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c
> index b69b689ea850..01a769f35e1d 100644
> --- a/lib/kunit/kunit-example-test.c
> +++ b/lib/kunit/kunit-example-test.c
> @@ -220,6 +220,14 @@ static void example_params_test(struct kunit *test)
> KUNIT_EXPECT_EQ(test, param->value % param->value, 0);
> }
>
> +/*
> + * This test should always pass. Can be used to practice filtering attributes.
> + */
> +static void example_slow_test(struct kunit *test)
> +{
> + KUNIT_EXPECT_EQ(test, 1 + 1, 2);
> +}
Would we want to actually make this test slow? e.g. introduce a delay
or a big loop or something.
Probably not (I think it'd be more irritating than illuminating), but
maybe worth thinking of.
> +
> /*
> * Here we make a list of all the test cases we want to add to the test suite
> * below.
> @@ -237,6 +245,7 @@ static struct kunit_case example_test_cases[] = {
> KUNIT_CASE(example_all_expect_macros_test),
> KUNIT_CASE(example_static_stub_test),
> KUNIT_CASE_PARAM(example_params_test, example_gen_params),
> + KUNIT_CASE_SLOW(example_slow_test),
> {}
> };
>
> --
> 2.41.0.162.gfafddb0af9-goog
>
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4003 bytes --]
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [RFC v1 2/6] kunit: Add speed attribute
2023-06-10 8:29 ` David Gow
@ 2023-06-13 20:37 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:37 UTC (permalink / raw)
To: David Gow
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 4:29 AM David Gow <davidgow@google.com> wrote:
>
> On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
> >
> > Add speed attribute to the test attribute API. This attribute will allow
> > users to mark tests with a category of speed.
> >
> > Currently the categories of speed proposed are: fast, normal, slow, and
> > very_slow. These are outlined in the enum kunit_speed. Note the speed
> > attribute can also be left as unset and then, will act as the default which
> > is "normal", during filtering.
>
> Do we need both "fast" and "normal". KUnit tests are normally very
> fast: I'm not sure there's a way to make them faster enough to make a
> separate category here useful.
>
This is a really interesting discussion. I see your point here. I will
remove the "fast" category unless there are any objections.
> >
> > Note speed is intended to be marked based on relative speeds rather than
> > quantitative speeds of KUnit tests. This is because tests may run on
> > various architectures at different speeds.
>
> My rule of thumb here is that a test is slow if it takes more than a
> "trivial" amount of time (<1s), regardless of the machine it's running
> on.
>
> While the actual speed taken varies a lot (the time_test_cases take ~3
> seconds on most fast, modern machines, even under something like qemu,
> but ~15 minutes on an old 486), it's the idea that a test is doing
> some significant amount of work (loops over many thousands or millions
> of entries, etc) that pretty comfortably divides these into "normal"
> and "slow".
>
> Most tests run very, very quickly on even very slow systems, as all
> they're doing is checking the result of one or two trivial
> calculations or functions.
This seems like a great rule of thumb to add to the documentation.
>
> > Add the macro KUNIT_CASE_SLOW to set a test as slow, as this is likely a
> > common use of the attributes API.
>
> I'd ask if we need a KUNIT_CASE_VERY_SLOW() as well, but let's leave
> that until we have something which uses it.
>
I would be happy to add this once something uses the "very_slow" attribute.
>
> > Add an example of marking a slow test to kunit-example-test.c.
> >
> > Signed-off-by: Rae Moar <rmoar@google.com>
> > ---
> > include/kunit/test.h | 31 ++++++++++++++++++++++-
> > lib/kunit/attributes.c | 45 +++++++++++++++++++++++++++++++++-
> > lib/kunit/kunit-example-test.c | 9 +++++++
> > 3 files changed, 83 insertions(+), 2 deletions(-)
> >
> > diff --git a/include/kunit/test.h b/include/kunit/test.h
> > index 1fc9155988e9..3d684723ae57 100644
> > --- a/include/kunit/test.h
> > +++ b/include/kunit/test.h
> > @@ -63,8 +63,26 @@ enum kunit_status {
> > KUNIT_SKIPPED,
> > };
> >
> > +/* Attribute struct/enum definitions */
> > +
> > +/*
> > + * Speed Attribute is stored as an enum and separated into categories of
> > + * speed: very_slowm, slow, normal, and fast. These speeds are relative
> > + * to other KUnit tests.
> > + */
> > +enum kunit_speed {
> > + KUNIT_SPEED_UNSET,
> > + KUNIT_SPEED_VERY_SLOW,
> > + KUNIT_SPEED_SLOW,
> > + KUNIT_SPEED_NORMAL,
> > + KUNIT_SPEED_FAST,
> > + KUNIT_SPEED_MAX = KUNIT_SPEED_FAST,
> > +};
>
> Question: Does it make sense to have these in this order: slow ->
> fast? I think it does ("speed>slow" seems more correct than
> "speed<slow"), but it'd be the other way round if we wanted to call
> this, e.g., size instead of speed.
>
> That being said, if it went the other way, we could rely on the fact
> that the default is fast, and not need a separate "unset" default...
>
Oh interesting. I hadn't considered changing the order. To me
"speed>slow" seems a bit more intuitive but I can see how "speed<slow"
would also make sense. Hmm this is an interesting idea. Let me know
if anyone has an opinion here else I will most likely keep it this
order.
> > +
> > /* Holds attributes for each test case and suite */
> > -struct kunit_attributes {};
> > +struct kunit_attributes {
> > + enum kunit_speed speed;
> > +};
> >
> > /**
> > * struct kunit_case - represents an individual test case.
> > @@ -150,6 +168,17 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
> > { .run_case = test_name, .name = #test_name, \
> > .attr = attributes }
> >
> > +/**
> > + * KUNIT_CASE_SLOW - A helper for creating a &struct kunit_case
> > + * with the slow attribute
> > + *
> > + * @test_name: a reference to a test case function.
> > + */
> > +
> > +#define KUNIT_CASE_SLOW(test_name) \
> > + { .run_case = test_name, .name = #test_name, \
> > + .attr.speed = KUNIT_SPEED_SLOW }
> > +
> > /**
> > * KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
> > *
> > diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
> > index 0ea641be795f..e17889f94693 100644
> > --- a/lib/kunit/attributes.c
> > +++ b/lib/kunit/attributes.c
> > @@ -28,9 +28,52 @@ struct kunit_attr {
> > void *attr_default;
> > };
> >
> > +/* String Lists for enum Attributes */
> > +
> > +static const char * const speed_str_list[] = {"unset", "very_slow", "slow", "normal", "fast"};
> > +
> > +/* To String Methods */
> > +
> > +static const char *attr_enum_to_string(void *attr, const char * const str_list[], bool *to_free)
> > +{
> > + long val = (long)attr;
> > +
> > + *to_free = false;
> > + if (!val)
> > + return NULL;
> > + return str_list[val];
> > +}
> > +
> > +static const char *attr_speed_to_string(void *attr, bool *to_free)
> > +{
> > + return attr_enum_to_string(attr, speed_str_list, to_free);
> > +}
> > +
> > +/* Get Attribute Methods */
> > +
> > +static void *attr_speed_get(void *test_or_suite, bool is_test)
> > +{
> > + struct kunit_suite *suite = is_test ? NULL : test_or_suite;
> > + struct kunit_case *test = is_test ? test_or_suite : NULL;
> > +
> > + if (test)
> > + return ((void *) test->attr.speed);
> > + else
> > + return ((void *) suite->attr.speed);
> > +}
> > +
> > +/* Attribute Struct Definitions */
> > +
> > +static const struct kunit_attr speed_attr = {
> > + .name = "speed",
> > + .get_attr = attr_speed_get,
> > + .to_string = attr_speed_to_string,
> > + .attr_default = (void *)KUNIT_SPEED_NORMAL,
> > +};
> > +
> > /* List of all Test Attributes */
> >
> > -static struct kunit_attr kunit_attr_list[1] = {};
> > +static struct kunit_attr kunit_attr_list[1] = {speed_attr};
>
> Nit: Can we remove the hardcoded [1] here, and let the compiler do this for us?
Yes, I will change this out.
>
> >
> > /* Helper Functions to Access Attributes */
> >
> > diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c
> > index b69b689ea850..01a769f35e1d 100644
> > --- a/lib/kunit/kunit-example-test.c
> > +++ b/lib/kunit/kunit-example-test.c
> > @@ -220,6 +220,14 @@ static void example_params_test(struct kunit *test)
> > KUNIT_EXPECT_EQ(test, param->value % param->value, 0);
> > }
> >
> > +/*
> > + * This test should always pass. Can be used to practice filtering attributes.
> > + */
> > +static void example_slow_test(struct kunit *test)
> > +{
> > + KUNIT_EXPECT_EQ(test, 1 + 1, 2);
> > +}
>
> Would we want to actually make this test slow? e.g. introduce a delay
> or a big loop or something.
> Probably not (I think it'd be more irritating than illuminating), but
> maybe worth thinking of.
>
I'm thinking not but it would make the concept clearer. I would
definitely change this if it is wanted.
Thanks!
-Rae
>
> > +
> > /*
> > * Here we make a list of all the test cases we want to add to the test suite
> > * below.
> > @@ -237,6 +245,7 @@ static struct kunit_case example_test_cases[] = {
> > KUNIT_CASE(example_all_expect_macros_test),
> > KUNIT_CASE(example_static_stub_test),
> > KUNIT_CASE_PARAM(example_params_test, example_gen_params),
> > + KUNIT_CASE_SLOW(example_slow_test),
> > {}
> > };
> >
> > --
> > 2.41.0.162.gfafddb0af9-goog
> >
^ permalink raw reply [flat|nested] 21+ messages in thread
* [RFC v1 3/6] kunit: Add ability to filter attributes
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
2023-06-10 0:51 ` [RFC v1 1/6] kunit: Add test attributes API structure Rae Moar
2023-06-10 0:51 ` [RFC v1 2/6] kunit: Add speed attribute Rae Moar
@ 2023-06-10 0:51 ` Rae Moar
2023-06-10 8:29 ` David Gow
2023-06-13 20:26 ` Kees Cook
2023-06-10 0:51 ` [RFC v1 4/6] kunit: tool: Add command line interface to filter and report attributes Rae Moar
` (3 subsequent siblings)
6 siblings, 2 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-10 0:51 UTC (permalink / raw)
To: shuah, davidgow, dlatypov, brendan.higgins
Cc: linux-kselftest, kunit-dev, linux-kernel, keescook,
linux-hardening, jstultz, tglx, sboyd, Rae Moar
Add filtering of test attributes. Users can filter tests using a
module_param_array called "filter". This functionality will be added to
kunit.py in the next patch.
The filters will be imputed in the format:
"<attribute_name><operation><attribute_value>"
Example: "speed>slow"
Operations include: >, <, >=, <=, !=, and =. These operations do not need
to act the same for every attribute.
Add method to parse inputted filters.
Add the process of filtering tests based on attributes. The process of
filtering follows these rules:
A test case with a set attribute overrides its parent suite's attribute
during filtering.
Also, if both the test case attribute and suite attribute are unset the
test acts as the default attribute value during filtering.
Finally, add a "filter" method for the speed attribute to parse and compare
enum values of kunit_speed.
Signed-off-by: Rae Moar <rmoar@google.com>
---
include/kunit/attributes.h | 22 +++++
lib/kunit/attributes.c | 172 +++++++++++++++++++++++++++++++++++++
lib/kunit/executor.c | 79 +++++++++++++----
lib/kunit/executor_test.c | 8 +-
4 files changed, 258 insertions(+), 23 deletions(-)
diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
index 9fcd184cce36..bca60d1181bb 100644
--- a/include/kunit/attributes.h
+++ b/include/kunit/attributes.h
@@ -9,6 +9,15 @@
#ifndef _KUNIT_ATTRIBUTES_H
#define _KUNIT_ATTRIBUTES_H
+/*
+ * struct kunit_attr_filter - representation of attributes filter with the
+ * attribute object and string input
+ */
+struct kunit_attr_filter {
+ struct kunit_attr *attr;
+ char *input;
+};
+
/*
* Print all test attributes for a test case or suite.
* Output format for test cases: "# <test_name>.<attribute>: <value>"
@@ -16,4 +25,17 @@
*/
void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
+/*
+ * Parse attributes filter input and return an object containing the attribute
+ * object and the string input.
+ */
+struct kunit_attr_filter kunit_parse_filter_attr(char *input, int *err);
+
+
+/*
+ * Returns a copy of the suite containing only tests that pass the filter.
+ */
+struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
+ struct kunit_attr_filter filter, int *err);
+
#endif /* _KUNIT_ATTRIBUTES_H */
diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
index e17889f94693..4f753a28e4ee 100644
--- a/lib/kunit/attributes.c
+++ b/lib/kunit/attributes.c
@@ -49,6 +49,66 @@ static const char *attr_speed_to_string(void *attr, bool *to_free)
return attr_enum_to_string(attr, speed_str_list, to_free);
}
+/* Filter Methods */
+
+static int int_filter(long val, const char *op, int input, int *err)
+{
+ if (!strncmp(op, "<=", 2))
+ return (val <= input);
+ else if (!strncmp(op, ">=", 2))
+ return (val >= input);
+ else if (!strncmp(op, "!=", 2))
+ return (val != input);
+ else if (!strncmp(op, ">", 1))
+ return (val > input);
+ else if (!strncmp(op, "<", 1))
+ return (val < input);
+ else if (!strncmp(op, "=", 1))
+ return (val == input);
+ *err = -EINVAL;
+ pr_err("kunit executor: invalid filter operation: %s\n", op);
+ return false;
+}
+
+static int attr_enum_filter(void *attr, const char *input, int *err,
+ const char * const str_list[], int max)
+{
+ int i, j, input_int;
+ long test_val = (long)attr;
+ const char *input_val;
+
+ for (i = 0; input[i]; i++) {
+ if (!strchr("<!=>", input[i])) {
+ input_val = input + i;
+ break;
+ }
+ }
+
+ if (!input_val) {
+ *err = -EINVAL;
+ pr_err("kunit executor: filter operation not found: %s\n", input);
+ return false;
+ }
+
+ for (j = 0; j <= max; j++) {
+ if (!strcmp(input_val, str_list[j]))
+ input_int = j;
+ }
+
+ if (!input_int) {
+ *err = -EINVAL;
+ pr_err("kunit executor: invalid filter input: %s\n", input);
+ return false;
+ }
+
+ return int_filter(test_val, input, input_int, err);
+}
+
+static int attr_speed_filter(void *attr, const char *input, int *err)
+{
+ return attr_enum_filter(attr, input, err, speed_str_list, KUNIT_SPEED_MAX);
+}
+
/* Get Attribute Methods */
static void *attr_speed_get(void *test_or_suite, bool is_test)
@@ -68,6 +128,7 @@ static const struct kunit_attr speed_attr = {
.name = "speed",
.get_attr = attr_speed_get,
.to_string = attr_speed_to_string,
+ .filter = attr_speed_filter,
.attr_default = (void *)KUNIT_SPEED_NORMAL,
};
@@ -106,3 +167,114 @@ void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level
}
}
}
+
+/* Helper Functions to Filter Attributes */
+
+struct kunit_attr_filter kunit_parse_filter_attr(char *input, int *err)
+{
+ struct kunit_attr_filter filter;
+ int i, j, op_index = 0;
+ int attr_index = -1;
+ char op;
+
+ /* Parse input until operation */
+ for (i = 0; input[i]; i++) {
+ if (strchr("<>!=", input[i])) {
+ op_index = i;
+ break;
+ }
+ if (input[i] == ' ')
+ break;
+ }
+
+ if (!op_index) {
+ *err = -EINVAL;
+ pr_err("kunit executor: filter operation not found: %s\n", input);
+ return filter;
+ }
+
+ op = input[op_index];
+ input[op_index] = '\0';
+
+ /* Find associated kunit_attr object */
+ for (j = 0; j < ARRAY_SIZE(kunit_attr_list); j++) {
+ if (!strcmp(input, kunit_attr_list[j].name)) {
+ attr_index = j;
+ break;
+ }
+ }
+
+ input[op_index] = op;
+ filter.input = input + op_index;
+
+ if (attr_index < 0) {
+ *err = -EINVAL;
+ pr_err("kunit executor: attribute not found: %s\n", input);
+ } else {
+ filter.attr = &kunit_attr_list[attr_index];
+ }
+
+ return filter;
+}
+
+struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
+ struct kunit_attr_filter filter, int *err)
+{
+ int n = 0;
+ struct kunit_case *filtered, *test_case;
+ struct kunit_suite *copy;
+ void *suite_val, *test_val;
+ bool suite_result, test_result, default_result;
+
+ /* Allocate memory for new copy of suite and list of test cases */
+ copy = kmemdup(suite, sizeof(*copy), GFP_KERNEL);
+ if (!copy)
+ return ERR_PTR(-ENOMEM);
+
+ kunit_suite_for_each_test_case(suite, test_case) { n++; }
+
+ filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL);
+ if (!filtered) {
+ kfree(copy);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ n = 0;
+
+ /* Save filtering result on default value */
+ default_result = filter.attr->filter(filter.attr->attr_default, filter.input, err);
+
+ /* Save suite attribute value and filtering result on that value */
+ suite_val = filter.attr->get_attr((void *)suite, false);
+ suite_result = filter.attr->filter(suite_val, filter.input, err);
+
+ /* For each test case, save test case if passes filtering. */
+ kunit_suite_for_each_test_case(suite, test_case) {
+ test_val = filter.attr->get_attr((void *) test_case, true);
+ test_result = filter.attr->filter(filter.attr->get_attr(test_case, true),
+ filter.input, err);
+ /*
+ * If attribute value of test case is set, filter on that value.
+ * If not, filter on suite value if set. If not, filter on
+ * default value.
+ */
+ if (test_val) {
+ if (test_result)
+ filtered[n++] = *test_case;
+ } else if (suite_val) {
+ if (suite_result)
+ filtered[n++] = *test_case;
+ } else if (default_result) {
+ filtered[n++] = *test_case;
+ }
+ }
+
+ if (n == 0) {
+ kfree(copy);
+ kfree(filtered);
+ return NULL;
+ }
+
+ copy->test_cases = filtered;
+ return copy;
+}
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index 767a84e32f06..c67657821eec 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -15,8 +15,12 @@ extern struct kunit_suite * const __kunit_suites_end[];
#if IS_BUILTIN(CONFIG_KUNIT)
+#define MAX_FILTERS 10 // Limit of number of attribute filters
static char *filter_glob_param;
static char *action_param;
+static int filter_count;
+static char *filter_param[MAX_FILTERS];
+
module_param_named(filter_glob, filter_glob_param, charp, 0);
MODULE_PARM_DESC(filter_glob,
@@ -26,15 +30,16 @@ MODULE_PARM_DESC(action,
"Changes KUnit executor behavior, valid values are:\n"
"<none>: run the tests like normal\n"
"'list' to list test names instead of running them.\n");
+module_param_array_named(filter, filter_param, charp, &filter_count, 0);
/* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */
-struct kunit_test_filter {
+struct kunit_glob_filter {
char *suite_glob;
char *test_glob;
};
/* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */
-static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,
+static void kunit_parse_filter_glob(struct kunit_glob_filter *parsed,
const char *filter_glob)
{
const int len = strlen(filter_glob);
@@ -56,7 +61,7 @@ static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,
/* Create a copy of suite with only tests that match test_glob. */
static struct kunit_suite *
-kunit_filter_tests(const struct kunit_suite *const suite, const char *test_glob)
+kunit_filter_glob_tests(const struct kunit_suite *const suite, const char *test_glob)
{
int n = 0;
struct kunit_case *filtered, *test_case;
@@ -110,12 +115,15 @@ static void kunit_free_suite_set(struct suite_set suite_set)
static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
const char *filter_glob,
+ char **filters,
+ int filter_count,
int *err)
{
- int i;
- struct kunit_suite **copy, *filtered_suite;
+ int i, j, k;
+ struct kunit_suite **copy, *filtered_suite, *new_filtered_suite;
struct suite_set filtered;
- struct kunit_test_filter filter;
+ struct kunit_glob_filter parsed_glob;
+ struct kunit_attr_filter parsed_filters[MAX_FILTERS];
const size_t max = suite_set->end - suite_set->start;
@@ -126,17 +134,49 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
return filtered;
}
- kunit_parse_filter_glob(&filter, filter_glob);
+ if (filter_glob)
+ kunit_parse_filter_glob(&parsed_glob, filter_glob);
- for (i = 0; &suite_set->start[i] != suite_set->end; i++) {
- if (!glob_match(filter.suite_glob, suite_set->start[i]->name))
- continue;
+ /* Parse attribute filters */
+ if (filter_count) {
+ for (j = 0; j < filter_count; j++) {
+ parsed_filters[j] = kunit_parse_filter_attr(filters[j], err);
+ if (*err)
+ return filtered;
+ }
+ }
- filtered_suite = kunit_filter_tests(suite_set->start[i], filter.test_glob);
- if (IS_ERR(filtered_suite)) {
- *err = PTR_ERR(filtered_suite);
- return filtered;
+ for (i = 0; &suite_set->start[i] != suite_set->end; i++) {
+ filtered_suite = suite_set->start[i];
+ if (filter_glob) {
+ if (!glob_match(parsed_glob.suite_glob, filtered_suite->name))
+ continue;
+ filtered_suite = kunit_filter_glob_tests(filtered_suite,
+ parsed_glob.test_glob);
+ if (IS_ERR(filtered_suite)) {
+ *err = PTR_ERR(filtered_suite);
+ return filtered;
+ }
}
+ if (filter_count) {
+ for (k = 0; k < filter_count; k++) {
+ new_filtered_suite = kunit_filter_attr_tests(filtered_suite,
+ parsed_filters[k], err);
+
+ /* Free previous copy of suite */
+ if (k > 0 || filter_glob)
+ kfree(filtered_suite);
+ filtered_suite = new_filtered_suite;
+
+ if (*err)
+ return filtered;
+ if (IS_ERR(filtered_suite)) {
+ *err = PTR_ERR(filtered_suite);
+ return filtered;
+ }
+ }
+ }
+
if (!filtered_suite)
continue;
@@ -144,8 +184,8 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
}
filtered.end = copy;
- kfree(filter.suite_glob);
- kfree(filter.test_glob);
+ kfree(parsed_glob.suite_glob);
+ kfree(parsed_glob.test_glob);
return filtered;
}
@@ -203,8 +243,9 @@ int kunit_run_all_tests(void)
goto out;
}
- if (filter_glob_param) {
- suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err);
+ if (filter_glob_param || filter_count) {
+ suite_set = kunit_filter_suites(&suite_set, filter_glob_param,
+ filter_param, filter_count, &err);
if (err) {
pr_err("kunit executor: error filtering suites: %d\n", err);
goto out;
@@ -218,7 +259,7 @@ int kunit_run_all_tests(void)
else
pr_err("kunit executor: unknown action '%s'\n", action_param);
- if (filter_glob_param) { /* a copy was made of each suite */
+ if (filter_glob_param || filter_count) { /* a copy was made of each suite */
kunit_free_suite_set(suite_set);
}
diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c
index ce6749af374d..4c8cb46857b2 100644
--- a/lib/kunit/executor_test.c
+++ b/lib/kunit/executor_test.c
@@ -24,7 +24,7 @@ static struct kunit_case dummy_test_cases[] = {
static void parse_filter_test(struct kunit *test)
{
- struct kunit_test_filter filter = {NULL, NULL};
+ struct kunit_glob_filter filter = {NULL, NULL};
kunit_parse_filter_glob(&filter, "suite");
KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite");
@@ -50,7 +50,7 @@ static void filter_suites_test(struct kunit *test)
subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
/* Want: suite1, suite2, NULL -> suite2, NULL */
- got = kunit_filter_suites(&suite_set, "suite2", &err);
+ got = kunit_filter_suites(&suite_set, "suite2", NULL, 0, &err);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
KUNIT_ASSERT_EQ(test, err, 0);
kfree_at_end(test, got.start);
@@ -74,7 +74,7 @@ static void filter_suites_test_glob_test(struct kunit *test)
subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
/* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */
- got = kunit_filter_suites(&suite_set, "suite2.test2", &err);
+ got = kunit_filter_suites(&suite_set, "suite2.test2", NULL, 0, &err);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
KUNIT_ASSERT_EQ(test, err, 0);
kfree_at_end(test, got.start);
@@ -100,7 +100,7 @@ static void filter_suites_to_empty_test(struct kunit *test)
subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases);
subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
- got = kunit_filter_suites(&suite_set, "not_found", &err);
+ got = kunit_filter_suites(&suite_set, "not_found", NULL, 0, &err);
KUNIT_ASSERT_EQ(test, err, 0);
kfree_at_end(test, got.start); /* just in case */
--
2.41.0.162.gfafddb0af9-goog
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [RFC v1 3/6] kunit: Add ability to filter attributes
2023-06-10 0:51 ` [RFC v1 3/6] kunit: Add ability to filter attributes Rae Moar
@ 2023-06-10 8:29 ` David Gow
2023-06-13 20:42 ` Rae Moar
2023-06-13 20:26 ` Kees Cook
1 sibling, 1 reply; 21+ messages in thread
From: David Gow @ 2023-06-10 8:29 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
[-- Attachment #1: Type: text/plain, Size: 20167 bytes --]
On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
>
> Add filtering of test attributes. Users can filter tests using a
> module_param_array called "filter". This functionality will be added to
> kunit.py in the next patch.
>
> The filters will be imputed in the format:
> "<attribute_name><operation><attribute_value>"
>
> Example: "speed>slow"
Maybe give a full command-line example of "kunit.filter=speed>slow"?
>
> Operations include: >, <, >=, <=, !=, and =. These operations do not need
> to act the same for every attribute.
I assume here that operations should act the same for attributes of
the same type, but a string attribute might behave differently from an
int, an enum, an array, etc.
As a design principle, I think we definitely want the same operation
to act the same way between different attributes, unless there's an
extraordinarily good reason.
>
> Add method to parse inputted filters.
>
> Add the process of filtering tests based on attributes. The process of
> filtering follows these rules:
>
> A test case with a set attribute overrides its parent suite's attribute
> during filtering.
>
> Also, if both the test case attribute and suite attribute are unset the
> test acts as the default attribute value during filtering.
This behaviour probably needs to be documented more clearly in the
final version.
As I understand it:
- Both tests and suites have attributes.
- Filtering always operates at a per-test level.
- If a test has an attribute set, then the test's value is filtered on.
- Otherwise, the value falls back to the suite's value.
- If neither are set, the attribute has a global "default" value, which is used.
- If an entire suite is filtered out, it's removed, giving the
appearance that filtering can operate on a suite level.
I actually quite like these rules, but we do need to document them.
I'd perhaps argue that the "default attribute" could be done away with
and we just rely on the filter function choosing whether or not
"unset" matches a filter or not, but on the other hand, it does make
reusing filter functions potentially easier.
>
> Finally, add a "filter" method for the speed attribute to parse and compare
> enum values of kunit_speed.
>
> Signed-off-by: Rae Moar <rmoar@google.com>
> ---
One other idea: do we want filtered-out tests to totally disappear (as
this patch does), to mark them as skipped (potentially useful, too),
or have configurable behaviour.
I think there are good reasons for each of those: having them totally
disappear is much cleaner, but it's also useful to see what tests
you're actually, well, skipping.
> include/kunit/attributes.h | 22 +++++
> lib/kunit/attributes.c | 172 +++++++++++++++++++++++++++++++++++++
> lib/kunit/executor.c | 79 +++++++++++++----
> lib/kunit/executor_test.c | 8 +-
> 4 files changed, 258 insertions(+), 23 deletions(-)
>
> diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
> index 9fcd184cce36..bca60d1181bb 100644
> --- a/include/kunit/attributes.h
> +++ b/include/kunit/attributes.h
> @@ -9,6 +9,15 @@
> #ifndef _KUNIT_ATTRIBUTES_H
> #define _KUNIT_ATTRIBUTES_H
>
> +/*
> + * struct kunit_attr_filter - representation of attributes filter with the
> + * attribute object and string input
> + */
> +struct kunit_attr_filter {
> + struct kunit_attr *attr;
> + char *input;
> +};
> +
> /*
> * Print all test attributes for a test case or suite.
> * Output format for test cases: "# <test_name>.<attribute>: <value>"
> @@ -16,4 +25,17 @@
> */
> void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
>
> +/*
> + * Parse attributes filter input and return an object containing the attribute
> + * object and the string input.
> + */
> +struct kunit_attr_filter kunit_parse_filter_attr(char *input, int *err);
Should we rename this kunit_parse_attr_filter, as it returns a
kunit_attr_filter?
> +
> +
> +/*
> + * Returns a copy of the suite containing only tests that pass the filter.
> + */
> +struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
> + struct kunit_attr_filter filter, int *err);
> +
> #endif /* _KUNIT_ATTRIBUTES_H */
> diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
> index e17889f94693..4f753a28e4ee 100644
> --- a/lib/kunit/attributes.c
> +++ b/lib/kunit/attributes.c
> @@ -49,6 +49,66 @@ static const char *attr_speed_to_string(void *attr, bool *to_free)
> return attr_enum_to_string(attr, speed_str_list, to_free);
> }
>
> +/* Filter Methods */
> +
> +static int int_filter(long val, const char *op, int input, int *err)
> +{
> + if (!strncmp(op, "<=", 2))
> + return (val <= input);
> + else if (!strncmp(op, ">=", 2))
> + return (val >= input);
> + else if (!strncmp(op, "!=", 2))
> + return (val != input);
> + else if (!strncmp(op, ">", 1))
> + return (val > input);
> + else if (!strncmp(op, "<", 1))
> + return (val < input);
> + else if (!strncmp(op, "=", 1))
> + return (val == input);
> + *err = -EINVAL;
> + pr_err("kunit executor: invalid filter operation: %s\n", op);
> + return false;
> +}
> +
> +static int attr_enum_filter(void *attr, const char *input, int *err,
> + const char * const str_list[], int max)
As this is a generic helper function to be used by multiple types of
attributes, let's document it a bit.
> +{
> + int i, j, input_int;
> + long test_val = (long)attr;
> + const char *input_val;
> +
> + for (i = 0; input[i]; i++) {
> + if (!strchr("<!=>", input[i])) {
Can we yoink this string of "operation characters" into a global or
#define or something, as it recurs a few times here, and it'd be best
to only have it in one place.
> + input_val = input + i;
> + break;
> + }
> + }
> +
> + if (!input_val) {
> + *err = -EINVAL;
> + pr_err("kunit executor: filter operation not found: %s\n", input);
> + return false;
> + }
> +
> + for (j = 0; j <= max; j++) {
> + if (!strcmp(input_val, str_list[j]))
> + input_int = j;
> + }
> +
> + if (!input_int) {
> + *err = -EINVAL;
> + pr_err("kunit executor: invalid filter input: %s\n", input);
> + return false;
> + }
> +
> + return int_filter(test_val, input, input_int, err);
> +}
> +
> +static int attr_speed_filter(void *attr, const char *input, int *err)
> +{
> + return attr_enum_filter(attr, input, err, speed_str_list, KUNIT_SPEED_MAX);
> +}
> +
> /* Get Attribute Methods */
>
> static void *attr_speed_get(void *test_or_suite, bool is_test)
> @@ -68,6 +128,7 @@ static const struct kunit_attr speed_attr = {
> .name = "speed",
> .get_attr = attr_speed_get,
> .to_string = attr_speed_to_string,
> + .filter = attr_speed_filter,
> .attr_default = (void *)KUNIT_SPEED_NORMAL,
> };
>
> @@ -106,3 +167,114 @@ void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level
> }
> }
> }
> +
> +/* Helper Functions to Filter Attributes */
> +
> +struct kunit_attr_filter kunit_parse_filter_attr(char *input, int *err)
> +{
> + struct kunit_attr_filter filter;
> + int i, j, op_index = 0;
> + int attr_index = -1;
> + char op;
> +
> + /* Parse input until operation */
> + for (i = 0; input[i]; i++) {
> + if (strchr("<>!=", input[i])) {
> + op_index = i;
> + break;
> + }
> + if (input[i] == ' ')
> + break;
> + }
> +
> + if (!op_index) {
> + *err = -EINVAL;
> + pr_err("kunit executor: filter operation not found: %s\n", input);
> + return filter;
> + }
> +
> + op = input[op_index];
> + input[op_index] = '\0';
> +
> + /* Find associated kunit_attr object */
> + for (j = 0; j < ARRAY_SIZE(kunit_attr_list); j++) {
> + if (!strcmp(input, kunit_attr_list[j].name)) {
> + attr_index = j;
> + break;
> + }
> + }
> +
> + input[op_index] = op;
> + filter.input = input + op_index;
> +
> + if (attr_index < 0) {
> + *err = -EINVAL;
> + pr_err("kunit executor: attribute not found: %s\n", input);
> + } else {
> + filter.attr = &kunit_attr_list[attr_index];
> + }
> +
> + return filter;
> +}
> +
> +struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
> + struct kunit_attr_filter filter, int *err)
> +{
> + int n = 0;
> + struct kunit_case *filtered, *test_case;
> + struct kunit_suite *copy;
> + void *suite_val, *test_val;
> + bool suite_result, test_result, default_result;
> +
> + /* Allocate memory for new copy of suite and list of test cases */
> + copy = kmemdup(suite, sizeof(*copy), GFP_KERNEL);
> + if (!copy)
> + return ERR_PTR(-ENOMEM);
> +
> + kunit_suite_for_each_test_case(suite, test_case) { n++; }
> +
> + filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL);
> + if (!filtered) {
> + kfree(copy);
> + return ERR_PTR(-ENOMEM);
> + }
> +
> + n = 0;
> +
> + /* Save filtering result on default value */
> + default_result = filter.attr->filter(filter.attr->attr_default, filter.input, err);
> +
> + /* Save suite attribute value and filtering result on that value */
> + suite_val = filter.attr->get_attr((void *)suite, false);
> + suite_result = filter.attr->filter(suite_val, filter.input, err);
> +
> + /* For each test case, save test case if passes filtering. */
> + kunit_suite_for_each_test_case(suite, test_case) {
> + test_val = filter.attr->get_attr((void *) test_case, true);
> + test_result = filter.attr->filter(filter.attr->get_attr(test_case, true),
> + filter.input, err);
> + /*
> + * If attribute value of test case is set, filter on that value.
> + * If not, filter on suite value if set. If not, filter on
> + * default value.
> + */
> + if (test_val) {
> + if (test_result)
> + filtered[n++] = *test_case;
> + } else if (suite_val) {
> + if (suite_result)
> + filtered[n++] = *test_case;
> + } else if (default_result) {
> + filtered[n++] = *test_case;
> + }
> + }
> +
> + if (n == 0) {
> + kfree(copy);
> + kfree(filtered);
> + return NULL;
> + }
> +
> + copy->test_cases = filtered;
> + return copy;
> +}
> diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
> index 767a84e32f06..c67657821eec 100644
> --- a/lib/kunit/executor.c
> +++ b/lib/kunit/executor.c
> @@ -15,8 +15,12 @@ extern struct kunit_suite * const __kunit_suites_end[];
>
> #if IS_BUILTIN(CONFIG_KUNIT)
>
> +#define MAX_FILTERS 10 // Limit of number of attribute filters
> static char *filter_glob_param;
> static char *action_param;
> +static int filter_count;
> +static char *filter_param[MAX_FILTERS];
> +
>
> module_param_named(filter_glob, filter_glob_param, charp, 0);
> MODULE_PARM_DESC(filter_glob,
> @@ -26,15 +30,16 @@ MODULE_PARM_DESC(action,
> "Changes KUnit executor behavior, valid values are:\n"
> "<none>: run the tests like normal\n"
> "'list' to list test names instead of running them.\n");
> +module_param_array_named(filter, filter_param, charp, &filter_count, 0);
>
> /* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */
> -struct kunit_test_filter {
> +struct kunit_glob_filter {
> char *suite_glob;
> char *test_glob;
> };
>
> /* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */
> -static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,
> +static void kunit_parse_filter_glob(struct kunit_glob_filter *parsed,
> const char *filter_glob)
> {
> const int len = strlen(filter_glob);
> @@ -56,7 +61,7 @@ static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,
>
> /* Create a copy of suite with only tests that match test_glob. */
> static struct kunit_suite *
> -kunit_filter_tests(const struct kunit_suite *const suite, const char *test_glob)
> +kunit_filter_glob_tests(const struct kunit_suite *const suite, const char *test_glob)
> {
> int n = 0;
> struct kunit_case *filtered, *test_case;
> @@ -110,12 +115,15 @@ static void kunit_free_suite_set(struct suite_set suite_set)
>
> static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
> const char *filter_glob,
> + char **filters,
> + int filter_count,
> int *err)
> {
> - int i;
> - struct kunit_suite **copy, *filtered_suite;
> + int i, j, k;
> + struct kunit_suite **copy, *filtered_suite, *new_filtered_suite;
> struct suite_set filtered;
> - struct kunit_test_filter filter;
> + struct kunit_glob_filter parsed_glob;
> + struct kunit_attr_filter parsed_filters[MAX_FILTERS];
>
> const size_t max = suite_set->end - suite_set->start;
>
> @@ -126,17 +134,49 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
> return filtered;
> }
>
> - kunit_parse_filter_glob(&filter, filter_glob);
> + if (filter_glob)
> + kunit_parse_filter_glob(&parsed_glob, filter_glob);
>
> - for (i = 0; &suite_set->start[i] != suite_set->end; i++) {
> - if (!glob_match(filter.suite_glob, suite_set->start[i]->name))
> - continue;
> + /* Parse attribute filters */
> + if (filter_count) {
> + for (j = 0; j < filter_count; j++) {
> + parsed_filters[j] = kunit_parse_filter_attr(filters[j], err);
> + if (*err)
> + return filtered;
> + }
> + }
>
> - filtered_suite = kunit_filter_tests(suite_set->start[i], filter.test_glob);
> - if (IS_ERR(filtered_suite)) {
> - *err = PTR_ERR(filtered_suite);
> - return filtered;
> + for (i = 0; &suite_set->start[i] != suite_set->end; i++) {
> + filtered_suite = suite_set->start[i];
> + if (filter_glob) {
> + if (!glob_match(parsed_glob.suite_glob, filtered_suite->name))
> + continue;
> + filtered_suite = kunit_filter_glob_tests(filtered_suite,
> + parsed_glob.test_glob);
> + if (IS_ERR(filtered_suite)) {
> + *err = PTR_ERR(filtered_suite);
> + return filtered;
> + }
> }
> + if (filter_count) {
> + for (k = 0; k < filter_count; k++) {
> + new_filtered_suite = kunit_filter_attr_tests(filtered_suite,
> + parsed_filters[k], err);
> +
> + /* Free previous copy of suite */
> + if (k > 0 || filter_glob)
> + kfree(filtered_suite);
> + filtered_suite = new_filtered_suite;
> +
> + if (*err)
> + return filtered;
> + if (IS_ERR(filtered_suite)) {
> + *err = PTR_ERR(filtered_suite);
> + return filtered;
> + }
> + }
> + }
> +
> if (!filtered_suite)
> continue;
>
> @@ -144,8 +184,8 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
> }
> filtered.end = copy;
>
> - kfree(filter.suite_glob);
> - kfree(filter.test_glob);
> + kfree(parsed_glob.suite_glob);
> + kfree(parsed_glob.test_glob);
> return filtered;
> }
>
> @@ -203,8 +243,9 @@ int kunit_run_all_tests(void)
> goto out;
> }
>
> - if (filter_glob_param) {
> - suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err);
> + if (filter_glob_param || filter_count) {
> + suite_set = kunit_filter_suites(&suite_set, filter_glob_param,
> + filter_param, filter_count, &err);
> if (err) {
> pr_err("kunit executor: error filtering suites: %d\n", err);
> goto out;
> @@ -218,7 +259,7 @@ int kunit_run_all_tests(void)
> else
> pr_err("kunit executor: unknown action '%s'\n", action_param);
>
> - if (filter_glob_param) { /* a copy was made of each suite */
> + if (filter_glob_param || filter_count) { /* a copy was made of each suite */
> kunit_free_suite_set(suite_set);
> }
>
> diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c
> index ce6749af374d..4c8cb46857b2 100644
> --- a/lib/kunit/executor_test.c
> +++ b/lib/kunit/executor_test.c
> @@ -24,7 +24,7 @@ static struct kunit_case dummy_test_cases[] = {
>
> static void parse_filter_test(struct kunit *test)
> {
> - struct kunit_test_filter filter = {NULL, NULL};
> + struct kunit_glob_filter filter = {NULL, NULL};
>
> kunit_parse_filter_glob(&filter, "suite");
> KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite");
> @@ -50,7 +50,7 @@ static void filter_suites_test(struct kunit *test)
> subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
>
> /* Want: suite1, suite2, NULL -> suite2, NULL */
> - got = kunit_filter_suites(&suite_set, "suite2", &err);
> + got = kunit_filter_suites(&suite_set, "suite2", NULL, 0, &err);
> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
> KUNIT_ASSERT_EQ(test, err, 0);
> kfree_at_end(test, got.start);
> @@ -74,7 +74,7 @@ static void filter_suites_test_glob_test(struct kunit *test)
> subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
>
> /* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */
> - got = kunit_filter_suites(&suite_set, "suite2.test2", &err);
> + got = kunit_filter_suites(&suite_set, "suite2.test2", NULL, 0, &err);
> KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
> KUNIT_ASSERT_EQ(test, err, 0);
> kfree_at_end(test, got.start);
> @@ -100,7 +100,7 @@ static void filter_suites_to_empty_test(struct kunit *test)
> subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases);
> subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
>
> - got = kunit_filter_suites(&suite_set, "not_found", &err);
> + got = kunit_filter_suites(&suite_set, "not_found", NULL, 0, &err);
> KUNIT_ASSERT_EQ(test, err, 0);
> kfree_at_end(test, got.start); /* just in case */
>
It'd be nice to add some more tests for attribute filtering specifically here.
> --
> 2.41.0.162.gfafddb0af9-goog
>
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4003 bytes --]
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [RFC v1 3/6] kunit: Add ability to filter attributes
2023-06-10 8:29 ` David Gow
@ 2023-06-13 20:42 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:42 UTC (permalink / raw)
To: David Gow
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 4:29 AM David Gow <davidgow@google.com> wrote:
>
> On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
> >
> > Add filtering of test attributes. Users can filter tests using a
> > module_param_array called "filter". This functionality will be added to
> > kunit.py in the next patch.
> >
> > The filters will be imputed in the format:
> > "<attribute_name><operation><attribute_value>"
> >
> > Example: "speed>slow"
>
> Maybe give a full command-line example of "kunit.filter=speed>slow"?
>
Hello,
Yes I will add that in the next version.
> >
> > Operations include: >, <, >=, <=, !=, and =. These operations do not need
> > to act the same for every attribute.
>
> I assume here that operations should act the same for attributes of
> the same type, but a string attribute might behave differently from an
> int, an enum, an array, etc.
>
> As a design principle, I think we definitely want the same operation
> to act the same way between different attributes, unless there's an
> extraordinarily good reason.
>
This is a mistake on my end. I should clarify that operations should
act the same for attributes of the same type but then may differ
between the types (like ints and strings).
> >
> > Add method to parse inputted filters.
> >
> > Add the process of filtering tests based on attributes. The process of
> > filtering follows these rules:
> >
> > A test case with a set attribute overrides its parent suite's attribute
> > during filtering.
> >
> > Also, if both the test case attribute and suite attribute are unset the
> > test acts as the default attribute value during filtering.
>
> This behaviour probably needs to be documented more clearly in the
> final version.
>
> As I understand it:
> - Both tests and suites have attributes.
> - Filtering always operates at a per-test level.
> - If a test has an attribute set, then the test's value is filtered on.
> - Otherwise, the value falls back to the suite's value.
> - If neither are set, the attribute has a global "default" value, which is used.
> - If an entire suite is filtered out, it's removed, giving the
> appearance that filtering can operate on a suite level.
>
Yes, this is a great description of how it should behave. I will be
more explicit in my description for the next version.
> I actually quite like these rules, but we do need to document them.
> I'd perhaps argue that the "default attribute" could be done away with
> and we just rely on the filter function choosing whether or not
> "unset" matches a filter or not, but on the other hand, it does make
> reusing filter functions potentially easier.
>
This is true I could do away with the default. However, I do think it
helps to document how an unset attribute acts.
It may seem more unclear why an unset attribute is filtered out for
speed<=slow but not speed>slow. But this could then be documented
separate from the code.
I'm leaning toward keeping it but let me know what you think.
> >
> > Finally, add a "filter" method for the speed attribute to parse and compare
> > enum values of kunit_speed.
> >
> > Signed-off-by: Rae Moar <rmoar@google.com>
> > ---
>
> One other idea: do we want filtered-out tests to totally disappear (as
> this patch does), to mark them as skipped (potentially useful, too),
> or have configurable behaviour.
>
> I think there are good reasons for each of those: having them totally
> disappear is much cleaner, but it's also useful to see what tests
> you're actually, well, skipping.
I like this idea a lot. I also really like the idea of this being a
configurable behavior.
>
> > include/kunit/attributes.h | 22 +++++
> > lib/kunit/attributes.c | 172 +++++++++++++++++++++++++++++++++++++
> > lib/kunit/executor.c | 79 +++++++++++++----
> > lib/kunit/executor_test.c | 8 +-
> > 4 files changed, 258 insertions(+), 23 deletions(-)
> >
> > diff --git a/include/kunit/attributes.h b/include/kunit/attributes.h
> > index 9fcd184cce36..bca60d1181bb 100644
> > --- a/include/kunit/attributes.h
> > +++ b/include/kunit/attributes.h
> > @@ -9,6 +9,15 @@
> > #ifndef _KUNIT_ATTRIBUTES_H
> > #define _KUNIT_ATTRIBUTES_H
> >
> > +/*
> > + * struct kunit_attr_filter - representation of attributes filter with the
> > + * attribute object and string input
> > + */
> > +struct kunit_attr_filter {
> > + struct kunit_attr *attr;
> > + char *input;
> > +};
> > +
> > /*
> > * Print all test attributes for a test case or suite.
> > * Output format for test cases: "# <test_name>.<attribute>: <value>"
> > @@ -16,4 +25,17 @@
> > */
> > void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level);
> >
> > +/*
> > + * Parse attributes filter input and return an object containing the attribute
> > + * object and the string input.
> > + */
> > +struct kunit_attr_filter kunit_parse_filter_attr(char *input, int *err);
>
> Should we rename this kunit_parse_attr_filter, as it returns a
> kunit_attr_filter?
>
I will change this. I think that is cleaner.
> > +
> > +
> > +/*
> > + * Returns a copy of the suite containing only tests that pass the filter.
> > + */
> > +struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
> > + struct kunit_attr_filter filter, int *err);
> > +
> > #endif /* _KUNIT_ATTRIBUTES_H */
> > diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
> > index e17889f94693..4f753a28e4ee 100644
> > --- a/lib/kunit/attributes.c
> > +++ b/lib/kunit/attributes.c
> > @@ -49,6 +49,66 @@ static const char *attr_speed_to_string(void *attr, bool *to_free)
> > return attr_enum_to_string(attr, speed_str_list, to_free);
> > }
> >
> > +/* Filter Methods */
> > +
> > +static int int_filter(long val, const char *op, int input, int *err)
> > +{
> > + if (!strncmp(op, "<=", 2))
> > + return (val <= input);
> > + else if (!strncmp(op, ">=", 2))
> > + return (val >= input);
> > + else if (!strncmp(op, "!=", 2))
> > + return (val != input);
> > + else if (!strncmp(op, ">", 1))
> > + return (val > input);
> > + else if (!strncmp(op, "<", 1))
> > + return (val < input);
> > + else if (!strncmp(op, "=", 1))
> > + return (val == input);
> > + *err = -EINVAL;
> > + pr_err("kunit executor: invalid filter operation: %s\n", op);
> > + return false;
> > +}
> > +
> > +static int attr_enum_filter(void *attr, const char *input, int *err,
> > + const char * const str_list[], int max)
>
> As this is a generic helper function to be used by multiple types of
> attributes, let's document it a bit.
Sounds great. I will add documentation here.
>
> > +{
> > + int i, j, input_int;
> > + long test_val = (long)attr;
> > + const char *input_val;
> > +
> > + for (i = 0; input[i]; i++) {
> > + if (!strchr("<!=>", input[i])) {
>
> Can we yoink this string of "operation characters" into a global or
> #define or something, as it recurs a few times here, and it'd be best
> to only have it in one place.
Right, that sounds good. I will change this.
>
> > + input_val = input + i;
> > + break;
> > + }
> > + }
> > +
> > + if (!input_val) {
> > + *err = -EINVAL;
> > + pr_err("kunit executor: filter operation not found: %s\n", input);
> > + return false;
> > + }
> > +
> > + for (j = 0; j <= max; j++) {
> > + if (!strcmp(input_val, str_list[j]))
> > + input_int = j;
> > + }
> > +
> > + if (!input_int) {
> > + *err = -EINVAL;
> > + pr_err("kunit executor: invalid filter input: %s\n", input);
> > + return false;
> > + }
> > +
> > + return int_filter(test_val, input, input_int, err);
> > +}
> > +
> > +static int attr_speed_filter(void *attr, const char *input, int *err)
> > +{
> > + return attr_enum_filter(attr, input, err, speed_str_list, KUNIT_SPEED_MAX);
> > +}
> > +
> > /* Get Attribute Methods */
> >
> > static void *attr_speed_get(void *test_or_suite, bool is_test)
> > @@ -68,6 +128,7 @@ static const struct kunit_attr speed_attr = {
> > .name = "speed",
> > .get_attr = attr_speed_get,
> > .to_string = attr_speed_to_string,
> > + .filter = attr_speed_filter,
> > .attr_default = (void *)KUNIT_SPEED_NORMAL,
> > };
> >
> > @@ -106,3 +167,114 @@ void kunit_print_attr(void *test_or_suite, bool is_test, unsigned int test_level
> > }
> > }
> > }
> > +
> > +/* Helper Functions to Filter Attributes */
> > +
> > +struct kunit_attr_filter kunit_parse_filter_attr(char *input, int *err)
> > +{
> > + struct kunit_attr_filter filter;
> > + int i, j, op_index = 0;
> > + int attr_index = -1;
> > + char op;
> > +
> > + /* Parse input until operation */
> > + for (i = 0; input[i]; i++) {
> > + if (strchr("<>!=", input[i])) {
> > + op_index = i;
> > + break;
> > + }
> > + if (input[i] == ' ')
> > + break;
> > + }
> > +
> > + if (!op_index) {
> > + *err = -EINVAL;
> > + pr_err("kunit executor: filter operation not found: %s\n", input);
> > + return filter;
> > + }
> > +
> > + op = input[op_index];
> > + input[op_index] = '\0';
> > +
> > + /* Find associated kunit_attr object */
> > + for (j = 0; j < ARRAY_SIZE(kunit_attr_list); j++) {
> > + if (!strcmp(input, kunit_attr_list[j].name)) {
> > + attr_index = j;
> > + break;
> > + }
> > + }
> > +
> > + input[op_index] = op;
> > + filter.input = input + op_index;
> > +
> > + if (attr_index < 0) {
> > + *err = -EINVAL;
> > + pr_err("kunit executor: attribute not found: %s\n", input);
> > + } else {
> > + filter.attr = &kunit_attr_list[attr_index];
> > + }
> > +
> > + return filter;
> > +}
> > +
> > +struct kunit_suite *kunit_filter_attr_tests(const struct kunit_suite *const suite,
> > + struct kunit_attr_filter filter, int *err)
> > +{
> > + int n = 0;
> > + struct kunit_case *filtered, *test_case;
> > + struct kunit_suite *copy;
> > + void *suite_val, *test_val;
> > + bool suite_result, test_result, default_result;
> > +
> > + /* Allocate memory for new copy of suite and list of test cases */
> > + copy = kmemdup(suite, sizeof(*copy), GFP_KERNEL);
> > + if (!copy)
> > + return ERR_PTR(-ENOMEM);
> > +
> > + kunit_suite_for_each_test_case(suite, test_case) { n++; }
> > +
> > + filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL);
> > + if (!filtered) {
> > + kfree(copy);
> > + return ERR_PTR(-ENOMEM);
> > + }
> > +
> > + n = 0;
> > +
> > + /* Save filtering result on default value */
> > + default_result = filter.attr->filter(filter.attr->attr_default, filter.input, err);
> > +
> > + /* Save suite attribute value and filtering result on that value */
> > + suite_val = filter.attr->get_attr((void *)suite, false);
> > + suite_result = filter.attr->filter(suite_val, filter.input, err);
> > +
> > + /* For each test case, save test case if passes filtering. */
> > + kunit_suite_for_each_test_case(suite, test_case) {
> > + test_val = filter.attr->get_attr((void *) test_case, true);
> > + test_result = filter.attr->filter(filter.attr->get_attr(test_case, true),
> > + filter.input, err);
> > + /*
> > + * If attribute value of test case is set, filter on that value.
> > + * If not, filter on suite value if set. If not, filter on
> > + * default value.
> > + */
> > + if (test_val) {
> > + if (test_result)
> > + filtered[n++] = *test_case;
> > + } else if (suite_val) {
> > + if (suite_result)
> > + filtered[n++] = *test_case;
> > + } else if (default_result) {
> > + filtered[n++] = *test_case;
> > + }
> > + }
> > +
> > + if (n == 0) {
> > + kfree(copy);
> > + kfree(filtered);
> > + return NULL;
> > + }
> > +
> > + copy->test_cases = filtered;
> > + return copy;
> > +}
> > diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
> > index 767a84e32f06..c67657821eec 100644
> > --- a/lib/kunit/executor.c
> > +++ b/lib/kunit/executor.c
> > @@ -15,8 +15,12 @@ extern struct kunit_suite * const __kunit_suites_end[];
> >
> > #if IS_BUILTIN(CONFIG_KUNIT)
> >
> > +#define MAX_FILTERS 10 // Limit of number of attribute filters
> > static char *filter_glob_param;
> > static char *action_param;
> > +static int filter_count;
> > +static char *filter_param[MAX_FILTERS];
> > +
> >
> > module_param_named(filter_glob, filter_glob_param, charp, 0);
> > MODULE_PARM_DESC(filter_glob,
> > @@ -26,15 +30,16 @@ MODULE_PARM_DESC(action,
> > "Changes KUnit executor behavior, valid values are:\n"
> > "<none>: run the tests like normal\n"
> > "'list' to list test names instead of running them.\n");
> > +module_param_array_named(filter, filter_param, charp, &filter_count, 0);
> >
> > /* glob_match() needs NULL terminated strings, so we need a copy of filter_glob_param. */
> > -struct kunit_test_filter {
> > +struct kunit_glob_filter {
> > char *suite_glob;
> > char *test_glob;
> > };
> >
> > /* Split "suite_glob.test_glob" into two. Assumes filter_glob is not empty. */
> > -static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,
> > +static void kunit_parse_filter_glob(struct kunit_glob_filter *parsed,
> > const char *filter_glob)
> > {
> > const int len = strlen(filter_glob);
> > @@ -56,7 +61,7 @@ static void kunit_parse_filter_glob(struct kunit_test_filter *parsed,
> >
> > /* Create a copy of suite with only tests that match test_glob. */
> > static struct kunit_suite *
> > -kunit_filter_tests(const struct kunit_suite *const suite, const char *test_glob)
> > +kunit_filter_glob_tests(const struct kunit_suite *const suite, const char *test_glob)
> > {
> > int n = 0;
> > struct kunit_case *filtered, *test_case;
> > @@ -110,12 +115,15 @@ static void kunit_free_suite_set(struct suite_set suite_set)
> >
> > static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
> > const char *filter_glob,
> > + char **filters,
> > + int filter_count,
> > int *err)
> > {
> > - int i;
> > - struct kunit_suite **copy, *filtered_suite;
> > + int i, j, k;
> > + struct kunit_suite **copy, *filtered_suite, *new_filtered_suite;
> > struct suite_set filtered;
> > - struct kunit_test_filter filter;
> > + struct kunit_glob_filter parsed_glob;
> > + struct kunit_attr_filter parsed_filters[MAX_FILTERS];
> >
> > const size_t max = suite_set->end - suite_set->start;
> >
> > @@ -126,17 +134,49 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
> > return filtered;
> > }
> >
> > - kunit_parse_filter_glob(&filter, filter_glob);
> > + if (filter_glob)
> > + kunit_parse_filter_glob(&parsed_glob, filter_glob);
> >
> > - for (i = 0; &suite_set->start[i] != suite_set->end; i++) {
> > - if (!glob_match(filter.suite_glob, suite_set->start[i]->name))
> > - continue;
> > + /* Parse attribute filters */
> > + if (filter_count) {
> > + for (j = 0; j < filter_count; j++) {
> > + parsed_filters[j] = kunit_parse_filter_attr(filters[j], err);
> > + if (*err)
> > + return filtered;
> > + }
> > + }
> >
> > - filtered_suite = kunit_filter_tests(suite_set->start[i], filter.test_glob);
> > - if (IS_ERR(filtered_suite)) {
> > - *err = PTR_ERR(filtered_suite);
> > - return filtered;
> > + for (i = 0; &suite_set->start[i] != suite_set->end; i++) {
> > + filtered_suite = suite_set->start[i];
> > + if (filter_glob) {
> > + if (!glob_match(parsed_glob.suite_glob, filtered_suite->name))
> > + continue;
> > + filtered_suite = kunit_filter_glob_tests(filtered_suite,
> > + parsed_glob.test_glob);
> > + if (IS_ERR(filtered_suite)) {
> > + *err = PTR_ERR(filtered_suite);
> > + return filtered;
> > + }
> > }
> > + if (filter_count) {
> > + for (k = 0; k < filter_count; k++) {
> > + new_filtered_suite = kunit_filter_attr_tests(filtered_suite,
> > + parsed_filters[k], err);
> > +
> > + /* Free previous copy of suite */
> > + if (k > 0 || filter_glob)
> > + kfree(filtered_suite);
> > + filtered_suite = new_filtered_suite;
> > +
> > + if (*err)
> > + return filtered;
> > + if (IS_ERR(filtered_suite)) {
> > + *err = PTR_ERR(filtered_suite);
> > + return filtered;
> > + }
> > + }
> > + }
> > +
> > if (!filtered_suite)
> > continue;
> >
> > @@ -144,8 +184,8 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set,
> > }
> > filtered.end = copy;
> >
> > - kfree(filter.suite_glob);
> > - kfree(filter.test_glob);
> > + kfree(parsed_glob.suite_glob);
> > + kfree(parsed_glob.test_glob);
> > return filtered;
> > }
> >
> > @@ -203,8 +243,9 @@ int kunit_run_all_tests(void)
> > goto out;
> > }
> >
> > - if (filter_glob_param) {
> > - suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err);
> > + if (filter_glob_param || filter_count) {
> > + suite_set = kunit_filter_suites(&suite_set, filter_glob_param,
> > + filter_param, filter_count, &err);
> > if (err) {
> > pr_err("kunit executor: error filtering suites: %d\n", err);
> > goto out;
> > @@ -218,7 +259,7 @@ int kunit_run_all_tests(void)
> > else
> > pr_err("kunit executor: unknown action '%s'\n", action_param);
> >
> > - if (filter_glob_param) { /* a copy was made of each suite */
> > + if (filter_glob_param || filter_count) { /* a copy was made of each suite */
> > kunit_free_suite_set(suite_set);
> > }
> >
> > diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c
> > index ce6749af374d..4c8cb46857b2 100644
> > --- a/lib/kunit/executor_test.c
> > +++ b/lib/kunit/executor_test.c
> > @@ -24,7 +24,7 @@ static struct kunit_case dummy_test_cases[] = {
> >
> > static void parse_filter_test(struct kunit *test)
> > {
> > - struct kunit_test_filter filter = {NULL, NULL};
> > + struct kunit_glob_filter filter = {NULL, NULL};
> >
> > kunit_parse_filter_glob(&filter, "suite");
> > KUNIT_EXPECT_STREQ(test, filter.suite_glob, "suite");
> > @@ -50,7 +50,7 @@ static void filter_suites_test(struct kunit *test)
> > subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
> >
> > /* Want: suite1, suite2, NULL -> suite2, NULL */
> > - got = kunit_filter_suites(&suite_set, "suite2", &err);
> > + got = kunit_filter_suites(&suite_set, "suite2", NULL, 0, &err);
> > KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
> > KUNIT_ASSERT_EQ(test, err, 0);
> > kfree_at_end(test, got.start);
> > @@ -74,7 +74,7 @@ static void filter_suites_test_glob_test(struct kunit *test)
> > subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
> >
> > /* Want: suite1, suite2, NULL -> suite2 (just test1), NULL */
> > - got = kunit_filter_suites(&suite_set, "suite2.test2", &err);
> > + got = kunit_filter_suites(&suite_set, "suite2.test2", NULL, 0, &err);
> > KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
> > KUNIT_ASSERT_EQ(test, err, 0);
> > kfree_at_end(test, got.start);
> > @@ -100,7 +100,7 @@ static void filter_suites_to_empty_test(struct kunit *test)
> > subsuite[0] = alloc_fake_suite(test, "suite1", dummy_test_cases);
> > subsuite[1] = alloc_fake_suite(test, "suite2", dummy_test_cases);
> >
> > - got = kunit_filter_suites(&suite_set, "not_found", &err);
> > + got = kunit_filter_suites(&suite_set, "not_found", NULL, 0, &err);
> > KUNIT_ASSERT_EQ(test, err, 0);
> > kfree_at_end(test, got.start); /* just in case */
> >
>
> It'd be nice to add some more tests for attribute filtering specifically here.
Yeah, I agree. I will go ahead and add some here.
Thanks for all the comments!
-Rae
>
>
>
> > --
> > 2.41.0.162.gfafddb0af9-goog
> >
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [RFC v1 3/6] kunit: Add ability to filter attributes
2023-06-10 0:51 ` [RFC v1 3/6] kunit: Add ability to filter attributes Rae Moar
2023-06-10 8:29 ` David Gow
@ 2023-06-13 20:26 ` Kees Cook
2023-06-13 20:58 ` Rae Moar
1 sibling, 1 reply; 21+ messages in thread
From: Kees Cook @ 2023-06-13 20:26 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, davidgow, dlatypov, brendan.higgins, linux-kselftest,
kunit-dev, linux-kernel, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 12:51:46AM +0000, Rae Moar wrote:
> Add filtering of test attributes. Users can filter tests using a
> module_param_array called "filter". This functionality will be added to
> kunit.py in the next patch.
>
> The filters will be imputed in the format:
> "<attribute_name><operation><attribute_value>"
>
> Example: "speed>slow"
>
> Operations include: >, <, >=, <=, !=, and =. These operations do not need
> to act the same for every attribute.
How is the "default" filter specified? Is explicitly unfiltered? (i.e.
"slow" stuff will run by default?) Or should there be a default filter
of "speed<slow" for the memcpy conversion?
But yes, I'm a fan of this whole series! I would much prefer this to
having one-off CONFIGs for slow tests. :)
--
Kees Cook
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [RFC v1 3/6] kunit: Add ability to filter attributes
2023-06-13 20:26 ` Kees Cook
@ 2023-06-13 20:58 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:58 UTC (permalink / raw)
To: Kees Cook
Cc: shuah, davidgow, dlatypov, brendan.higgins, linux-kselftest,
kunit-dev, linux-kernel, linux-hardening, jstultz, tglx, sboyd
On Tue, Jun 13, 2023 at 4:26 PM Kees Cook <keescook@chromium.org> wrote:
>
> On Sat, Jun 10, 2023 at 12:51:46AM +0000, Rae Moar wrote:
> > Add filtering of test attributes. Users can filter tests using a
> > module_param_array called "filter". This functionality will be added to
> > kunit.py in the next patch.
> >
> > The filters will be imputed in the format:
> > "<attribute_name><operation><attribute_value>"
> >
> > Example: "speed>slow"
> >
> > Operations include: >, <, >=, <=, !=, and =. These operations do not need
> > to act the same for every attribute.
>
> How is the "default" filter specified? Is explicitly unfiltered? (i.e.
> "slow" stuff will run by default?) Or should there be a default filter
> of "speed<slow" for the memcpy conversion?
>
> But yes, I'm a fan of this whole series! I would much prefer this to
> having one-off CONFIGs for slow tests. :)
>
Hello!
Great to hear that you are happy to see this series.
Currently if no filter is specified, tests will run unfiltered (so the
slow tests will run by default).
But I think the idea of having a "default" filter is really
interesting. I would definitely be open to adding a default filter
that only runs tests with speeds faster than slow, which could then be
overridden by any filter input.
This also means there could be suite specific default filters but that
may be a future implementation since we currently only have one
attribute.
Or alternatively we could have a file that includes a list of default
filters that could then be inputted and altered based on suite.
Thanks!
Rae
> --
> Kees Cook
^ permalink raw reply [flat|nested] 21+ messages in thread
* [RFC v1 4/6] kunit: tool: Add command line interface to filter and report attributes
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
` (2 preceding siblings ...)
2023-06-10 0:51 ` [RFC v1 3/6] kunit: Add ability to filter attributes Rae Moar
@ 2023-06-10 0:51 ` Rae Moar
2023-06-10 8:29 ` David Gow
2023-06-10 0:51 ` [RFC v1 5/6] kunit: memcpy: Mark tests as slow using test attributes Rae Moar
` (2 subsequent siblings)
6 siblings, 1 reply; 21+ messages in thread
From: Rae Moar @ 2023-06-10 0:51 UTC (permalink / raw)
To: shuah, davidgow, dlatypov, brendan.higgins
Cc: linux-kselftest, kunit-dev, linux-kernel, keescook,
linux-hardening, jstultz, tglx, sboyd, Rae Moar
Add ability to use kunit.py to filter attributes and to report a list of
tests including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow
This filter would run only the tests that are marked as slow. Note there
cannot be spaces within a filter.
As said in the previous patch, filters can have different operations: <, >,
<=, >=, !=, and =. Note that the characters < and > are often interpreted
by the shell, so they may need to be quoted or escaped.
Example: --filter "speed>=normal" or –filter speed\>=normal
This filter would run only the tests that have the speed faster than or
equal to normal.
Add flag "--list_tests" to output a list of tests and their attributes
without running tests. This will be useful to see test attributes and which
tests will run with given filters.
Example of the output of these tests:
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Signed-off-by: Rae Moar <rmoar@google.com>
---
tools/testing/kunit/kunit.py | 34 +++++++++++++++++----
tools/testing/kunit/kunit_kernel.py | 6 ++--
tools/testing/kunit/kunit_tool_test.py | 41 +++++++++++++-------------
3 files changed, 54 insertions(+), 27 deletions(-)
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index 3905c43369c3..661c39f7acf5 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -55,8 +55,10 @@ class KunitExecRequest(KunitParseRequest):
build_dir: str
timeout: int
filter_glob: str
+ filter: Optional[List[str]]
kernel_args: Optional[List[str]]
run_isolated: Optional[str]
+ list_tests: Optional[bool]
@dataclass
class KunitRequest(KunitExecRequest, KunitBuildRequest):
@@ -100,7 +102,7 @@ def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
return build_tests(linux, request)
-def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
+def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]:
args = ['kunit.action=list']
if request.kernel_args:
args.extend(request.kernel_args)
@@ -108,13 +110,17 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest)
output = linux.run_kernel(args=args,
timeout=request.timeout,
filter_glob=request.filter_glob,
+ filter=request.filter,
build_dir=request.build_dir)
lines = kunit_parser.extract_tap_lines(output)
# Hack! Drop the dummy TAP version header that the executor prints out.
lines.pop()
# Filter out any extraneous non-test output that might have gotten mixed in.
- return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
+ return output
+
+def _get_tests(output: Iterable[str]) -> List[str]:
+ return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
def _suites_from_test_list(tests: List[str]) -> List[str]:
"""Extracts all the suites from an ordered list of tests."""
@@ -132,8 +138,14 @@ def _suites_from_test_list(tests: List[str]) -> List[str]:
def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
filter_globs = [request.filter_glob]
+ if request.list_tests:
+ output = _list_tests(linux, request)
+ for line in output:
+ print(line.rstrip())
+ return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0)
if request.run_isolated:
- tests = _list_tests(linux, request)
+ output = _list_tests(linux, request)
+ tests = _get_tests(output)
if request.run_isolated == 'test':
filter_globs = tests
elif request.run_isolated == 'suite':
@@ -155,6 +167,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
args=request.kernel_args,
timeout=request.timeout,
filter_glob=filter_glob,
+ filter=request.filter,
build_dir=request.build_dir)
_, test_result = parse_tests(request, metadata, run_result)
@@ -341,6 +354,11 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
nargs='?',
default='',
metavar='filter_glob')
+ parser.add_argument('--filter',
+ help='Filter which KUnit tests run by attributes'
+ 'e.g. speed=fast or speed=>low',
+ type=str,
+ nargs='*')
parser.add_argument('--kernel_args',
help='Kernel command-line parameters. Maybe be repeated',
action='append', metavar='')
@@ -350,6 +368,8 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
'what ran before it.',
type=str,
choices=['suite', 'test'])
+ parser.add_argument('--list_tests', help='If set, list all tests and attributes.',
+ action='store_true')
def add_parse_opts(parser: argparse.ArgumentParser) -> None:
parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. '
@@ -398,8 +418,10 @@ def run_handler(cli_args: argparse.Namespace) -> None:
json=cli_args.json,
timeout=cli_args.timeout,
filter_glob=cli_args.filter_glob,
+ filter=cli_args.filter,
kernel_args=cli_args.kernel_args,
- run_isolated=cli_args.run_isolated)
+ run_isolated=cli_args.run_isolated,
+ list_tests=cli_args.list_tests)
result = run_tests(linux, request)
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
@@ -441,8 +463,10 @@ def exec_handler(cli_args: argparse.Namespace) -> None:
json=cli_args.json,
timeout=cli_args.timeout,
filter_glob=cli_args.filter_glob,
+ filter=cli_args.filter,
kernel_args=cli_args.kernel_args,
- run_isolated=cli_args.run_isolated)
+ run_isolated=cli_args.run_isolated,
+ list_tests=cli_args.list_tests)
result = exec_tests(linux, exec_request)
stdout.print_with_timestamp((
'Elapsed time: %.3fs\n') % (result.elapsed_time))
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index 7f648802caf6..62cb8200f60e 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -330,11 +330,13 @@ class LinuxSourceTree:
return False
return self.validate_config(build_dir)
- def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', timeout: Optional[int]=None) -> Iterator[str]:
+ def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: Optional[List[str]]=None, timeout: Optional[int]=None) -> Iterator[str]:
if not args:
args = []
if filter_glob:
- args.append('kunit.filter_glob='+filter_glob)
+ args.append('kunit.filter_glob=' + filter_glob)
+ if filter:
+ args.append('kunit.filter=' + (','.join(filter)))
args.append('kunit.enable=1')
process = self._ops.start(args, build_dir)
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index be35999bb84f..4a7f3112d06c 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -14,6 +14,7 @@ import tempfile, shutil # Handling test_tmpdir
import itertools
import json
import os
+import re
import signal
import subprocess
from typing import Iterable
@@ -597,7 +598,7 @@ class KUnitMainTest(unittest.TestCase):
self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 0)
self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir='.kunit', filter_glob='', timeout=300)
+ args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_run_passes_args_pass(self):
@@ -605,7 +606,7 @@ class KUnitMainTest(unittest.TestCase):
self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir='.kunit', filter_glob='', timeout=300)
+ args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_exec_passes_args_fail(self):
@@ -629,7 +630,7 @@ class KUnitMainTest(unittest.TestCase):
kunit.main(['run'])
self.assertEqual(e.exception.code, 1)
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir='.kunit', filter_glob='', timeout=300)
+ args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
self.print_mock.assert_any_call(StrContains(' 0 tests run!'))
def test_exec_raw_output(self):
@@ -670,13 +671,13 @@ class KUnitMainTest(unittest.TestCase):
self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
kunit.main(['run', '--raw_output', 'filter_glob'])
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir='.kunit', filter_glob='filter_glob', timeout=300)
+ args=None, build_dir='.kunit', filter_glob='filter_glob', filter=None, timeout=300)
def test_exec_timeout(self):
timeout = 3453
kunit.main(['exec', '--timeout', str(timeout)])
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir='.kunit', filter_glob='', timeout=timeout)
+ args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=timeout)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_run_timeout(self):
@@ -684,7 +685,7 @@ class KUnitMainTest(unittest.TestCase):
kunit.main(['run', '--timeout', str(timeout)])
self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir='.kunit', filter_glob='', timeout=timeout)
+ args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=timeout)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_run_builddir(self):
@@ -692,7 +693,7 @@ class KUnitMainTest(unittest.TestCase):
kunit.main(['run', '--build_dir=.kunit'])
self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir=build_dir, filter_glob='', timeout=300)
+ args=None, build_dir=build_dir, filter_glob='', filter=None, timeout=300)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_config_builddir(self):
@@ -710,7 +711,7 @@ class KUnitMainTest(unittest.TestCase):
build_dir = '.kunit'
kunit.main(['exec', '--build_dir', build_dir])
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=None, build_dir=build_dir, filter_glob='', timeout=300)
+ args=None, build_dir=build_dir, filter_glob='', filter=None, timeout=300)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_run_kunitconfig(self):
@@ -786,7 +787,7 @@ class KUnitMainTest(unittest.TestCase):
kunit.main(['run', '--kernel_args=a=1', '--kernel_args=b=2'])
self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=['a=1','b=2'], build_dir='.kunit', filter_glob='', timeout=300)
+ args=['a=1','b=2'], build_dir='.kunit', filter_glob='', filter=None, timeout=300)
self.print_mock.assert_any_call(StrContains('Testing complete.'))
def test_list_tests(self):
@@ -794,12 +795,13 @@ class KUnitMainTest(unittest.TestCase):
self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
got = kunit._list_tests(self.linux_source_mock,
- kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'suite'))
+ kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, None, 'suite', False))
+ tests = kunit._get_tests(got)
- self.assertEqual(got, want)
+ self.assertEqual(tests, want)
# Should respect the user's filter glob when listing tests.
self.linux_source_mock.run_kernel.assert_called_once_with(
- args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', timeout=300)
+ args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter=None, timeout=300)
@mock.patch.object(kunit, '_list_tests')
@@ -809,10 +811,10 @@ class KUnitMainTest(unittest.TestCase):
# Should respect the user's filter glob when listing tests.
mock_tests.assert_called_once_with(mock.ANY,
- kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, 'suite'))
+ kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, None, 'suite', False))
self.linux_source_mock.run_kernel.assert_has_calls([
- mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300),
- mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300),
+ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter=None, timeout=300),
+ mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter=None, timeout=300),
])
@mock.patch.object(kunit, '_list_tests')
@@ -822,13 +824,12 @@ class KUnitMainTest(unittest.TestCase):
# Should respect the user's filter glob when listing tests.
mock_tests.assert_called_once_with(mock.ANY,
- kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'test'))
+ kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, None, 'test', False))
self.linux_source_mock.run_kernel.assert_has_calls([
- mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300),
- mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300),
- mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', timeout=300),
+ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter=None, timeout=300),
+ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter=None, timeout=300),
+ mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter=None, timeout=300),
])
-
if __name__ == '__main__':
unittest.main()
--
2.41.0.162.gfafddb0af9-goog
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [RFC v1 4/6] kunit: tool: Add command line interface to filter and report attributes
2023-06-10 0:51 ` [RFC v1 4/6] kunit: tool: Add command line interface to filter and report attributes Rae Moar
@ 2023-06-10 8:29 ` David Gow
2023-06-13 20:44 ` Rae Moar
0 siblings, 1 reply; 21+ messages in thread
From: David Gow @ 2023-06-10 8:29 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
[-- Attachment #1: Type: text/plain, Size: 18729 bytes --]
On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
>
> Add ability to use kunit.py to filter attributes and to report a list of
> tests including attributes without running tests.
>
> Add flag "--filter" to input filters on test attributes. Tests will be
> filtered out if they do not match all inputted filters.
>
> Example: --filter speed=slow
>
> This filter would run only the tests that are marked as slow. Note there
> cannot be spaces within a filter.
Within a filter's name, value, or the entire filter string. Is this a
restriction we can remove?
>
> As said in the previous patch, filters can have different operations: <, >,
> <=, >=, !=, and =. Note that the characters < and > are often interpreted
> by the shell, so they may need to be quoted or escaped.
>
> Example: --filter "speed>=normal" or –filter speed\>=normal
>
> This filter would run only the tests that have the speed faster than or
> equal to normal.
>
> Add flag "--list_tests" to output a list of tests and their attributes
> without running tests. This will be useful to see test attributes and which
> tests will run with given filters.
Please note that this comes from the kernel's kunit.action=list option.
>
> Example of the output of these tests:
> example
> example.test_1
> example.test_2
> # example.test_2.speed: slow
>
> This output includes a suite, example, with two test cases, test_1 and
> test_2. And in this instance test_2 has been marked as slow.
>
It's unrelated, so perhaps best split out into its own patch, but I'd
love the option to list tests without the attributes as well. That
would allow doing things like piping the list of tests to wc -l to
count them, etc.
> Signed-off-by: Rae Moar <rmoar@google.com>
> ---
> tools/testing/kunit/kunit.py | 34 +++++++++++++++++----
> tools/testing/kunit/kunit_kernel.py | 6 ++--
> tools/testing/kunit/kunit_tool_test.py | 41 +++++++++++++-------------
> 3 files changed, 54 insertions(+), 27 deletions(-)
>
> diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
> index 3905c43369c3..661c39f7acf5 100755
> --- a/tools/testing/kunit/kunit.py
> +++ b/tools/testing/kunit/kunit.py
> @@ -55,8 +55,10 @@ class KunitExecRequest(KunitParseRequest):
> build_dir: str
> timeout: int
> filter_glob: str
> + filter: Optional[List[str]]
> kernel_args: Optional[List[str]]
> run_isolated: Optional[str]
> + list_tests: Optional[bool]
>
> @dataclass
> class KunitRequest(KunitExecRequest, KunitBuildRequest):
> @@ -100,7 +102,7 @@ def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
>
> return build_tests(linux, request)
>
> -def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
> +def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]:
> args = ['kunit.action=list']
> if request.kernel_args:
> args.extend(request.kernel_args)
> @@ -108,13 +110,17 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest)
> output = linux.run_kernel(args=args,
> timeout=request.timeout,
> filter_glob=request.filter_glob,
> + filter=request.filter,
> build_dir=request.build_dir)
> lines = kunit_parser.extract_tap_lines(output)
> # Hack! Drop the dummy TAP version header that the executor prints out.
> lines.pop()
>
> # Filter out any extraneous non-test output that might have gotten mixed in.
> - return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
> + return output
> +
> +def _get_tests(output: Iterable[str]) -> List[str]:
> + return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
>
> def _suites_from_test_list(tests: List[str]) -> List[str]:
> """Extracts all the suites from an ordered list of tests."""
> @@ -132,8 +138,14 @@ def _suites_from_test_list(tests: List[str]) -> List[str]:
>
> def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
> filter_globs = [request.filter_glob]
> + if request.list_tests:
> + output = _list_tests(linux, request)
> + for line in output:
> + print(line.rstrip())
> + return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0)
> if request.run_isolated:
> - tests = _list_tests(linux, request)
> + output = _list_tests(linux, request)
> + tests = _get_tests(output)
> if request.run_isolated == 'test':
> filter_globs = tests
> elif request.run_isolated == 'suite':
> @@ -155,6 +167,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
> args=request.kernel_args,
> timeout=request.timeout,
> filter_glob=filter_glob,
> + filter=request.filter,
> build_dir=request.build_dir)
>
> _, test_result = parse_tests(request, metadata, run_result)
> @@ -341,6 +354,11 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
> nargs='?',
> default='',
> metavar='filter_glob')
> + parser.add_argument('--filter',
> + help='Filter which KUnit tests run by attributes'
> + 'e.g. speed=fast or speed=>low',
> + type=str,
> + nargs='*')
> parser.add_argument('--kernel_args',
> help='Kernel command-line parameters. Maybe be repeated',
> action='append', metavar='')
> @@ -350,6 +368,8 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
> 'what ran before it.',
> type=str,
> choices=['suite', 'test'])
> + parser.add_argument('--list_tests', help='If set, list all tests and attributes.',
> + action='store_true')
>
> def add_parse_opts(parser: argparse.ArgumentParser) -> None:
> parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. '
> @@ -398,8 +418,10 @@ def run_handler(cli_args: argparse.Namespace) -> None:
> json=cli_args.json,
> timeout=cli_args.timeout,
> filter_glob=cli_args.filter_glob,
> + filter=cli_args.filter,
> kernel_args=cli_args.kernel_args,
> - run_isolated=cli_args.run_isolated)
> + run_isolated=cli_args.run_isolated,
> + list_tests=cli_args.list_tests)
> result = run_tests(linux, request)
> if result.status != KunitStatus.SUCCESS:
> sys.exit(1)
> @@ -441,8 +463,10 @@ def exec_handler(cli_args: argparse.Namespace) -> None:
> json=cli_args.json,
> timeout=cli_args.timeout,
> filter_glob=cli_args.filter_glob,
> + filter=cli_args.filter,
> kernel_args=cli_args.kernel_args,
> - run_isolated=cli_args.run_isolated)
> + run_isolated=cli_args.run_isolated,
> + list_tests=cli_args.list_tests)
> result = exec_tests(linux, exec_request)
> stdout.print_with_timestamp((
> 'Elapsed time: %.3fs\n') % (result.elapsed_time))
> diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
> index 7f648802caf6..62cb8200f60e 100644
> --- a/tools/testing/kunit/kunit_kernel.py
> +++ b/tools/testing/kunit/kunit_kernel.py
> @@ -330,11 +330,13 @@ class LinuxSourceTree:
> return False
> return self.validate_config(build_dir)
>
> - def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', timeout: Optional[int]=None) -> Iterator[str]:
> + def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: Optional[List[str]]=None, timeout: Optional[int]=None) -> Iterator[str]:
> if not args:
> args = []
> if filter_glob:
> - args.append('kunit.filter_glob='+filter_glob)
> + args.append('kunit.filter_glob=' + filter_glob)
> + if filter:
> + args.append('kunit.filter=' + (','.join(filter)))
> args.append('kunit.enable=1')
>
> process = self._ops.start(args, build_dir)
> diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
> index be35999bb84f..4a7f3112d06c 100755
> --- a/tools/testing/kunit/kunit_tool_test.py
> +++ b/tools/testing/kunit/kunit_tool_test.py
> @@ -14,6 +14,7 @@ import tempfile, shutil # Handling test_tmpdir
> import itertools
> import json
> import os
> +import re
> import signal
> import subprocess
> from typing import Iterable
> @@ -597,7 +598,7 @@ class KUnitMainTest(unittest.TestCase):
> self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 0)
> self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir='.kunit', filter_glob='', timeout=300)
> + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_run_passes_args_pass(self):
> @@ -605,7 +606,7 @@ class KUnitMainTest(unittest.TestCase):
> self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir='.kunit', filter_glob='', timeout=300)
> + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_exec_passes_args_fail(self):
> @@ -629,7 +630,7 @@ class KUnitMainTest(unittest.TestCase):
> kunit.main(['run'])
> self.assertEqual(e.exception.code, 1)
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir='.kunit', filter_glob='', timeout=300)
> + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> self.print_mock.assert_any_call(StrContains(' 0 tests run!'))
>
> def test_exec_raw_output(self):
> @@ -670,13 +671,13 @@ class KUnitMainTest(unittest.TestCase):
> self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
> kunit.main(['run', '--raw_output', 'filter_glob'])
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir='.kunit', filter_glob='filter_glob', timeout=300)
> + args=None, build_dir='.kunit', filter_glob='filter_glob', filter=None, timeout=300)
>
> def test_exec_timeout(self):
> timeout = 3453
> kunit.main(['exec', '--timeout', str(timeout)])
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir='.kunit', filter_glob='', timeout=timeout)
> + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=timeout)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_run_timeout(self):
> @@ -684,7 +685,7 @@ class KUnitMainTest(unittest.TestCase):
> kunit.main(['run', '--timeout', str(timeout)])
> self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir='.kunit', filter_glob='', timeout=timeout)
> + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=timeout)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_run_builddir(self):
> @@ -692,7 +693,7 @@ class KUnitMainTest(unittest.TestCase):
> kunit.main(['run', '--build_dir=.kunit'])
> self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir=build_dir, filter_glob='', timeout=300)
> + args=None, build_dir=build_dir, filter_glob='', filter=None, timeout=300)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_config_builddir(self):
> @@ -710,7 +711,7 @@ class KUnitMainTest(unittest.TestCase):
> build_dir = '.kunit'
> kunit.main(['exec', '--build_dir', build_dir])
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=None, build_dir=build_dir, filter_glob='', timeout=300)
> + args=None, build_dir=build_dir, filter_glob='', filter=None, timeout=300)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_run_kunitconfig(self):
> @@ -786,7 +787,7 @@ class KUnitMainTest(unittest.TestCase):
> kunit.main(['run', '--kernel_args=a=1', '--kernel_args=b=2'])
> self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=['a=1','b=2'], build_dir='.kunit', filter_glob='', timeout=300)
> + args=['a=1','b=2'], build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> self.print_mock.assert_any_call(StrContains('Testing complete.'))
>
> def test_list_tests(self):
> @@ -794,12 +795,13 @@ class KUnitMainTest(unittest.TestCase):
> self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
>
> got = kunit._list_tests(self.linux_source_mock,
> - kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'suite'))
> + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, None, 'suite', False))
> + tests = kunit._get_tests(got)
>
> - self.assertEqual(got, want)
> + self.assertEqual(tests, want)
> # Should respect the user's filter glob when listing tests.
> self.linux_source_mock.run_kernel.assert_called_once_with(
> - args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', timeout=300)
> + args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter=None, timeout=300)
>
>
> @mock.patch.object(kunit, '_list_tests')
> @@ -809,10 +811,10 @@ class KUnitMainTest(unittest.TestCase):
>
> # Should respect the user's filter glob when listing tests.
> mock_tests.assert_called_once_with(mock.ANY,
> - kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, 'suite'))
> + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, None, 'suite', False))
> self.linux_source_mock.run_kernel.assert_has_calls([
> - mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300),
> - mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300),
> + mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter=None, timeout=300),
> + mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter=None, timeout=300),
> ])
>
> @mock.patch.object(kunit, '_list_tests')
> @@ -822,13 +824,12 @@ class KUnitMainTest(unittest.TestCase):
>
> # Should respect the user's filter glob when listing tests.
> mock_tests.assert_called_once_with(mock.ANY,
> - kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'test'))
> + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, None, 'test', False))
> self.linux_source_mock.run_kernel.assert_has_calls([
> - mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300),
> - mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300),
> - mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', timeout=300),
> + mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter=None, timeout=300),
> + mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter=None, timeout=300),
> + mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter=None, timeout=300),
> ])
>
> -
> if __name__ == '__main__':
> unittest.main()
> --
> 2.41.0.162.gfafddb0af9-goog
>
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4003 bytes --]
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [RFC v1 4/6] kunit: tool: Add command line interface to filter and report attributes
2023-06-10 8:29 ` David Gow
@ 2023-06-13 20:44 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:44 UTC (permalink / raw)
To: David Gow
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 4:29 AM David Gow <davidgow@google.com> wrote:
>
> On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
> >
> > Add ability to use kunit.py to filter attributes and to report a list of
> > tests including attributes without running tests.
> >
> > Add flag "--filter" to input filters on test attributes. Tests will be
> > filtered out if they do not match all inputted filters.
> >
> > Example: --filter speed=slow
> >
> > This filter would run only the tests that are marked as slow. Note there
> > cannot be spaces within a filter.
>
> Within a filter's name, value, or the entire filter string. Is this a
> restriction we can remove?
>
Currently this implementation does not allow for spaces anywhere in
the filter string so for the example: "--filter speed=slow", there
cannot be spaces within "speed=slow".
I would be interested in removing this restriction by allowing the
user to use quotes if there are spaces.
I originally thought this would be potentially a future change.
However, it may be best to implement earlier as it would cause the
implementation to change quite a bit. The module_param_array may need
to be changed to be a string that is then parsed.
> >
> > As said in the previous patch, filters can have different operations: <, >,
> > <=, >=, !=, and =. Note that the characters < and > are often interpreted
> > by the shell, so they may need to be quoted or escaped.
> >
> > Example: --filter "speed>=normal" or –filter speed\>=normal
> >
> > This filter would run only the tests that have the speed faster than or
> > equal to normal.
> >
> > Add flag "--list_tests" to output a list of tests and their attributes
> > without running tests. This will be useful to see test attributes and which
> > tests will run with given filters.
>
> Please note that this comes from the kernel's kunit.action=list option.
Got it. Will do in the next version.
> >
> > Example of the output of these tests:
> > example
> > example.test_1
> > example.test_2
> > # example.test_2.speed: slow
> >
> > This output includes a suite, example, with two test cases, test_1 and
> > test_2. And in this instance test_2 has been marked as slow.
> >
>
> It's unrelated, so perhaps best split out into its own patch, but I'd
> love the option to list tests without the attributes as well. That
> would allow doing things like piping the list of tests to wc -l to
> count them, etc.
>
I really like this idea of allowing two options: list tests only and
then also include attributes. I wonder if I should include the tests
in the second option. My instinct would be yes (to show all tests not
just those with attributes) but let me know what you think.
Thanks!
-Rae
>
> > Signed-off-by: Rae Moar <rmoar@google.com>
> > ---
> > tools/testing/kunit/kunit.py | 34 +++++++++++++++++----
> > tools/testing/kunit/kunit_kernel.py | 6 ++--
> > tools/testing/kunit/kunit_tool_test.py | 41 +++++++++++++-------------
> > 3 files changed, 54 insertions(+), 27 deletions(-)
> >
> > diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
> > index 3905c43369c3..661c39f7acf5 100755
> > --- a/tools/testing/kunit/kunit.py
> > +++ b/tools/testing/kunit/kunit.py
> > @@ -55,8 +55,10 @@ class KunitExecRequest(KunitParseRequest):
> > build_dir: str
> > timeout: int
> > filter_glob: str
> > + filter: Optional[List[str]]
> > kernel_args: Optional[List[str]]
> > run_isolated: Optional[str]
> > + list_tests: Optional[bool]
> >
> > @dataclass
> > class KunitRequest(KunitExecRequest, KunitBuildRequest):
> > @@ -100,7 +102,7 @@ def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
> >
> > return build_tests(linux, request)
> >
> > -def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
> > +def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]:
> > args = ['kunit.action=list']
> > if request.kernel_args:
> > args.extend(request.kernel_args)
> > @@ -108,13 +110,17 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest)
> > output = linux.run_kernel(args=args,
> > timeout=request.timeout,
> > filter_glob=request.filter_glob,
> > + filter=request.filter,
> > build_dir=request.build_dir)
> > lines = kunit_parser.extract_tap_lines(output)
> > # Hack! Drop the dummy TAP version header that the executor prints out.
> > lines.pop()
> >
> > # Filter out any extraneous non-test output that might have gotten mixed in.
> > - return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
> > + return output
> > +
> > +def _get_tests(output: Iterable[str]) -> List[str]:
> > + return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
> >
> > def _suites_from_test_list(tests: List[str]) -> List[str]:
> > """Extracts all the suites from an ordered list of tests."""
> > @@ -132,8 +138,14 @@ def _suites_from_test_list(tests: List[str]) -> List[str]:
> >
> > def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
> > filter_globs = [request.filter_glob]
> > + if request.list_tests:
> > + output = _list_tests(linux, request)
> > + for line in output:
> > + print(line.rstrip())
> > + return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0)
> > if request.run_isolated:
> > - tests = _list_tests(linux, request)
> > + output = _list_tests(linux, request)
> > + tests = _get_tests(output)
> > if request.run_isolated == 'test':
> > filter_globs = tests
> > elif request.run_isolated == 'suite':
> > @@ -155,6 +167,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
> > args=request.kernel_args,
> > timeout=request.timeout,
> > filter_glob=filter_glob,
> > + filter=request.filter,
> > build_dir=request.build_dir)
> >
> > _, test_result = parse_tests(request, metadata, run_result)
> > @@ -341,6 +354,11 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
> > nargs='?',
> > default='',
> > metavar='filter_glob')
> > + parser.add_argument('--filter',
> > + help='Filter which KUnit tests run by attributes'
> > + 'e.g. speed=fast or speed=>low',
> > + type=str,
> > + nargs='*')
> > parser.add_argument('--kernel_args',
> > help='Kernel command-line parameters. Maybe be repeated',
> > action='append', metavar='')
> > @@ -350,6 +368,8 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None:
> > 'what ran before it.',
> > type=str,
> > choices=['suite', 'test'])
> > + parser.add_argument('--list_tests', help='If set, list all tests and attributes.',
> > + action='store_true')
> >
> > def add_parse_opts(parser: argparse.ArgumentParser) -> None:
> > parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. '
> > @@ -398,8 +418,10 @@ def run_handler(cli_args: argparse.Namespace) -> None:
> > json=cli_args.json,
> > timeout=cli_args.timeout,
> > filter_glob=cli_args.filter_glob,
> > + filter=cli_args.filter,
> > kernel_args=cli_args.kernel_args,
> > - run_isolated=cli_args.run_isolated)
> > + run_isolated=cli_args.run_isolated,
> > + list_tests=cli_args.list_tests)
> > result = run_tests(linux, request)
> > if result.status != KunitStatus.SUCCESS:
> > sys.exit(1)
> > @@ -441,8 +463,10 @@ def exec_handler(cli_args: argparse.Namespace) -> None:
> > json=cli_args.json,
> > timeout=cli_args.timeout,
> > filter_glob=cli_args.filter_glob,
> > + filter=cli_args.filter,
> > kernel_args=cli_args.kernel_args,
> > - run_isolated=cli_args.run_isolated)
> > + run_isolated=cli_args.run_isolated,
> > + list_tests=cli_args.list_tests)
> > result = exec_tests(linux, exec_request)
> > stdout.print_with_timestamp((
> > 'Elapsed time: %.3fs\n') % (result.elapsed_time))
> > diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
> > index 7f648802caf6..62cb8200f60e 100644
> > --- a/tools/testing/kunit/kunit_kernel.py
> > +++ b/tools/testing/kunit/kunit_kernel.py
> > @@ -330,11 +330,13 @@ class LinuxSourceTree:
> > return False
> > return self.validate_config(build_dir)
> >
> > - def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', timeout: Optional[int]=None) -> Iterator[str]:
> > + def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: Optional[List[str]]=None, timeout: Optional[int]=None) -> Iterator[str]:
> > if not args:
> > args = []
> > if filter_glob:
> > - args.append('kunit.filter_glob='+filter_glob)
> > + args.append('kunit.filter_glob=' + filter_glob)
> > + if filter:
> > + args.append('kunit.filter=' + (','.join(filter)))
> > args.append('kunit.enable=1')
> >
> > process = self._ops.start(args, build_dir)
> > diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
> > index be35999bb84f..4a7f3112d06c 100755
> > --- a/tools/testing/kunit/kunit_tool_test.py
> > +++ b/tools/testing/kunit/kunit_tool_test.py
> > @@ -14,6 +14,7 @@ import tempfile, shutil # Handling test_tmpdir
> > import itertools
> > import json
> > import os
> > +import re
> > import signal
> > import subprocess
> > from typing import Iterable
> > @@ -597,7 +598,7 @@ class KUnitMainTest(unittest.TestCase):
> > self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 0)
> > self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir='.kunit', filter_glob='', timeout=300)
> > + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_run_passes_args_pass(self):
> > @@ -605,7 +606,7 @@ class KUnitMainTest(unittest.TestCase):
> > self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> > self.assertEqual(self.linux_source_mock.run_kernel.call_count, 1)
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir='.kunit', filter_glob='', timeout=300)
> > + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_exec_passes_args_fail(self):
> > @@ -629,7 +630,7 @@ class KUnitMainTest(unittest.TestCase):
> > kunit.main(['run'])
> > self.assertEqual(e.exception.code, 1)
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir='.kunit', filter_glob='', timeout=300)
> > + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> > self.print_mock.assert_any_call(StrContains(' 0 tests run!'))
> >
> > def test_exec_raw_output(self):
> > @@ -670,13 +671,13 @@ class KUnitMainTest(unittest.TestCase):
> > self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
> > kunit.main(['run', '--raw_output', 'filter_glob'])
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir='.kunit', filter_glob='filter_glob', timeout=300)
> > + args=None, build_dir='.kunit', filter_glob='filter_glob', filter=None, timeout=300)
> >
> > def test_exec_timeout(self):
> > timeout = 3453
> > kunit.main(['exec', '--timeout', str(timeout)])
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir='.kunit', filter_glob='', timeout=timeout)
> > + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=timeout)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_run_timeout(self):
> > @@ -684,7 +685,7 @@ class KUnitMainTest(unittest.TestCase):
> > kunit.main(['run', '--timeout', str(timeout)])
> > self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir='.kunit', filter_glob='', timeout=timeout)
> > + args=None, build_dir='.kunit', filter_glob='', filter=None, timeout=timeout)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_run_builddir(self):
> > @@ -692,7 +693,7 @@ class KUnitMainTest(unittest.TestCase):
> > kunit.main(['run', '--build_dir=.kunit'])
> > self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir=build_dir, filter_glob='', timeout=300)
> > + args=None, build_dir=build_dir, filter_glob='', filter=None, timeout=300)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_config_builddir(self):
> > @@ -710,7 +711,7 @@ class KUnitMainTest(unittest.TestCase):
> > build_dir = '.kunit'
> > kunit.main(['exec', '--build_dir', build_dir])
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=None, build_dir=build_dir, filter_glob='', timeout=300)
> > + args=None, build_dir=build_dir, filter_glob='', filter=None, timeout=300)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_run_kunitconfig(self):
> > @@ -786,7 +787,7 @@ class KUnitMainTest(unittest.TestCase):
> > kunit.main(['run', '--kernel_args=a=1', '--kernel_args=b=2'])
> > self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=['a=1','b=2'], build_dir='.kunit', filter_glob='', timeout=300)
> > + args=['a=1','b=2'], build_dir='.kunit', filter_glob='', filter=None, timeout=300)
> > self.print_mock.assert_any_call(StrContains('Testing complete.'))
> >
> > def test_list_tests(self):
> > @@ -794,12 +795,13 @@ class KUnitMainTest(unittest.TestCase):
> > self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
> >
> > got = kunit._list_tests(self.linux_source_mock,
> > - kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'suite'))
> > + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, None, 'suite', False))
> > + tests = kunit._get_tests(got)
> >
> > - self.assertEqual(got, want)
> > + self.assertEqual(tests, want)
> > # Should respect the user's filter glob when listing tests.
> > self.linux_source_mock.run_kernel.assert_called_once_with(
> > - args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', timeout=300)
> > + args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter=None, timeout=300)
> >
> >
> > @mock.patch.object(kunit, '_list_tests')
> > @@ -809,10 +811,10 @@ class KUnitMainTest(unittest.TestCase):
> >
> > # Should respect the user's filter glob when listing tests.
> > mock_tests.assert_called_once_with(mock.ANY,
> > - kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, 'suite'))
> > + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, None, 'suite', False))
> > self.linux_source_mock.run_kernel.assert_has_calls([
> > - mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300),
> > - mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300),
> > + mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter=None, timeout=300),
> > + mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter=None, timeout=300),
> > ])
> >
> > @mock.patch.object(kunit, '_list_tests')
> > @@ -822,13 +824,12 @@ class KUnitMainTest(unittest.TestCase):
> >
> > # Should respect the user's filter glob when listing tests.
> > mock_tests.assert_called_once_with(mock.ANY,
> > - kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'test'))
> > + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, None, 'test', False))
> > self.linux_source_mock.run_kernel.assert_has_calls([
> > - mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300),
> > - mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300),
> > - mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', timeout=300),
> > + mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter=None, timeout=300),
> > + mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter=None, timeout=300),
> > + mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter=None, timeout=300),
> > ])
> >
> > -
> > if __name__ == '__main__':
> > unittest.main()
> > --
> > 2.41.0.162.gfafddb0af9-goog
> >
^ permalink raw reply [flat|nested] 21+ messages in thread
* [RFC v1 5/6] kunit: memcpy: Mark tests as slow using test attributes
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
` (3 preceding siblings ...)
2023-06-10 0:51 ` [RFC v1 4/6] kunit: tool: Add command line interface to filter and report attributes Rae Moar
@ 2023-06-10 0:51 ` Rae Moar
2023-06-10 8:29 ` David Gow
2023-06-10 0:51 ` [RFC v1 6/6] kunit: time: Mark test " Rae Moar
2023-06-10 8:29 ` [RFC v1 0/6] kunit: Add test attributes API David Gow
6 siblings, 1 reply; 21+ messages in thread
From: Rae Moar @ 2023-06-10 0:51 UTC (permalink / raw)
To: shuah, davidgow, dlatypov, brendan.higgins
Cc: linux-kselftest, kunit-dev, linux-kernel, keescook,
linux-hardening, jstultz, tglx, sboyd, Rae Moar
Mark slow memcpy KUnit tests using test attributes.
Tests marked as slow are as follows: memcpy_large_test, memmove_test,
memmove_large_test, and memmove_overlap_test.
These tests were the slowest of the memcpy tests and relatively slower to
most other KUnit tests. Most of these tests are already skipped when
CONFIG_MEMCPY_SLOW_KUNIT_TEST is not enabled.
These tests can now be filtered on using the KUnit test attribute filtering
feature. Example: --filter "speed>slow". This will run only the tests that
have speeds faster than slow. The slow attribute will also be outputted in
KTAP.
Signed-off-by: Rae Moar <rmoar@google.com>
---
lib/memcpy_kunit.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c
index 887926f04731..440aee705ccc 100644
--- a/lib/memcpy_kunit.c
+++ b/lib/memcpy_kunit.c
@@ -551,10 +551,10 @@ static void strtomem_test(struct kunit *test)
static struct kunit_case memcpy_test_cases[] = {
KUNIT_CASE(memset_test),
KUNIT_CASE(memcpy_test),
- KUNIT_CASE(memcpy_large_test),
- KUNIT_CASE(memmove_test),
- KUNIT_CASE(memmove_large_test),
- KUNIT_CASE(memmove_overlap_test),
+ KUNIT_CASE_SLOW(memcpy_large_test),
+ KUNIT_CASE_SLOW(memmove_test),
+ KUNIT_CASE_SLOW(memmove_large_test),
+ KUNIT_CASE_SLOW(memmove_overlap_test),
KUNIT_CASE(strtomem_test),
{}
};
--
2.41.0.162.gfafddb0af9-goog
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [RFC v1 5/6] kunit: memcpy: Mark tests as slow using test attributes
2023-06-10 0:51 ` [RFC v1 5/6] kunit: memcpy: Mark tests as slow using test attributes Rae Moar
@ 2023-06-10 8:29 ` David Gow
2023-06-13 20:44 ` Rae Moar
0 siblings, 1 reply; 21+ messages in thread
From: David Gow @ 2023-06-10 8:29 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
[-- Attachment #1: Type: text/plain, Size: 2123 bytes --]
On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
>
> Mark slow memcpy KUnit tests using test attributes.
>
> Tests marked as slow are as follows: memcpy_large_test, memmove_test,
> memmove_large_test, and memmove_overlap_test.
>
> These tests were the slowest of the memcpy tests and relatively slower to
> most other KUnit tests. Most of these tests are already skipped when
> CONFIG_MEMCPY_SLOW_KUNIT_TEST is not enabled.
I assume the plan will be to eventually remove the
CONFIG_MEMCPY_SLOW_KUNIT_TEST option and just rely on the "speed"
attribute to filter these out. That has the disadvantage that the
tests will still be built, but is probably the nicer long-term
solution.
I suppose we could remove it in this patch, too, but I suspect it
makes more sense to have a deprecation period to make sure the
attributes are working well. That being said, maybe add a note to the
CONFIG_MEMCPY_SLOW_KUNIT_TEST help text to advertise this?
>
> These tests can now be filtered on using the KUnit test attribute filtering
> feature. Example: --filter "speed>slow". This will run only the tests that
> have speeds faster than slow. The slow attribute will also be outputted in
> KTAP.
>
> Signed-off-by: Rae Moar <rmoar@google.com>
> ---
> lib/memcpy_kunit.c | 8 ++++----
> 1 file changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c
> index 887926f04731..440aee705ccc 100644
> --- a/lib/memcpy_kunit.c
> +++ b/lib/memcpy_kunit.c
> @@ -551,10 +551,10 @@ static void strtomem_test(struct kunit *test)
> static struct kunit_case memcpy_test_cases[] = {
> KUNIT_CASE(memset_test),
> KUNIT_CASE(memcpy_test),
> - KUNIT_CASE(memcpy_large_test),
> - KUNIT_CASE(memmove_test),
> - KUNIT_CASE(memmove_large_test),
> - KUNIT_CASE(memmove_overlap_test),
> + KUNIT_CASE_SLOW(memcpy_large_test),
> + KUNIT_CASE_SLOW(memmove_test),
> + KUNIT_CASE_SLOW(memmove_large_test),
> + KUNIT_CASE_SLOW(memmove_overlap_test),
> KUNIT_CASE(strtomem_test),
> {}
> };
> --
> 2.41.0.162.gfafddb0af9-goog
>
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4003 bytes --]
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [RFC v1 5/6] kunit: memcpy: Mark tests as slow using test attributes
2023-06-10 8:29 ` David Gow
@ 2023-06-13 20:44 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:44 UTC (permalink / raw)
To: David Gow
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 4:29 AM David Gow <davidgow@google.com> wrote:
>
> On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
> >
> > Mark slow memcpy KUnit tests using test attributes.
> >
> > Tests marked as slow are as follows: memcpy_large_test, memmove_test,
> > memmove_large_test, and memmove_overlap_test.
> >
> > These tests were the slowest of the memcpy tests and relatively slower to
> > most other KUnit tests. Most of these tests are already skipped when
> > CONFIG_MEMCPY_SLOW_KUNIT_TEST is not enabled.
>
> I assume the plan will be to eventually remove the
> CONFIG_MEMCPY_SLOW_KUNIT_TEST option and just rely on the "speed"
> attribute to filter these out. That has the disadvantage that the
> tests will still be built, but is probably the nicer long-term
> solution.
>
> I suppose we could remove it in this patch, too, but I suspect it
> makes more sense to have a deprecation period to make sure the
> attributes are working well. That being said, maybe add a note to the
> CONFIG_MEMCPY_SLOW_KUNIT_TEST help text to advertise this?
>
Yes that was the plan but I should definitely document that here and
then I like the idea for adding the note with
CONFIG_MEMCPY_SLOW_KUNIT_TEST.
Thanks!
-Rae
>
> >
> > These tests can now be filtered on using the KUnit test attribute filtering
> > feature. Example: --filter "speed>slow". This will run only the tests that
> > have speeds faster than slow. The slow attribute will also be outputted in
> > KTAP.
> >
> > Signed-off-by: Rae Moar <rmoar@google.com>
> > ---
> > lib/memcpy_kunit.c | 8 ++++----
> > 1 file changed, 4 insertions(+), 4 deletions(-)
> >
> > diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c
> > index 887926f04731..440aee705ccc 100644
> > --- a/lib/memcpy_kunit.c
> > +++ b/lib/memcpy_kunit.c
> > @@ -551,10 +551,10 @@ static void strtomem_test(struct kunit *test)
> > static struct kunit_case memcpy_test_cases[] = {
> > KUNIT_CASE(memset_test),
> > KUNIT_CASE(memcpy_test),
> > - KUNIT_CASE(memcpy_large_test),
> > - KUNIT_CASE(memmove_test),
> > - KUNIT_CASE(memmove_large_test),
> > - KUNIT_CASE(memmove_overlap_test),
> > + KUNIT_CASE_SLOW(memcpy_large_test),
> > + KUNIT_CASE_SLOW(memmove_test),
> > + KUNIT_CASE_SLOW(memmove_large_test),
> > + KUNIT_CASE_SLOW(memmove_overlap_test),
> > KUNIT_CASE(strtomem_test),
> > {}
> > };
> > --
> > 2.41.0.162.gfafddb0af9-goog
> >
^ permalink raw reply [flat|nested] 21+ messages in thread
* [RFC v1 6/6] kunit: time: Mark test as slow using test attributes
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
` (4 preceding siblings ...)
2023-06-10 0:51 ` [RFC v1 5/6] kunit: memcpy: Mark tests as slow using test attributes Rae Moar
@ 2023-06-10 0:51 ` Rae Moar
2023-06-10 8:29 ` [RFC v1 0/6] kunit: Add test attributes API David Gow
6 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-10 0:51 UTC (permalink / raw)
To: shuah, davidgow, dlatypov, brendan.higgins
Cc: linux-kselftest, kunit-dev, linux-kernel, keescook,
linux-hardening, jstultz, tglx, sboyd, Rae Moar
Mark the time KUnit test, time64_to_tm_test_date_range, as slow using test
attributes.
This test ran relatively much slower than most other KUnit tests.
By marking this test as slow, the test can now be filtered on using the
KUnit test attribute filtering feature. Example: --filter "speed>slow".
This will run only the tests that have speeds faster than slow. The slow
attribute will also be outputted in KTAP.
Signed-off-by: Rae Moar <rmoar@google.com>
---
kernel/time/time_test.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/time/time_test.c b/kernel/time/time_test.c
index 831e8e779ace..ca058c8af6ba 100644
--- a/kernel/time/time_test.c
+++ b/kernel/time/time_test.c
@@ -86,7 +86,7 @@ static void time64_to_tm_test_date_range(struct kunit *test)
}
static struct kunit_case time_test_cases[] = {
- KUNIT_CASE(time64_to_tm_test_date_range),
+ KUNIT_CASE_SLOW(time64_to_tm_test_date_range),
{}
};
--
2.41.0.162.gfafddb0af9-goog
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [RFC v1 0/6] kunit: Add test attributes API
2023-06-10 0:51 [RFC v1 0/6] kunit: Add test attributes API Rae Moar
` (5 preceding siblings ...)
2023-06-10 0:51 ` [RFC v1 6/6] kunit: time: Mark test " Rae Moar
@ 2023-06-10 8:29 ` David Gow
2023-06-13 20:34 ` Rae Moar
6 siblings, 1 reply; 21+ messages in thread
From: David Gow @ 2023-06-10 8:29 UTC (permalink / raw)
To: Rae Moar
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
[-- Attachment #1: Type: text/plain, Size: 4997 bytes --]
On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
>
> Hello everyone,
>
> This is an RFC patch series to propose the addition of a test attributes
> framework to KUnit.
>
> There has been interest in filtering out "slow" KUnit tests. Most notably,
> a new config, CONFIG_MEMCPY_SLOW_KUNIT_TEST, has been added to exclude
> particularly slow memcpy tests
> (https://lore.kernel.org/all/20230118200653.give.574-kees@kernel.org/).
Awesome: this is a long overdue feature.
> This proposed attributes framework would be used to save and access test
> associated data, including whether a test is slow. These attributes would
> be reportable (via KTAP and command line output) and some will be
> filterable.
Why wouldn't they all be filterable? I guess I can imagine some where
filtering wouldn't be useful, but I can't think of a technical reason
why the filter shouldn't work.
Also, as I understand it, I think this could also work with data which
is not "saved" in this kunit_attributes struct you define. So we could
have attributes which are generated automatically from other
information about the test.
I could definitely see value in being able to filter on things like
"is_parameterised" or "runs_at_init" or similar.
Finally, it'd be really great if these attributes could apply to
individual parameters in parameterised tests, in which case we could
have and filter on the parameter value. That seems like it could be
incredibly useful.
>
> This framework is designed to allow for the addition of other attributes in
> the future. These attributes could include whether the test is flaky,
> associated test files, etc.
A small part of me is hesitant to add this much framework code for
only one attribute, so it'd be nice to look into at least having an
RFC for some of these. Hopefully we don't actually have flaky tests,
but "is_deterministic" would be nice (alongside a future ability to
inject a random seed, or similar). Other properties like
"is_threadsafe", "is_reentrant", etc could be useful for future
features. And I'm sure there could be some more subsystem-specific
things which would be useful to filter on, too.
Some of these could probably replace the need for custom code to make
the test skip itself if dependencies aren't met, too, which would be
fun.
I'm not sure "associated test files" quite gels perfectly with this
system as-is (assuming I understand what that refers to). If it's the
ability to "attach" extra data (logs, etc) to the KTAP output, that's
possibly best injected at runtime, or added by the separate parser,
rather than hardcoded in the kernel.
>
> Note that this could intersect with the discussions on how to format
> test-associated data in KTAP v2 that I am also involved in
> (https://lore.kernel.org/all/20230420205734.1288498-1-rmoar@google.com/).
>
I definitely need to re-read and respond to that. I'm not 100%
thrilled with the output format here, and I think the goal with KTAP
"test associated data" is, as you say, related, but not identical to
this. There'd definitely be data which doesn't make sense as a KUnit
attribute which we might want to add to the output (e.g., data only
calcuated while the test runs), and attributes which we might not want
to always print out with the results.
> If the overall idea seems good, I'll make sure to add tests/documentation,
> and more patches marking existing tests as slow to the patch series.
>
I think the series is good overall. If no-one else objects, let's move
forward with it.
I'd definitely prefer to see a few more tests and some documentation.
Having another attribute would be great, too, though I can certainly
live with that being a separate series.
> Thanks!
> Rae
>
> Rae Moar (6):
> kunit: Add test attributes API structure
> kunit: Add speed attribute
> kunit: Add ability to filter attributes
> kunit: tool: Add command line interface to filter and report
> attributes
> kunit: memcpy: Mark tests as slow using test attributes
> kunit: time: Mark test as slow using test attributes
>
> include/kunit/attributes.h | 41 ++++
> include/kunit/test.h | 62 ++++++
> kernel/time/time_test.c | 2 +-
> lib/kunit/Makefile | 3 +-
> lib/kunit/attributes.c | 280 +++++++++++++++++++++++++
> lib/kunit/executor.c | 89 ++++++--
> lib/kunit/executor_test.c | 8 +-
> lib/kunit/kunit-example-test.c | 9 +
> lib/kunit/test.c | 17 +-
> lib/memcpy_kunit.c | 8 +-
> tools/testing/kunit/kunit.py | 34 ++-
> tools/testing/kunit/kunit_kernel.py | 6 +-
> tools/testing/kunit/kunit_tool_test.py | 41 ++--
> 13 files changed, 536 insertions(+), 64 deletions(-)
> create mode 100644 include/kunit/attributes.h
> create mode 100644 lib/kunit/attributes.c
>
>
> base-commit: fefdb43943c1a0d87e1b43ae4d03e5f9a1d058f4
> --
> 2.41.0.162.gfafddb0af9-goog
>
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4003 bytes --]
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [RFC v1 0/6] kunit: Add test attributes API
2023-06-10 8:29 ` [RFC v1 0/6] kunit: Add test attributes API David Gow
@ 2023-06-13 20:34 ` Rae Moar
0 siblings, 0 replies; 21+ messages in thread
From: Rae Moar @ 2023-06-13 20:34 UTC (permalink / raw)
To: David Gow
Cc: shuah, dlatypov, brendan.higgins, linux-kselftest, kunit-dev,
linux-kernel, keescook, linux-hardening, jstultz, tglx, sboyd
On Sat, Jun 10, 2023 at 4:29 AM David Gow <davidgow@google.com> wrote:
>
> On Sat, 10 Jun 2023 at 08:52, Rae Moar <rmoar@google.com> wrote:
> >
> > Hello everyone,
> >
> > This is an RFC patch series to propose the addition of a test attributes
> > framework to KUnit.
> >
> > There has been interest in filtering out "slow" KUnit tests. Most notably,
> > a new config, CONFIG_MEMCPY_SLOW_KUNIT_TEST, has been added to exclude
> > particularly slow memcpy tests
> > (https://lore.kernel.org/all/20230118200653.give.574-kees@kernel.org/).
>
> Awesome: this is a long overdue feature.
>
Hi David!
Thank you for all the comments!
> > This proposed attributes framework would be used to save and access test
> > associated data, including whether a test is slow. These attributes would
> > be reportable (via KTAP and command line output) and some will be
> > filterable.
>
> Why wouldn't they all be filterable? I guess I can imagine some where
> filtering wouldn't be useful, but I can't think of a technical reason
> why the filter shouldn't work.
I am definitely open to all attributes being filterable. My
reservation is that I can imagine an attribute with a complex data
type that would cause the filtering method to be difficult to
implement. If the attribute does not benefit much from being
filterable, I wonder if it is worth requiring the filtering method to
be implemented in that case.
Perhaps for now all attributes are filterable and then if this becomes
the case, this is addressed then?
>
> Also, as I understand it, I think this could also work with data which
> is not "saved" in this kunit_attributes struct you define. So we could
> have attributes which are generated automatically from other
> information about the test.
> I could definitely see value in being able to filter on things like
> "is_parameterised" or "runs_at_init" or similar.
>
Yes! This is a great benefit of this flexible structure for
attributes. I would definitely be interested in implementing
"is_parameterised" and "runs_at_init" in future patches.
> Finally, it'd be really great if these attributes could apply to
> individual parameters in parameterised tests, in which case we could
> have and filter on the parameter value. That seems like it could be
> incredibly useful.
>
Yes, this would be an exciting extension for this project. I have
started thinking about this as potentially a follow up project.
> >
> > This framework is designed to allow for the addition of other attributes in
> > the future. These attributes could include whether the test is flaky,
> > associated test files, etc.
>
> A small part of me is hesitant to add this much framework code for
> only one attribute, so it'd be nice to look into at least having an
> RFC for some of these. Hopefully we don't actually have flaky tests,
> but "is_deterministic" would be nice (alongside a future ability to
> inject a random seed, or similar). Other properties like
> "is_threadsafe", "is_reentrant", etc could be useful for future
> features. And I'm sure there could be some more subsystem-specific
> things which would be useful to filter on, too.
>
I understand the reservations to add this large framework for one
attribute. I would definitely consider adding an additional attribute
to this RFC or creating a separate RFC.
I would be happy to go ahead and add "is_deterministic" if there is
interest. As well as potentially "is_threadsafe", "is_reentrant", etc
in future patches.
> Some of these could probably replace the need for custom code to make
> the test skip itself if dependencies aren't met, too, which would be
> fun.
This would be great!
>
> I'm not sure "associated test files" quite gels perfectly with this
> system as-is (assuming I understand what that refers to). If it's the
> ability to "attach" extra data (logs, etc) to the KTAP output, that's
> possibly best injected at runtime, or added by the separate parser,
> rather than hardcoded in the kernel.
>
Hmm I see what you are saying here. If the associated test files of
interest are best injected at runtime I am happy to scrap this idea
for now.
> >
> > Note that this could intersect with the discussions on how to format
> > test-associated data in KTAP v2 that I am also involved in
> > (https://lore.kernel.org/all/20230420205734.1288498-1-rmoar@google.com/).
> >
> I definitely need to re-read and respond to that. I'm not 100%
> thrilled with the output format here, and I think the goal with KTAP
> "test associated data" is, as you say, related, but not identical to
> this. There'd definitely be data which doesn't make sense as a KUnit
> attribute which we might want to add to the output (e.g., data only
> calcuated while the test runs), and attributes which we might not want
> to always print out with the results.
>
I have thought much about the differences between the two concepts. My
current understanding with KTAP metadata and KUnit attributes is that
they are not going to be perfect mirrors of each other but the KUnit
attributes framework can help to save and output some of the KTAP
metadata.
I am happy to be flexible on the output format or see more discussion
on KTAP metadata in general. What part of the output is most
concerning?
> > If the overall idea seems good, I'll make sure to add tests/documentation,
> > and more patches marking existing tests as slow to the patch series.
> >
>
> I think the series is good overall. If no-one else objects, let's move
> forward with it.
> I'd definitely prefer to see a few more tests and some documentation.
> Having another attribute would be great, too, though I can certainly
> live with that being a separate series.
Great! Yes I will add documentation and more tests in the next
versions. I will also work on the implementation of another attribute.
>
>
> > Thanks!
> > Rae
> >
> > Rae Moar (6):
> > kunit: Add test attributes API structure
> > kunit: Add speed attribute
> > kunit: Add ability to filter attributes
> > kunit: tool: Add command line interface to filter and report
> > attributes
> > kunit: memcpy: Mark tests as slow using test attributes
> > kunit: time: Mark test as slow using test attributes
> >
> > include/kunit/attributes.h | 41 ++++
> > include/kunit/test.h | 62 ++++++
> > kernel/time/time_test.c | 2 +-
> > lib/kunit/Makefile | 3 +-
> > lib/kunit/attributes.c | 280 +++++++++++++++++++++++++
> > lib/kunit/executor.c | 89 ++++++--
> > lib/kunit/executor_test.c | 8 +-
> > lib/kunit/kunit-example-test.c | 9 +
> > lib/kunit/test.c | 17 +-
> > lib/memcpy_kunit.c | 8 +-
> > tools/testing/kunit/kunit.py | 34 ++-
> > tools/testing/kunit/kunit_kernel.py | 6 +-
> > tools/testing/kunit/kunit_tool_test.py | 41 ++--
> > 13 files changed, 536 insertions(+), 64 deletions(-)
> > create mode 100644 include/kunit/attributes.h
> > create mode 100644 lib/kunit/attributes.c
> >
> >
> > base-commit: fefdb43943c1a0d87e1b43ae4d03e5f9a1d058f4
> > --
> > 2.41.0.162.gfafddb0af9-goog
> >
^ permalink raw reply [flat|nested] 21+ messages in thread