[PATCH bpf-next 0/3] selftests/bpf: Fix prog_tests framework self-tests

[PATCH bpf-next 0/3] selftests/bpf: Fix prog_tests framework self-tests

[KaFai Wan]

Three prog_tests framework self-tests fail depending on how test_progs
is invoked:

- ./test_progs -t prog_tests_framework/test_fail -v
  test_prog_tests_framework:FAIL:skip_cnt_check
  unexpected skip_cnt_check: actual 0 != expected 2

- ./test_progs -t prog_tests_framework_expected_msgs -v
  Caught signal #11!
  Stack trace:
  ./test_progs(crash_handler+0x38)[0x55620006fbb0]

- ./test_progs -t prog_tests_framework_compare_text -v
  test_prog_tests_framework_compare_text:PASS:mismatch_rc 0 nsec
  Caught signal #11!
  Stack trace:
  ./test_progs(crash_handler+0x38)[0x564ddbfcfbb0]

Patch #1 counts the subtests that actually ran, so the skip and error
count checks no longer fail when -t or -n filters out some subtests.

Patches #2 and #3 skip the subtests that inspect the log buffer when the
output is not captured, such as with -v in non-parallel mode, instead of
crashing on a NULL log_buf.

---
KaFai Wan (3):
  selftests/bpf: Fix prog_tests framework subtest count checks
  selftests/bpf: Skip expected_msgs subtests when stdout is not saved
  selftests/bpf: Skip compare_text subtest when stdout is not saved

 .../bpf/prog_tests/prog_tests_framework.c     | 34 ++++++++++++++-----
 1 file changed, 25 insertions(+), 9 deletions(-)

-- 
2.43.0

[PATCH bpf-next 1/3] selftests/bpf: Fix prog_tests framework subtest count checks

[KaFai Wan]

The hardcoded expected skip_cnt/error_cnt in test_prog_tests_framework()
mismatch the real count when the binary is run with -t or -n filters,
since some subtests are not selected. Count the executed subtests and
compare against that value instead.

Fixes: 2324257dbd68 ("selftests/bpf: Refactor prog_tests logging and test execution")
Signed-off-by: KaFai Wan <kafai.wan@linux.dev>
---
 .../selftests/bpf/prog_tests/prog_tests_framework.c    | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
index d111fe105447..53347f379924 100644
--- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
+++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
@@ -13,6 +13,7 @@ static void clear_test_state(struct test_state *state)
 void test_prog_tests_framework(void)
 {
 	struct test_state *state = env.test_state;
+	int subtest_cnt;
 
 	/* in all the ASSERT calls below we need to return on the first
 	 * error due to the fact that we are cleaning the test state after
@@ -30,25 +31,30 @@ void test_prog_tests_framework(void)
 		return;
 	clear_test_state(state);
 
+	subtest_cnt = 0;
 	if (test__start_subtest("test_skip_subtest")) {
 		test__skip();
 		test__end_subtest();
+		subtest_cnt++;
 	}
 	if (test__start_subtest("test_skip_subtest")) {
 		test__skip();
 		test__end_subtest();
+		subtest_cnt++;
 	}
-	if (!ASSERT_EQ(state->skip_cnt, 2, "skip_cnt_check"))
+	if (!ASSERT_EQ(state->skip_cnt, subtest_cnt, "skip_cnt_check"))
 		return;
 	if (!ASSERT_EQ(state->subtest_num, 3, "subtest_num_check"))
 		return;
 	clear_test_state(state);
 
+	subtest_cnt = 0;
 	if (test__start_subtest("test_fail_subtest")) {
 		test__fail();
 		test__end_subtest();
+		subtest_cnt++;
 	}
-	if (!ASSERT_EQ(state->error_cnt, 1, "error_cnt_check"))
+	if (!ASSERT_EQ(state->error_cnt, subtest_cnt, "error_cnt_check"))
 		return;
 	if (!ASSERT_EQ(state->subtest_num, 4, "subtest_num_check"))
 		return;
-- 
2.43.0

[PATCH bpf-next 2/3] selftests/bpf: Skip expected_msgs subtests when stdout is not saved

[KaFai Wan]

stdio_hijack_init() does not set subtest_state->stdout_saved when
running with -v in non-parallel mode, since output goes straight to
stdout. In that case subtest_state->log_buf is left NULL, so the
expected-messages subtests crash when inspecting it. Skip them when
stdout was not saved.

Fixes: 34c513be3dad ("selftests/bpf: __not_msg() tag for test_loader framework")
Signed-off-by: KaFai Wan <kafai.wan@linux.dev>
---
 .../testing/selftests/bpf/prog_tests/prog_tests_framework.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
index 53347f379924..b4cd7c7ff439 100644
--- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
+++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
@@ -169,6 +169,12 @@ void test_prog_tests_framework_expected_msgs(void)
 
 	for (i = 0; i < ARRAY_SIZE(cases); i++) {
 		if (test__start_subtest(cases[i].name)) {
+			if (!env.subtest_state->stdout_saved) {
+				test__skip();
+				test__end_subtest();
+				continue;
+			}
+
 			error_cnt = env.subtest_state->error_cnt;
 			msgs.patterns = cases[i].pats;
 			msgs.cnt = 0;
-- 
2.43.0

[PATCH bpf-next 3/3] selftests/bpf: Skip compare_text subtest when stdout is not saved

[KaFai Wan]

stdio_hijack_init() does not set subtest_state->stdout_saved when
running with -v in non-parallel mode, since output goes straight to
stdout. In that case subtest_state->log_buf is left NULL, so the
compare_text_mismatch subtest crashes when inspecting it. Skip it when
stdout was not saved.

Fixes: bd71904697c6 ("selftests/bpf: Add compare_text_to_expected() helper")
Signed-off-by: KaFai Wan <kafai.wan@linux.dev>
---
 .../bpf/prog_tests/prog_tests_framework.c      | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
index b4cd7c7ff439..ddb6cdc1579f 100644
--- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
+++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c
@@ -203,14 +203,18 @@ void test_prog_tests_framework_compare_text(void)
 	}
 
 	if (test__start_subtest("compare_text_mismatch")) {
-		err = compare_text_to_expected("line two\n", "line one\n");
-		fflush(stdout);
+		if (!env.subtest_state->stdout_saved) {
+			test__skip();
+		} else {
+			err = compare_text_to_expected("line two\n", "line one\n");
+			fflush(stdout);
 
-		ASSERT_EQ(err, -1, "mismatch_rc");
-		ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one",
-				  "diff_has_expected");
-		ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two",
-				  "diff_has_actual");
+			ASSERT_EQ(err, -1, "mismatch_rc");
+			ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one",
+					  "diff_has_expected");
+			ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two",
+					  "diff_has_actual");
+		}
 		test__end_subtest();
 	}
 }
-- 
2.43.0