[PATCHv3 bpf-next 0/2] selftests/bpf: Add more uprobe multi tests

[PATCHv3 bpf-next 0/2] selftests/bpf: Add more uprobe multi tests

[Jiri Olsa]

hi,
adding more uprobe multi tests for failed attachments
inside the uprobe register code.

v3 changes:
  - renamed several variables/functions, null check [Andrii]
  - fixed CI issue

thanks,
jirka

---
Jiri Olsa (2):
      selftests/bpf: Add uprobe fail tests for uprobe multi
      selftests/bpf: Add uprobe multi consumers test

 tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c | 331 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c |  39 +++++++++++
 2 files changed, 370 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c

[PATCHv3 bpf-next 1/2] selftests/bpf: Add uprobe fail tests for uprobe multi

[Jiri Olsa]

Adding tests for checking on recovery after failing to
attach uprobe.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 .../bpf/prog_tests/uprobe_multi_test.c        | 118 ++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
index bf6ca8e3eb13..e6255d4df81d 100644
--- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
+++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
@@ -516,6 +516,122 @@ static void test_attach_api_fails(void)
 	uprobe_multi__destroy(skel);
 }
 
+#ifdef __x86_64__
+noinline void uprobe_multi_error_func(void)
+{
+	/*
+	 * If --fcf-protection=branch is enabled the gcc generates endbr as
+	 * first instruction, so marking the exact address of int3 with the
+	 * symbol to be used in the attach_uprobe_fail_trap test below.
+	 */
+	asm volatile (
+		".globl uprobe_multi_error_func_int3;	\n"
+		"uprobe_multi_error_func_int3:		\n"
+		"int3					\n"
+	);
+}
+
+/*
+ * Attaching uprobe on uprobe_multi_error_func results in error
+ * because it already starts with int3 instruction.
+ */
+static void attach_uprobe_fail_trap(struct uprobe_multi *skel)
+{
+	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
+	const char *syms[4] = {
+		"uprobe_multi_func_1",
+		"uprobe_multi_func_2",
+		"uprobe_multi_func_3",
+		"uprobe_multi_error_func_int3",
+	};
+
+	opts.syms = syms;
+	opts.cnt = ARRAY_SIZE(syms);
+
+	skel->links.uprobe = bpf_program__attach_uprobe_multi(skel->progs.uprobe, -1,
+							      "/proc/self/exe", NULL, &opts);
+	if (!ASSERT_ERR_PTR(skel->links.uprobe, "bpf_program__attach_uprobe_multi")) {
+		bpf_link__destroy(skel->links.uprobe);
+		skel->links.uprobe = NULL;
+	}
+}
+#else
+static void attach_uprobe_fail_trap(struct uprobe_multi *skel) { }
+#endif
+
+short sema_1 __used, sema_2 __used;
+
+static void attach_uprobe_fail_refctr(struct uprobe_multi *skel)
+{
+	unsigned long *tmp_offsets = NULL, *tmp_ref_ctr_offsets = NULL;
+	unsigned long offsets[3], ref_ctr_offsets[3];
+	LIBBPF_OPTS(bpf_link_create_opts, opts);
+	const char *path = "/proc/self/exe";
+	const char *syms[3] = {
+		"uprobe_multi_func_1",
+		"uprobe_multi_func_2",
+	};
+	const char *sema[3] = {
+		"sema_1",
+		"sema_2",
+	};
+	int prog_fd, link_fd, err;
+
+	prog_fd = bpf_program__fd(skel->progs.uprobe_extra);
+
+	err = elf_resolve_syms_offsets("/proc/self/exe", 2, (const char **) &syms,
+				       &tmp_offsets, STT_FUNC);
+	if (!ASSERT_OK(err, "elf_resolve_syms_offsets_func"))
+		return;
+
+	err = elf_resolve_syms_offsets("/proc/self/exe", 2, (const char **) &sema,
+				       &tmp_ref_ctr_offsets, STT_OBJECT);
+	if (!ASSERT_OK(err, "elf_resolve_syms_offsets_sema"))
+		goto cleanup;
+
+	/*
+	 * We attach to 3 uprobes on 2 functions, so 2 uprobes share single function,
+	 * but with different ref_ctr_offset which is not allowed and results in fail.
+	 */
+	offsets[0] = tmp_offsets[0]; /* uprobe_multi_func_1 */
+	offsets[1] = tmp_offsets[1]; /* uprobe_multi_func_2 */
+	offsets[2] = tmp_offsets[1]; /* uprobe_multi_func_2 */
+
+	ref_ctr_offsets[0] = tmp_ref_ctr_offsets[0]; /* sema_1 */
+	ref_ctr_offsets[1] = tmp_ref_ctr_offsets[1]; /* sema_2 */
+	ref_ctr_offsets[2] = tmp_ref_ctr_offsets[0]; /* sema_1, error */
+
+	opts.uprobe_multi.path = path;
+	opts.uprobe_multi.offsets = (const unsigned long *) &offsets;
+	opts.uprobe_multi.ref_ctr_offsets = (const unsigned long *) &ref_ctr_offsets;
+	opts.uprobe_multi.cnt = 3;
+
+	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts);
+	if (!ASSERT_ERR(link_fd, "link_fd"))
+		close(link_fd);
+
+cleanup:
+	free(tmp_ref_ctr_offsets);
+	free(tmp_offsets);
+}
+
+static void test_attach_uprobe_fails(void)
+{
+	struct uprobe_multi *skel = NULL;
+
+	skel = uprobe_multi__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "uprobe_multi__open_and_load"))
+		return;
+
+	/* attach fails due to adding uprobe on trap instruction, x86_64 only */
+	attach_uprobe_fail_trap(skel);
+
+	/* attach fail due to wrong ref_ctr_offs on one of the uprobes */
+	attach_uprobe_fail_refctr(skel);
+
+	uprobe_multi__destroy(skel);
+}
+
 static void __test_link_api(struct child *child)
 {
 	int prog_fd, link1_fd = -1, link2_fd = -1, link3_fd = -1, link4_fd = -1;
@@ -703,4 +819,6 @@ void test_uprobe_multi_test(void)
 		test_bench_attach_usdt();
 	if (test__start_subtest("attach_api_fails"))
 		test_attach_api_fails();
+	if (test__start_subtest("attach_uprobe_fails"))
+		test_attach_uprobe_fails();
 }
-- 
2.45.2

[PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

[Jiri Olsa]

Adding test that attaches/detaches multiple consumers on
single uprobe and verifies all were hit as expected.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 .../bpf/prog_tests/uprobe_multi_test.c        | 213 ++++++++++++++++++
 .../bpf/progs/uprobe_multi_consumers.c        |  39 ++++
 2 files changed, 252 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c

diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
index e6255d4df81d..27708110ea20 100644
--- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
+++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
@@ -6,6 +6,7 @@
 #include "uprobe_multi.skel.h"
 #include "uprobe_multi_bench.skel.h"
 #include "uprobe_multi_usdt.skel.h"
+#include "uprobe_multi_consumers.skel.h"
 #include "bpf/libbpf_internal.h"
 #include "testing_helpers.h"
 #include "../sdt.h"
@@ -731,6 +732,216 @@ static void test_link_api(void)
 	__test_link_api(child);
 }
 
+static struct bpf_program *
+get_program(struct uprobe_multi_consumers *skel, int prog)
+{
+	switch (prog) {
+	case 0:
+		return skel->progs.uprobe_0;
+	case 1:
+		return skel->progs.uprobe_1;
+	case 2:
+		return skel->progs.uprobe_2;
+	case 3:
+		return skel->progs.uprobe_3;
+	default:
+		ASSERT_FAIL("get_program");
+		return NULL;
+	}
+}
+
+static struct bpf_link **
+get_link(struct uprobe_multi_consumers *skel, int link)
+{
+	switch (link) {
+	case 0:
+		return &skel->links.uprobe_0;
+	case 1:
+		return &skel->links.uprobe_1;
+	case 2:
+		return &skel->links.uprobe_2;
+	case 3:
+		return &skel->links.uprobe_3;
+	default:
+		ASSERT_FAIL("get_link");
+		return NULL;
+	}
+}
+
+static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx)
+{
+	struct bpf_program *prog = get_program(skel, idx);
+	struct bpf_link **link = get_link(skel, idx);
+	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
+
+	if (!prog || !link)
+		return -1;
+
+	/*
+	 * bit/prog: 0,1 uprobe entry
+	 * bit/prog: 2,3 uprobe return
+	 */
+	opts.retprobe = idx == 2 || idx == 3;
+
+	*link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe",
+						"uprobe_consumer_test",
+						&opts);
+	if (!ASSERT_OK_PTR(*link, "bpf_program__attach_uprobe_multi"))
+		return -1;
+	return 0;
+}
+
+static void uprobe_detach(struct uprobe_multi_consumers *skel, int idx)
+{
+	struct bpf_link **link = get_link(skel, idx);
+
+	bpf_link__destroy(*link);
+	*link = NULL;
+}
+
+static bool test_bit(int bit, unsigned long val)
+{
+	return val & (1 << bit);
+}
+
+noinline int
+uprobe_consumer_test(struct uprobe_multi_consumers *skel,
+		     unsigned long before, unsigned long after)
+{
+	int idx;
+
+	/* detach uprobe for each unset programs in 'before' state ... */
+	for (idx = 0; idx < 4; idx++) {
+		if (test_bit(idx, before) && !test_bit(idx, after))
+			uprobe_detach(skel, idx);
+	}
+
+	/* ... and attach all new programs in 'after' state */
+	for (idx = 0; idx < 4; idx++) {
+		if (!test_bit(idx, before) && test_bit(idx, after)) {
+			if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_after"))
+				return -1;
+		}
+	}
+	return 0;
+}
+
+static void consumer_test(struct uprobe_multi_consumers *skel,
+			  unsigned long before, unsigned long after)
+{
+	int err, idx;
+
+	printf("consumer_test before %lu after %lu\n", before, after);
+
+	/* 'before' is each, we attach uprobe for every set idx */
+	for (idx = 0; idx < 4; idx++) {
+		if (test_bit(idx, before)) {
+			if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
+				goto cleanup;
+		}
+	}
+
+	err = uprobe_consumer_test(skel, before, after);
+	if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
+		goto cleanup;
+
+	for (idx = 0; idx < 4; idx++) {
+		const char *fmt = "BUG";
+		__u64 val = 0;
+
+		if (idx < 2) {
+			/*
+			 * uprobe entry
+			 *   +1 if define in 'before'
+			 */
+			if (test_bit(idx, before))
+				val++;
+			fmt = "prog 0/1: uprobe";
+		} else {
+			/*
+			 * uprobe return is tricky ;-)
+			 *
+			 * to trigger uretprobe consumer, the uretprobe needs to be installed,
+			 * which means one of the 'return' uprobes was alive when probe was hit:
+			 *
+			 *   idxs: 2/3 uprobe return in 'installed' mask
+			 *
+			 * in addition if 'after' state removes everything that was installed in
+			 * 'before' state, then uprobe kernel object goes away and return uprobe
+			 * is not installed and we won't hit it even if it's in 'after' state.
+			 */
+			unsigned long had_uretprobes  = before & 0b1100; // is uretprobe installed
+			unsigned long probe_preserved = before & after;  // did uprobe go away
+
+			if (had_uretprobes && probe_preserved && test_bit(idx, after))
+				val++;
+			fmt = "idx 2/3: uretprobe";
+		}
+
+		ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
+		skel->bss->uprobe_result[idx] = 0;
+	}
+
+cleanup:
+	for (idx = 0; idx < 4; idx++)
+		uprobe_detach(skel, idx);
+}
+
+static void test_consumers(void)
+{
+	struct uprobe_multi_consumers *skel;
+	int before, after;
+
+	skel = uprobe_multi_consumers__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "uprobe_multi_consumers__open_and_load"))
+		return;
+
+	/*
+	 * The idea of this test is to try all possible combinations of
+	 * uprobes consumers attached on single function.
+	 *
+	 *  - 2 uprobe entry consumer
+	 *  - 2 uprobe exit consumers
+	 *
+	 * The test uses 4 uprobes attached on single function, but that
+	 * translates into single uprobe with 4 consumers in kernel.
+	 *
+	 * The before/after values present the state of attached consumers
+	 * before and after the probed function:
+	 *
+	 *  bit/prog 0,1 : uprobe entry
+	 *  bit/prog 2,3 : uprobe return
+	 *
+	 * For example for:
+	 *
+	 *   before = 0b0101
+	 *   after  = 0b0110
+	 *
+	 * it means that before we call 'uprobe_consumer_test' we attach
+	 * uprobes defined in 'before' value:
+	 *
+	 *   - bit/prog 0: uprobe entry
+	 *   - bit/prog 2: uprobe return
+	 *
+	 * uprobe_consumer_test is called and inside it we attach and detach
+	 * uprobes based on 'after' value:
+	 *
+	 *   - bit/prog 0: stays untouched
+	 *   - bit/prog 2: uprobe return is detached
+	 *
+	 * uprobe_consumer_test returns and we check counters values increased
+	 * by bpf programs on each uprobe to match the expected count based on
+	 * before/after bits.
+	 */
+
+	for (before = 0; before < 16; before++) {
+		for (after = 0; after < 16; after++)
+			consumer_test(skel, before, after);
+	}
+
+	uprobe_multi_consumers__destroy(skel);
+}
+
 static void test_bench_attach_uprobe(void)
 {
 	long attach_start_ns = 0, attach_end_ns = 0;
@@ -821,4 +1032,6 @@ void test_uprobe_multi_test(void)
 		test_attach_api_fails();
 	if (test__start_subtest("attach_uprobe_fails"))
 		test_attach_uprobe_fails();
+	if (test__start_subtest("consumers"))
+		test_consumers();
 }
diff --git a/tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c b/tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
new file mode 100644
index 000000000000..7e0fdcbbd242
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <stdbool.h>
+#include "bpf_kfuncs.h"
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+__u64 uprobe_result[4];
+
+SEC("uprobe.multi")
+int uprobe_0(struct pt_regs *ctx)
+{
+	uprobe_result[0]++;
+	return 0;
+}
+
+SEC("uprobe.multi")
+int uprobe_1(struct pt_regs *ctx)
+{
+	uprobe_result[1]++;
+	return 0;
+}
+
+SEC("uprobe.multi")
+int uprobe_2(struct pt_regs *ctx)
+{
+	uprobe_result[2]++;
+	return 0;
+}
+
+SEC("uprobe.multi")
+int uprobe_3(struct pt_regs *ctx)
+{
+	uprobe_result[3]++;
+	return 0;
+}
-- 
2.45.2

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

[Andrii Nakryiko]

On Mon, Jul 22, 2024 at 1:28 PM Jiri Olsa <jolsa@kernel.org> wrote:
>
> Adding test that attaches/detaches multiple consumers on
> single uprobe and verifies all were hit as expected.
>
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---
>  .../bpf/prog_tests/uprobe_multi_test.c        | 213 ++++++++++++++++++
>  .../bpf/progs/uprobe_multi_consumers.c        |  39 ++++
>  2 files changed, 252 insertions(+)
>  create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
>

[...]

> +               } else {
> +                       /*
> +                        * uprobe return is tricky ;-)
> +                        *
> +                        * to trigger uretprobe consumer, the uretprobe needs to be installed,
> +                        * which means one of the 'return' uprobes was alive when probe was hit:
> +                        *
> +                        *   idxs: 2/3 uprobe return in 'installed' mask
> +                        *
> +                        * in addition if 'after' state removes everything that was installed in
> +                        * 'before' state, then uprobe kernel object goes away and return uprobe
> +                        * is not installed and we won't hit it even if it's in 'after' state.
> +                        */
> +                       unsigned long had_uretprobes  = before & 0b1100; // is uretprobe installed
> +                       unsigned long probe_preserved = before & after;  // did uprobe go away

fixed C++-style comments, pushed to bpf-next


> +
> +                       if (had_uretprobes && probe_preserved && test_bit(idx, after))
> +                               val++;
> +                       fmt = "idx 2/3: uretprobe";
> +               }
> +
> +               ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
> +               skel->bss->uprobe_result[idx] = 0;
> +       }
> +
> +cleanup:
> +       for (idx = 0; idx < 4; idx++)
> +               uprobe_detach(skel, idx);
> +}

[...]

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

[Ihor Solodrai]

On Monday, July 22nd, 2024 at 1:27 PM, Jiri Olsa <jolsa@kernel.org> wrote:

> 
> 
> Adding test that attaches/detaches multiple consumers on
> single uprobe and verifies all were hit as expected.
> 
> Signed-off-by: Jiri Olsa jolsa@kernel.org
> 
> ---
> .../bpf/prog_tests/uprobe_multi_test.c | 213 ++++++++++++++++++
> .../bpf/progs/uprobe_multi_consumers.c | 39 ++++
> 2 files changed, 252 insertions(+)
> create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
> 
> diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
> index e6255d4df81d..27708110ea20 100644
> --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
> +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
> @@ -6,6 +6,7 @@
> #include "uprobe_multi.skel.h"
> #include "uprobe_multi_bench.skel.h"
> #include "uprobe_multi_usdt.skel.h"
> +#include "uprobe_multi_consumers.skel.h"
> #include "bpf/libbpf_internal.h"
> #include "testing_helpers.h"
> #include "../sdt.h"
> @@ -731,6 +732,216 @@ static void test_link_api(void)
> __test_link_api(child);
> }
> 

[...]

> +
> +static void consumer_test(struct uprobe_multi_consumers skel,
> + unsigned long before, unsigned long after)
> +{
> + int err, idx;
> +
> + printf("consumer_test before %lu after %lu\n", before, after);
> +
> + / 'before' is each, we attach uprobe for every set idx */
> + for (idx = 0; idx < 4; idx++) {
> + if (test_bit(idx, before)) {
> + if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
> + goto cleanup;
> + }
> + }
> +
> + err = uprobe_consumer_test(skel, before, after);
> + if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
> + goto cleanup;
> +
> + for (idx = 0; idx < 4; idx++) {
> + const char fmt = "BUG";
> + __u64 val = 0;
> +
> + if (idx < 2) {
> + /
> + * uprobe entry
> + * +1 if define in 'before'
> + /
> + if (test_bit(idx, before))
> + val++;
> + fmt = "prog 0/1: uprobe";
> + } else {
> + /
> + * uprobe return is tricky ;-)
> + *
> + * to trigger uretprobe consumer, the uretprobe needs to be installed,
> + * which means one of the 'return' uprobes was alive when probe was hit:
> + *
> + * idxs: 2/3 uprobe return in 'installed' mask
> + *
> + * in addition if 'after' state removes everything that was installed in
> + * 'before' state, then uprobe kernel object goes away and return uprobe
> + * is not installed and we won't hit it even if it's in 'after' state.
> + */
> + unsigned long had_uretprobes = before & 0b1100; // is uretprobe installed
> + unsigned long probe_preserved = before & after; // did uprobe go away
> +
> + if (had_uretprobes && probe_preserved && test_bit(idx, after))
> + val++;
> + fmt = "idx 2/3: uretprobe";
> + }

Jiri, Andrii,

This test case started failing since upstream got merged into bpf-next,
starting from commit https://git.kernel.org/bpf/bpf-next/c/440b65232829

A snippet from the test log:

    consumer_test before 4 after 8
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    consumer_test:PASS:uprobe_attach_before 0 nsec
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
    consumer_test:PASS:uprobe_consumer_test 0 nsec
    consumer_test:PASS:prog 0/1: uprobe 0 nsec
    consumer_test:PASS:prog 0/1: uprobe 0 nsec
    consumer_test:PASS:idx 2/3: uretprobe 0 nsec
    consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0
    consumer_test before 4 after 9
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    consumer_test:PASS:uprobe_attach_before 0 nsec
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
    consumer_test:PASS:uprobe_consumer_test 0 nsec
    consumer_test:PASS:prog 0/1: uprobe 0 nsec
    consumer_test:PASS:prog 0/1: uprobe 0 nsec
    consumer_test:PASS:idx 2/3: uretprobe 0 nsec
    consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0
    consumer_test before 4 after 10
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    consumer_test:PASS:uprobe_attach_before 0 nsec
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
    uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
    uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
    consumer_test:PASS:uprobe_consumer_test 0 nsec
    consumer_test:PASS:prog 0/1: uprobe 0 nsec
    consumer_test:PASS:prog 0/1: uprobe 0 nsec
    consumer_test:PASS:idx 2/3: uretprobe 0 nsec
    consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0


I couldn't figure out the reason as I have very shallow understanding
of what's happening in the test.

Jiri, could you please look into it?

I excluded this test from BPF CI for now.

Thank you!

> +
> + ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
> 
> + skel->bss->uprobe_result[idx] = 0;
> 
> + }
> +
> +cleanup:
> + for (idx = 0; idx < 4; idx++)
> + uprobe_detach(skel, idx);
> +}

[...]

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

[Andrii Nakryiko]

On Tue, Sep 24, 2024 at 2:33 AM Ihor Solodrai <ihor.solodrai@pm.me> wrote:
>
> On Monday, July 22nd, 2024 at 1:27 PM, Jiri Olsa <jolsa@kernel.org> wrote:
>
> >
> >
> > Adding test that attaches/detaches multiple consumers on
> > single uprobe and verifies all were hit as expected.
> >
> > Signed-off-by: Jiri Olsa jolsa@kernel.org
> >
> > ---
> > .../bpf/prog_tests/uprobe_multi_test.c | 213 ++++++++++++++++++
> > .../bpf/progs/uprobe_multi_consumers.c | 39 ++++
> > 2 files changed, 252 insertions(+)
> > create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
> >
> > diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
> > index e6255d4df81d..27708110ea20 100644
> > --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
> > +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c
> > @@ -6,6 +6,7 @@
> > #include "uprobe_multi.skel.h"
> > #include "uprobe_multi_bench.skel.h"
> > #include "uprobe_multi_usdt.skel.h"
> > +#include "uprobe_multi_consumers.skel.h"
> > #include "bpf/libbpf_internal.h"
> > #include "testing_helpers.h"
> > #include "../sdt.h"
> > @@ -731,6 +732,216 @@ static void test_link_api(void)
> > __test_link_api(child);
> > }
> >
>
> [...]
>
> > +
> > +static void consumer_test(struct uprobe_multi_consumers skel,
> > + unsigned long before, unsigned long after)
> > +{
> > + int err, idx;
> > +
> > + printf("consumer_test before %lu after %lu\n", before, after);
> > +
> > + / 'before' is each, we attach uprobe for every set idx */
> > + for (idx = 0; idx < 4; idx++) {
> > + if (test_bit(idx, before)) {
> > + if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
> > + goto cleanup;
> > + }
> > + }
> > +
> > + err = uprobe_consumer_test(skel, before, after);
> > + if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
> > + goto cleanup;
> > +
> > + for (idx = 0; idx < 4; idx++) {
> > + const char fmt = "BUG";
> > + __u64 val = 0;
> > +
> > + if (idx < 2) {
> > + /
> > + * uprobe entry
> > + * +1 if define in 'before'
> > + /
> > + if (test_bit(idx, before))
> > + val++;
> > + fmt = "prog 0/1: uprobe";
> > + } else {
> > + /
> > + * uprobe return is tricky ;-)
> > + *
> > + * to trigger uretprobe consumer, the uretprobe needs to be installed,
> > + * which means one of the 'return' uprobes was alive when probe was hit:
> > + *
> > + * idxs: 2/3 uprobe return in 'installed' mask
> > + *
> > + * in addition if 'after' state removes everything that was installed in
> > + * 'before' state, then uprobe kernel object goes away and return uprobe
> > + * is not installed and we won't hit it even if it's in 'after' state.
> > + */
> > + unsigned long had_uretprobes = before & 0b1100; // is uretprobe installed
> > + unsigned long probe_preserved = before & after; // did uprobe go away
> > +
> > + if (had_uretprobes && probe_preserved && test_bit(idx, after))
> > + val++;
> > + fmt = "idx 2/3: uretprobe";
> > + }
>
> Jiri, Andrii,
>
> This test case started failing since upstream got merged into bpf-next,
> starting from commit https://git.kernel.org/bpf/bpf-next/c/440b65232829
>
> A snippet from the test log:
>
>     consumer_test before 4 after 8
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     consumer_test:PASS:uprobe_attach_before 0 nsec
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
>     consumer_test:PASS:uprobe_consumer_test 0 nsec
>     consumer_test:PASS:prog 0/1: uprobe 0 nsec
>     consumer_test:PASS:prog 0/1: uprobe 0 nsec
>     consumer_test:PASS:idx 2/3: uretprobe 0 nsec
>     consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0
>     consumer_test before 4 after 9
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     consumer_test:PASS:uprobe_attach_before 0 nsec
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
>     consumer_test:PASS:uprobe_consumer_test 0 nsec
>     consumer_test:PASS:prog 0/1: uprobe 0 nsec
>     consumer_test:PASS:prog 0/1: uprobe 0 nsec
>     consumer_test:PASS:idx 2/3: uretprobe 0 nsec
>     consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0
>     consumer_test before 4 after 10
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     consumer_test:PASS:uprobe_attach_before 0 nsec
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
>     uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec
>     uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec
>     consumer_test:PASS:uprobe_consumer_test 0 nsec
>     consumer_test:PASS:prog 0/1: uprobe 0 nsec
>     consumer_test:PASS:prog 0/1: uprobe 0 nsec
>     consumer_test:PASS:idx 2/3: uretprobe 0 nsec
>     consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0
>
>
> I couldn't figure out the reason as I have very shallow understanding
> of what's happening in the test.
>
> Jiri, could you please look into it?
>
> I excluded this test from BPF CI for now.

Thanks for the mitigation! I think this is due to my recent RCU and
refcounting changes to uprobes/uretprobes, which went through
tip/perf/core initially. And now that tip and bpf-next trees
converged, this condition:

  > unsigned long probe_preserved = before & after; // did uprobe go away

is no longer correct, and uretprobe can be activated if there was
*any* uretprobe installed before.

So the test needs adjustment, but I don't think anything really broke.
I don't remember exactly (and given the conferencing schedule and
quite bad internet can't test quickly), but I think the condition
should now be:

unsigned long probe_preserved = after & 0x1100;

(though we might want to also rename the variable to be a bit more
meaningful now).

Anyways, I don't think this is critical and we can address this later.
But if anyone is willing to send a fix, I'd appreciate it, of course!

>
> Thank you!
>
> > +
> > + ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
> >
> > + skel->bss->uprobe_result[idx] = 0;
> >
> > + }
> > +
> > +cleanup:
> > + for (idx = 0; idx < 4; idx++)
> > + uprobe_detach(skel, idx);
> > +}
>
> [...]
>

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

[Jiri Olsa]

On Tue, Sep 24, 2024 at 07:44:50AM +0200, Andrii Nakryiko wrote:

SNIP

> > > + /
> > > + * uprobe return is tricky ;-)
> > > + *
> > > + * to trigger uretprobe consumer, the uretprobe needs to be installed,
> > > + * which means one of the 'return' uprobes was alive when probe was hit:
> > > + *
> > > + * idxs: 2/3 uprobe return in 'installed' mask
> > > + *
> > > + * in addition if 'after' state removes everything that was installed in
> > > + * 'before' state, then uprobe kernel object goes away and return uprobe
> > > + * is not installed and we won't hit it even if it's in 'after' state.
> > > + */
> > > + unsigned long had_uretprobes = before & 0b1100; // is uretprobe installed
> > > + unsigned long probe_preserved = before & after; // did uprobe go away
> > > +
> > > + if (had_uretprobes && probe_preserved && test_bit(idx, after))
> > > + val++;
> > > + fmt = "idx 2/3: uretprobe";
> > > + }
> >
> > Jiri, Andrii,
> >
> > This test case started failing since upstream got merged into bpf-next,
> > starting from commit https://git.kernel.org/bpf/bpf-next/c/440b65232829

thanks for the report

SNIP

> 
> Thanks for the mitigation! I think this is due to my recent RCU and
> refcounting changes to uprobes/uretprobes, which went through
> tip/perf/core initially. And now that tip and bpf-next trees
> converged, this condition:
> 
>   > unsigned long probe_preserved = before & after; // did uprobe go away
> 
> is no longer correct, and uretprobe can be activated if there was
> *any* uretprobe installed before.
> 
> So the test needs adjustment, but I don't think anything really broke.
> I don't remember exactly (and given the conferencing schedule and
> quite bad internet can't test quickly), but I think the condition
> should now be:
> 
> unsigned long probe_preserved = after & 0x1100;
> 
> (though we might want to also rename the variable to be a bit more
> meaningful now).
> 
> Anyways, I don't think this is critical and we can address this later.
> But if anyone is willing to send a fix, I'd appreciate it, of course!

I think we can remove that check completely.. I sent the patch, let's discuss there ;-)

thanks,
jirka

Re: [PATCHv3 bpf-next 0/2] selftests/bpf: Add more uprobe multi tests

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 536 bytes --]

Dear patch submitter,

CI has tested the following submission:
Status:     SUCCESS
Name:       [PATCHv3,bpf-next,0/2] selftests/bpf: Add more uprobe multi tests
Patchwork:  https://patchwork.kernel.org/project/netdevbpf/list/?series=873053&state=*
Matrix:     https://github.com/kernel-patches/bpf/actions/runs/10048022329

No further action is necessary on your part.


Please note: this email is coming from an unmonitored mailbox. If you have
questions or feedback, please reach out to the Meta Kernel CI team at
kernel-ci@meta.com.

Re: [PATCHv3 bpf-next 0/2] selftests/bpf: Add more uprobe multi tests

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 3148 bytes --]

Dear patch submitter,

CI has tested the following submission:
Status:     FAILURE
Name:       [PATCHv3,bpf-next,0/2] selftests/bpf: Add more uprobe multi tests
Patchwork:  https://patchwork.kernel.org/project/netdevbpf/list/?series=873053&state=*
Matrix:     https://github.com/kernel-patches/bpf/actions/runs/10050702681

Failed jobs:
test_progs_no_alu32-aarch64-gcc: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779250571
test_progs_no_alu32-s390x-gcc: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779381610
test_progs_no_alu32-x86_64-gcc: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779406344
test_progs_no_alu32-x86_64-llvm-17: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779390782
test_progs_no_alu32-x86_64-llvm-18: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779387928

First test_progs failure (test_progs_no_alu32-aarch64-gcc):
#134 libbpf_get_fd_by_id_opts
libbpf: prog 'check_access': BPF program load failed: Invalid argument
libbpf: prog 'check_access': -- BEGIN PROG LOAD LOG --
0: R1=ctx() R10=fp0
; int BPF_PROG(check_access, struct bpf_map *map, fmode_t fmode) @ test_libbpf_get_fd_by_id_opts.c:27
0: (b7) r0 = 0                        ; R0_w=0
1: (79) r2 = *(u64 *)(r1 +0)
func 'bpf_lsm_bpf_map' arg0 has btf_id 2072 type STRUCT 'bpf_map'
2: R1=ctx() R2_w=trusted_ptr_bpf_map()
; if (map != (struct bpf_map *)&data_input) @ test_libbpf_get_fd_by_id_opts.c:29
2: (18) r3 = 0xffff0000c341f800       ; R3_w=map_ptr(map=data_input,ks=4,vs=4)
4: (5d) if r2 != r3 goto pc+4         ; R2_w=trusted_ptr_bpf_map() R3_w=map_ptr(map=data_input,ks=4,vs=4)
; int BPF_PROG(check_access, struct bpf_map *map, fmode_t fmode) @ test_libbpf_get_fd_by_id_opts.c:27
5: (79) r0 = *(u64 *)(r1 +8)          ; R0_w=scalar() R1=ctx()
; if (fmode & FMODE_WRITE) @ test_libbpf_get_fd_by_id_opts.c:32
6: (67) r0 <<= 62                     ; R0_w=scalar(smax=0x4000000000000000,umax=0xc000000000000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xc000000000000000))
7: (c7) r0 s>>= 63                    ; R0_w=scalar(smin=smin32=-1,smax=smax32=0)
;  @ test_libbpf_get_fd_by_id_opts.c:0
8: (57) r0 &= -13                     ; R0_w=scalar(smax=0x7ffffffffffffff3,umax=0xfffffffffffffff3,smax32=0x7ffffff3,umax32=0xfffffff3,var_off=(0x0; 0xfffffffffffffff3))
; int BPF_PROG(check_access, struct bpf_map *map, fmode_t fmode) @ test_libbpf_get_fd_by_id_opts.c:27
9: (95) exit
At program exit the register R0 has smax=9223372036854775795 should have been in [-4095, 0]
processed 9 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0
-- END PROG LOAD LOG --
libbpf: prog 'check_access': failed to load: -22
libbpf: failed to load object 'test_libbpf_get_fd_by_id_opts'
libbpf: failed to load BPF skeleton 'test_libbpf_get_fd_by_id_opts': -22
test_libbpf_get_fd_by_id_opts:FAIL:test_libbpf_get_fd_by_id_opts__open_and_load unexpected error: -22


Please note: this email is coming from an unmonitored mailbox. If you have
questions or feedback, please reach out to the Meta Kernel CI team at
kernel-ci@meta.com.

Re: [PATCHv3 bpf-next 0/2] selftests/bpf: Add more uprobe multi tests

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 536 bytes --]

Dear patch submitter,

CI has tested the following submission:
Status:     SUCCESS
Name:       [PATCHv3,bpf-next,0/2] selftests/bpf: Add more uprobe multi tests
Patchwork:  https://patchwork.kernel.org/project/netdevbpf/list/?series=873053&state=*
Matrix:     https://github.com/kernel-patches/bpf/actions/runs/10051533401

No further action is necessary on your part.


Please note: this email is coming from an unmonitored mailbox. If you have
questions or feedback, please reach out to the Meta Kernel CI team at
kernel-ci@meta.com.

Re: [PATCHv3 bpf-next 0/2] selftests/bpf: Add more uprobe multi tests

[patchwork-bot+netdevbpf]

Hello:

This series was applied to bpf/bpf-next.git (master)
by Andrii Nakryiko <andrii@kernel.org>:

On Mon, 22 Jul 2024 22:27:56 +0200 you wrote:
> hi,
> adding more uprobe multi tests for failed attachments
> inside the uprobe register code.
> 
> v3 changes:
>   - renamed several variables/functions, null check [Andrii]
>   - fixed CI issue
> 
> [...]

Here is the summary with links:
  - [PATCHv3,bpf-next,1/2] selftests/bpf: Add uprobe fail tests for uprobe multi
    https://git.kernel.org/bpf/bpf-next/c/f5ee7d559a7c
  - [PATCHv3,bpf-next,2/2] selftests/bpf: Add uprobe multi consumers test
    https://git.kernel.org/bpf/bpf-next/c/c5ec71c325e7

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html