[PATCHv3 bpf-next 0/3] bpf: tracing_multi link info support

[PATCHv3 bpf-next 0/3] bpf: tracing_multi link info support

[Jiri Olsa]

hi,
adding link info support for tracing_multi link.

Adding the kernel support with related selftests and bpftool support
to display the tracing_multi link info, the output looks like:

  # bpftool link
  ...
  61: tracing_multi  prog 167
          attach_type trace_fentry_multi  btf_obj_id 1  count 3
          btf_id           addr             cookie           func [module]
          92598            ffffffff825017c4 10               bpf_fentry_test1
          92600            ffffffff82503814 30               bpf_fentry_test2
          92601            ffffffff82503824 20               bpf_fentry_test3
          pids test_progs(1540)

thanks,
jirka


v1: https://lore.kernel.org/bpf/20260621204524.61067-1-jolsa@kernel.org/
v2: https://lore.kernel.org/bpf/20260623142417.275892-1-jolsa@kernel.org/

v3 changes:
- added acks [Andrii, Leon]
- display cookies as decimal numbers [Andrii]
- renamed obj_id to btf_obj_id [Andrii]
- several fixes in bpftool change [Andrii]

---
Jiri Olsa (3):
      bpf: Add tracing_multi link info support
      selftests/bpf: Add tracing_multi link info tests
      bpftool: Add tracing_multi link info output

 include/uapi/linux/bpf.h                                |   9 +++
 kernel/trace/bpf_trace.c                                |  55 +++++++++++++++++
 tools/bpf/bpftool/link.c                                | 133 ++++++++++++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h                          |   9 +++
 tools/testing/selftests/bpf/prog_tests/fill_link_info.c | 242 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/testing/selftests/bpf/progs/test_fill_link_info.c |   6 ++
 6 files changed, 454 insertions(+)

[PATCHv3 bpf-next 1/3] bpf: Add tracing_multi link info support

[Jiri Olsa]

Adding BPF_OBJ_GET_INFO_BY_FD support for tracing_multi links.

We expose following tracing_multi link data:
- attach_type of the program
- number of ids
- array of BTF ids
- array of its related kernel addresses
- array of cookies

The change follows the kprobe_multi and uprobe_multi link-info convention
of optional output arrays with an in/out count,

On top of standard tracing link data we also expose addresses, because they
are useful info for user (especially when the attachment was done via pattern).
This data is hidden when kallsyms does not allow exposing kernel pointer values.

Assisted-by: Codex:GPT-5
Acked-by: Leon Hwang <leon.hwang@linux.dev>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 include/uapi/linux/bpf.h       |  9 ++++++
 kernel/trace/bpf_trace.c       | 55 ++++++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h |  9 ++++++
 3 files changed, 73 insertions(+)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 89b36de5fdbb..4180262814e7 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -6839,6 +6839,15 @@ struct bpf_link_info {
 			__u32 flags;
 			__u32 pid;
 		} uprobe_multi;
+		struct {
+			__u32 attach_type;
+			__u32 count; /* in/out: tracing_multi target count */
+			__u32 btf_obj_id;
+			__u32 :32;
+			__aligned_u64 ids;
+			__aligned_u64 addrs;
+			__aligned_u64 cookies;
+		} tracing_multi;
 		struct {
 			__u32 type; /* enum bpf_perf_event_type */
 			__u32 :32;
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 82f8feea6931..0711810f4562 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -3699,6 +3699,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link)
 	kvfree(tr_link);
 }
 
+static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link,
+						 struct bpf_link_info *info)
+{
+	u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies);
+	u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs);
+	u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids);
+	struct bpf_tracing_multi_link *tr_link;
+	u32 ucount = info->tracing_multi.count;
+	bool has_cookies, show_addrs;
+	int err = 0;
+
+	if ((uids || ucookies || uaddrs) && !ucount)
+		return -EINVAL;
+
+	tr_link = container_of(link, struct bpf_tracing_multi_link, link);
+
+	info->tracing_multi.attach_type = tr_link->link.attach_type;
+	info->tracing_multi.count = tr_link->nodes_cnt;
+	info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf);
+
+	if (!uids && !ucookies && !uaddrs)
+		return 0;
+
+	if (ucount < tr_link->nodes_cnt)
+		err = -ENOSPC;
+	else
+		ucount = tr_link->nodes_cnt;
+
+	has_cookies = !!tr_link->cookies;
+	show_addrs = kallsyms_show_value(current_cred());
+
+	for (int i = 0; i < ucount; i++) {
+		struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i];
+		u64 addr, cookie;
+		u32 id;
+
+		bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id);
+
+		addr = show_addrs ? mnode->trampoline->ip : 0;
+		cookie = has_cookies ? tr_link->cookies[i] : 0;
+
+		if (uids && put_user(id, uids + i))
+			return -EFAULT;
+		if (uaddrs && put_user(addr, uaddrs + i))
+			return -EFAULT;
+		if (ucookies && put_user(cookie, ucookies + i))
+			return -EFAULT;
+
+		cond_resched();
+	}
+
+	return err;
+}
+
 #ifdef CONFIG_PROC_FS
 static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
 					  struct seq_file *seq)
@@ -3729,6 +3783,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
 static const struct bpf_link_ops bpf_tracing_multi_link_lops = {
 	.release = bpf_tracing_multi_link_release,
 	.dealloc_deferred = bpf_tracing_multi_link_dealloc,
+	.fill_link_info = bpf_tracing_multi_link_fill_link_info,
 #ifdef CONFIG_PROC_FS
 	.show_fdinfo = bpf_tracing_multi_show_fdinfo,
 #endif
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 89b36de5fdbb..4180262814e7 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -6839,6 +6839,15 @@ struct bpf_link_info {
 			__u32 flags;
 			__u32 pid;
 		} uprobe_multi;
+		struct {
+			__u32 attach_type;
+			__u32 count; /* in/out: tracing_multi target count */
+			__u32 btf_obj_id;
+			__u32 :32;
+			__aligned_u64 ids;
+			__aligned_u64 addrs;
+			__aligned_u64 cookies;
+		} tracing_multi;
 		struct {
 			__u32 type; /* enum bpf_perf_event_type */
 			__u32 :32;
-- 
2.54.0

[PATCHv3 bpf-next 2/3] selftests/bpf: Add tracing_multi link info tests

[Jiri Olsa]

Adding tracing_multi link info tests that follow the kprobe_multi
and uprobe_multi tests logic.

Assisted-by: Codex:GPT-5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 .../selftests/bpf/prog_tests/fill_link_info.c | 242 ++++++++++++++++++
 .../selftests/bpf/progs/test_fill_link_info.c |   6 +
 2 files changed, 248 insertions(+)

diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
index f589eefbf9fb..0918321c8e63 100644
--- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
+++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
@@ -4,6 +4,7 @@
 #include <string.h>
 #include <linux/bpf.h>
 #include <linux/limits.h>
+#include <bpf/btf.h>
 #include <test_progs.h>
 #include "trace_helpers.h"
 #include "test_fill_link_info.skel.h"
@@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 };
 #define KPROBE_FUNC "bpf_fentry_test1"
 static __u64 kprobe_addr;
 
+static const char * const tmulti_syms[] = {
+	"bpf_fentry_test2",
+	"bpf_fentry_test1",
+	"bpf_fentry_test3",
+};
+
+static __u64 tmulti_cookies[] = { 30, 10, 20 };
+#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms)
+
+struct tmulti_target {
+	const char *name;
+	__u64 addr;
+	__u64 cookie;
+	__u32 id;
+};
+
 #define UPROBE_FILE "/proc/self/exe"
 static ssize_t uprobe_offset;
 /* uprobe attach point */
@@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel,
 	bpf_link__destroy(link);
 }
 
+static int tmulti_target_cmp(const void *a, const void *b)
+{
+	const struct tmulti_target *ta = a;
+	const struct tmulti_target *tb = b;
+
+	return (ta->id > tb->id) - (ta->id < tb->id);
+}
+
+static int setup_tmulti_targets(const struct bpf_program *prog,
+				struct tmulti_target *targets,
+				__u32 *btf_obj_id)
+{
+	struct bpf_prog_info prog_info;
+	__u32 len = sizeof(prog_info);
+	struct btf *btf;
+	int err, i;
+	__s32 id;
+
+	btf = btf__load_vmlinux_btf();
+	if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf"))
+		return -1;
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC);
+		if (!ASSERT_GT(id, 0, "btf__find_by_name_kind"))
+			goto error;
+
+		targets[i].name = tmulti_syms[i];
+		targets[i].addr = ksym_get_addr(tmulti_syms[i]);
+		targets[i].cookie = tmulti_cookies[i];
+		targets[i].id = id;
+	}
+
+	memset(&prog_info, 0, len);
+	err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len);
+	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd"))
+		goto error;
+	if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id"))
+		goto error;
+	*btf_obj_id = prog_info.attach_btf_obj_id;
+
+	/*
+	 * The kernel tracing multi attach sorts ids. We sort as well,
+	 * so we can easily compare ids and cookies later.
+	 */
+	qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp);
+	btf__free(btf);
+	return 0;
+
+error:
+	btf__free(btf);
+	return -1;
+}
+
+static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog,
+					  const struct tmulti_target *targets,
+					  __u32 btf_obj_id, bool has_cookies)
+{
+	enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog);
+	__u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT];
+	__u32 ids[TRACING_MULTI_CNT];
+	struct bpf_link_info info;
+	__u32 len = sizeof(info);
+	int err, i;
+
+	memset(&info, 0, sizeof(info));
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
+		return -1;
+
+	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
+		return -1;
+
+	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");
+	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
+
+	memset(ids, 0, sizeof(ids));
+	memset(cookies, 0, sizeof(cookies));
+	memset(addrs, 0, sizeof(addrs));
+
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.addrs = ptr_to_u64(addrs);
+	info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
+		return -1;
+
+	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
+		return -1;
+
+	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");
+	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
+	ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id");
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids");
+		ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies");
+
+		if (targets[i].addr) {
+			struct ksym *ksym;
+
+			if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs"))
+				return -1;
+			ksym = ksym_search(addrs[i]);
+			if (!ASSERT_OK_PTR(ksym, "ksym_search"))
+				return -1;
+			ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name");
+		} else {
+			ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs");
+		}
+	}
+
+	return 0;
+}
+
+static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets)
+{
+	__u32 ids[TRACING_MULTI_CNT] = {};
+	struct bpf_link_info info;
+	__u32 len = sizeof(info);
+	int err, i;
+
+	/* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count");
+
+	/* Smaller than actual count provided -> ENOSPC */
+	memset(ids, 0, sizeof(ids));
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.count = TRACING_MULTI_CNT - 1;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count");
+	for (i = 0; i < TRACING_MULTI_CNT - 1; i++)
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids");
+	/* check that the last entry is not populated */
+	ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids");
+
+	/* Bigger than actual count provided -> OK */
+	memset(ids, 0, sizeof(ids));
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.count = TRACING_MULTI_CNT + 1;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_OK(err, "tracing_multi.big_count");
+	for (i = 0; i < TRACING_MULTI_CNT; i++)
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids");
+
+	/* Invalid ids pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids");
+
+	/* Invalid cookies pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.cookies = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies");
+
+	/* Invalid addrs pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.addrs = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs");
+}
+
+static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel,
+					      bool has_cookies, bool invalid)
+{
+	LIBBPF_OPTS(bpf_tracing_multi_opts, opts);
+	struct tmulti_target targets[TRACING_MULTI_CNT];
+	__u32 ids[TRACING_MULTI_CNT], btf_obj_id;
+	__u64 cookies[TRACING_MULTI_CNT];
+	struct bpf_link *link;
+	int link_fd, err, i;
+
+#ifndef __x86_64__
+	test__skip();
+	return;
+#endif
+
+	if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id))
+		return;
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		ids[i] = targets[i].id;
+		cookies[i] = targets[i].cookie;
+	}
+
+	opts.ids = ids;
+	opts.cnt = TRACING_MULTI_CNT;
+	if (has_cookies)
+		opts.cookies = cookies;
+
+	link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi"))
+		return;
+
+	link_fd = bpf_link__fd(link);
+	if (invalid) {
+		verify_tracing_multi_invalid_user_buffer(link_fd, targets);
+	} else {
+		err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run,
+						     targets, btf_obj_id, has_cookies);
+		ASSERT_OK(err, "verify_tracing_multi_link_info");
+	}
+
+	bpf_link__destroy(link);
+}
+
 #define SEC(name) __attribute__((section(name), used))
 
 static short uprobe_link_info_sema_1 SEC(".probes");
@@ -640,6 +875,13 @@ void test_fill_link_info(void)
 	if (test__start_subtest("kprobe_multi_invalid_ubuff"))
 		test_kprobe_multi_fill_link_info(skel, true, true, true);
 
+	if (test__start_subtest("tracing_multi_link_info")) {
+		test_tracing_multi_fill_link_info(skel, false, false);
+		test_tracing_multi_fill_link_info(skel, true, false);
+	}
+	if (test__start_subtest("tracing_multi_invalid_ubuff"))
+		test_tracing_multi_fill_link_info(skel, true, true);
+
 	if (test__start_subtest("uprobe_multi_link_info"))
 		test_uprobe_multi_fill_link_info(skel, false, false);
 	if (test__start_subtest("uretprobe_multi_link_info"))
diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c
index 137bd6292163..c85081538e93 100644
--- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c
+++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c
@@ -58,4 +58,10 @@ int BPF_PROG(umulti_run)
 	return 0;
 }
 
+SEC("fentry.multi")
+int BPF_PROG(tmulti_run)
+{
+	return 0;
+}
+
 char _license[] SEC("license") = "GPL";
-- 
2.54.0

[PATCHv3 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Jiri Olsa]

Adding bpftool support to show tracing_multi link details,
the new output looks like:

  # bpftool link
  ...
  61: tracing_multi  prog 167
          attach_type trace_fentry_multi  btf_obj_id 1  count 3
          btf_id           addr             cookie           func [module]
          92598            ffffffff825017c4 10               bpf_fentry_test1
          92600            ffffffff82503814 30               bpf_fentry_test2
          92601            ffffffff82503824 20               bpf_fentry_test3
          pids test_progs(1540)

Assisted-by: Codex:GPT-5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 tools/bpf/bpftool/link.c | 133 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index bdcd717b0348..eb0ba9d71692 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
 	return (__u64 *) u64_to_ptr(val);
 }
 
+static __u32 *u64_to_u32_arr(__u64 val)
+{
+	return (__u32 *)u64_to_ptr(val);
+}
+
+static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
+{
+	struct kernel_sym *sym;
+
+	if (!addr)
+		return NULL;
+
+	sym = kernel_syms_search(&dd, addr);
+	if (!sym && is_ibt_enabled && addr >= 4)
+		sym = kernel_syms_search(&dd, addr - 4);
+
+	return sym;
+}
+
 static void
 show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
@@ -403,6 +422,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
 	jsonw_end_array(json_wtr);
 }
 
+static void
+show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+	bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+	__u64 *addrs, *cookies;
+	__u32 i, *ids;
+
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	show_symbol = !!dd.sym_count;
+
+	show_link_attach_type_json(info->tracing_multi.attach_type, wtr);
+	jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count);
+	jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id);
+	jsonw_name(wtr, "funcs");
+
+	jsonw_start_array(wtr);
+
+	ids = u64_to_u32_arr(info->tracing_multi.ids);
+	addrs = u64_to_arr(info->tracing_multi.addrs);
+	cookies = u64_to_arr(info->tracing_multi.cookies);
+
+	for (i = 0; i < info->tracing_multi.count; i++) {
+		struct kernel_sym *sym;
+		__u64 addr = addrs[i];
+
+		sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+		jsonw_start_object(wtr);
+		jsonw_uint_field(wtr, "id", ids[i]);
+		jsonw_uint_field(wtr, "addr", addr);
+		if (sym) {
+			jsonw_string_field(wtr, "func", sym->name);
+			if (sym->module[0] == '\0') {
+				jsonw_name(wtr, "module");
+				jsonw_null(wtr);
+			} else {
+				jsonw_string_field(wtr, "module", sym->module);
+			}
+		}
+		jsonw_uint_field(wtr, "cookie", cookies[i]);
+		jsonw_end_object(wtr);
+	}
+	jsonw_end_array(wtr);
+}
+
 static void
 show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
@@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_UPROBE_MULTI:
 		show_uprobe_multi_json(info, json_wtr);
 		break;
+	case BPF_LINK_TYPE_TRACING_MULTI:
+		show_tracing_multi_json(info, json_wtr);
+		break;
 	case BPF_LINK_TYPE_PERF_EVENT:
 		switch (info->perf_event.type) {
 		case BPF_PERF_EVENT_EVENT:
@@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
 	}
 }
 
+static void show_tracing_multi_plain(struct bpf_link_info *info)
+{
+	bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+	__u64 *addrs, *cookies;
+	__u32 i, *ids;
+
+	if (!info->tracing_multi.count)
+		return;
+
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	show_symbol = !!dd.sym_count;
+
+	printf("\n\t");
+	show_link_attach_type_plain(info->tracing_multi.attach_type);
+	printf("btf_obj_id %u  ", info->tracing_multi.btf_obj_id);
+	printf("count %u  ", info->tracing_multi.count);
+
+	printf("\n\t%-16s %-16s %-16s %s",
+	       "btf_id", "addr", "cookie", "func [module]");
+
+	ids = u64_to_u32_arr(info->tracing_multi.ids);
+	addrs = u64_to_arr(info->tracing_multi.addrs);
+	cookies = u64_to_arr(info->tracing_multi.cookies);
+
+	for (i = 0; i < info->tracing_multi.count; i++) {
+		__u64 addr = addrs[i];
+		struct kernel_sym *sym;
+
+		sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+		printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]);
+		if (sym) {
+			printf(" %s", sym->name);
+			if (sym->module[0] != '\0')
+				printf(" [%s]", sym->module);
+		}
+	}
+}
+
 static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
 {
 	const char *buf;
@@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_UPROBE_MULTI:
 		show_uprobe_multi_plain(info);
 		break;
+	case BPF_LINK_TYPE_TRACING_MULTI:
+		show_tracing_multi_plain(info);
+		break;
 	case BPF_LINK_TYPE_PERF_EVENT:
 		switch (info->perf_event.type) {
 		case BPF_PERF_EVENT_EVENT:
@@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 static int do_show_link(int fd)
 {
 	__u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL;
+	__u32 *ids = NULL;
 	struct bpf_link_info info;
 	__u32 len = sizeof(info);
 	char path_buf[PATH_MAX];
@@ -1114,6 +1226,26 @@ static int do_show_link(int fd)
 			goto again;
 		}
 	}
+	if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) {
+		count = info.tracing_multi.count;
+		if (count) {
+			ids = calloc(count, sizeof(__u32));
+			addrs = calloc(count, sizeof(__u64));
+			cookies = calloc(count, sizeof(__u64));
+			if (!ids) {
+				p_err("mem alloc failed");
+				close(fd);
+				free(cookies);
+				free(addrs);
+				free(ids);
+				return -ENOMEM;
+			}
+			info.tracing_multi.ids = ptr_to_u64(ids);
+			info.tracing_multi.addrs = ptr_to_u64(addrs);
+			info.tracing_multi.cookies = ptr_to_u64(cookies);
+			goto again;
+		}
+	}
 	if (info.type == BPF_LINK_TYPE_PERF_EVENT) {
 		switch (info.perf_event.type) {
 		case BPF_PERF_EVENT_TRACEPOINT:
@@ -1153,6 +1285,7 @@ static int do_show_link(int fd)
 	free(cookies);
 	free(offsets);
 	free(addrs);
+	free(ids);
 	close(fd);
 	return 0;
 }
-- 
2.54.0

Re: [PATCHv3 bpf-next 2/3] selftests/bpf: Add tracing_multi link info tests

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Medium] The kernel implementation of `.fill_link_info` for `BPF_LINK_TYPE_TRACING_MULTI` is completely missing, which would cause the new tests to unconditionally fail at runtime.
- [Low] The patch introduces a compilation failure because it attempts to access a non-existent `tracing_multi` member in `struct bpf_link_info`.
--

commit 0277eab302404c5b366c836f0d334f0e2681388e
Author: Jiri Olsa <jolsa@kernel.org>

selftests/bpf: Add tracing_multi link info tests

Adding tracing_multi link info tests that follow the kprobe_multi
and uprobe_multi tests logic.

> diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
> index f589eefbf9fbd..0918321c8e63d 100644
> --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
> +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
[ ... ]
> @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel,
[ ... ]
> +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog,
> +					  const struct tmulti_target *targets,
> +					  __u32 btf_obj_id, bool has_cookies)
> +{
> +	enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog);
> +	__u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT];
> +	__u32 ids[TRACING_MULTI_CNT];
> +	struct bpf_link_info info;
> +	__u32 len = sizeof(info);
> +	int err, i;
> +
> +	memset(&info, 0, sizeof(info));
> +	err = bpf_link_get_info_by_fd(fd, &info, &len);
> +	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
> +		return -1;
> +
> +	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
> +		return -1;
> +
> +	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");

[Severity: Low]
Does this code fail to compile?  It looks like struct bpf_link_info does not
have a tracing_multi member.

[Severity: Medium]
Will this test unconditionally fail at runtime?

Looking at bpf_tracing_multi_link_lops in kernel/trace/bpf_trace.c, it
appears the .fill_link_info callback is missing for TRACING_MULTI links.

Since bpf_link_get_info_by_fd() returns 0 without populating link-specific
info or returning expected error codes when the callback is missing, won't
these assertions fail?

> +	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
> +
> +	memset(ids, 0, sizeof(ids));
> +	memset(cookies, 0, sizeof(cookies));
> +	memset(addrs, 0, sizeof(addrs));
> +
> +	info.tracing_multi.ids = ptr_to_u64(ids);

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260629143844.862369-3-jolsa@kernel.org?part=1

Re: [PATCHv3 bpf-next 3/3] bpftool: Add tracing_multi link info output

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Medium] The kernel does not implement `.fill_link_info` for `tracing_multi` links, preventing `bpftool` from querying the data.
- [Low] The patch references `tracing_multi` in `struct bpf_link_info`, but this member is missing from the UAPI header, leading to a compilation error.
--

commit e1b00f15ceb03dae490ba9269f47251c966b9424
Author: Jiri Olsa <jolsa@kernel.org>
bpftool: Add tracing_multi link info output

This commit adds support to bpftool to show tracing_multi link details
such as attach type, btf_id, address, cookie, and function module.

> diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> index bdcd717b0348f..eb0ba9d716924 100644
> --- a/tools/bpf/bpftool/link.c
> +++ b/tools/bpf/bpftool/link.c

[ ... ]

> @@ -1114,6 +1226,26 @@ static int do_show_link(int fd)
>  			goto again;
>  		}
>  	}
> +	if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) {

[Severity: Low]
The patch references tracing_multi in struct bpf_link_info here, but looking
at include/uapi/linux/bpf.h, there doesn't seem to be a tracing_multi member
defined in the union. Will this cause a compilation error when building bpftool?

[Severity: Medium]
Additionally, it appears the kernel-side bpf_tracing_multi_link_lops in
kernel/trace/bpf_trace.c does not yet implement a .fill_link_info callback.
Without that callback, will bpf_link_get_info_by_fd() fail to populate the
requested tracing_multi arrays (ids, addrs, cookies) for the tool to display,
even after the UAPI is updated?

> +		count = info.tracing_multi.count;
> +		if (count) {
> +			ids = calloc(count, sizeof(__u32));
> +			addrs = calloc(count, sizeof(__u64));
> +			cookies = calloc(count, sizeof(__u64));
> +			if (!ids) {
> +				p_err("mem alloc failed");
> +				close(fd);
> +				free(cookies);
> +				free(addrs);
> +				free(ids);
> +				return -ENOMEM;
> +			}
> +			info.tracing_multi.ids = ptr_to_u64(ids);
> +			info.tracing_multi.addrs = ptr_to_u64(addrs);
> +			info.tracing_multi.cookies = ptr_to_u64(cookies);
> +			goto again;
> +		}
> +	}

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260629143844.862369-4-jolsa@kernel.org?part=1

Re: [PATCHv3 bpf-next 3/3] bpftool: Add tracing_multi link info output

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 1239 bytes --]

> diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> index bdcd717b0348..eb0ba9d71692 100644
> --- a/tools/bpf/bpftool/link.c
> +++ b/tools/bpf/bpftool/link.c
> @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
>  	return (__u64 *) u64_to_ptr(val);
>  }
>
> +static __u32 *u64_to_u32_arr(__u64 val)
> +{
> +	return (__u32 *)u64_to_ptr(val);
> +}
> +
> +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
> +{
> +	struct kernel_sym *sym;
> +
> +	if (!addr)
> +		return NULL;
> +
> +	sym = kernel_syms_search(&dd, addr);
> +	if (!sym && is_ibt_enabled && addr >= 4)
> +		sym = kernel_syms_search(&dd, addr - 4);

When is_ibt_enabled is set, can addr ever be resolved by the first
kernel_syms_search(&dd, addr) call, or does a match always require the
addr - 4 adjustment?

If a match is only ever possible at addr - 4, would an if/else be clearer
here than trying addr first and then falling back to addr - 4?

> +
> +	return sym;
> +}

[ ... ]


---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/28384089901

Re: [PATCHv3 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Alexei Starovoitov]

On Mon Jun 29, 2026 at 7:38 AM PDT, Jiri Olsa wrote:
> +	if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) {
> +		count = info.tracing_multi.count;
> +		if (count) {
> +			ids = calloc(count, sizeof(__u32));
> +			addrs = calloc(count, sizeof(__u64));
> +			cookies = calloc(count, sizeof(__u64));
> +			if (!ids) {
> +				p_err("mem alloc failed");
> +				close(fd);
> +				free(cookies);
> +				free(addrs);
> +				free(ids);
> +				return -ENOMEM;
> +			}
> +			info.tracing_multi.ids = ptr_to_u64(ids);
> +			info.tracing_multi.addrs = ptr_to_u64(addrs);
> +			info.tracing_multi.cookies = ptr_to_u64(cookies);
> +			goto again;
> +		}

The above doesn't look correct.
Please de-llm your code before posting.

pw-bot: cr

Re: [PATCHv3 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Jiri Olsa]

On Mon, Jun 29, 2026 at 10:28:08AM -0700, Alexei Starovoitov wrote:
> On Mon Jun 29, 2026 at 7:38 AM PDT, Jiri Olsa wrote:
> > +	if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) {
> > +		count = info.tracing_multi.count;
> > +		if (count) {
> > +			ids = calloc(count, sizeof(__u32));
> > +			addrs = calloc(count, sizeof(__u64));
> > +			cookies = calloc(count, sizeof(__u64));
> > +			if (!ids) {
> > +				p_err("mem alloc failed");
> > +				close(fd);
> > +				free(cookies);
> > +				free(addrs);
> > +				free(ids);
> > +				return -ENOMEM;
> > +			}
> > +			info.tracing_multi.ids = ptr_to_u64(ids);
> > +			info.tracing_multi.addrs = ptr_to_u64(addrs);
> > +			info.tracing_multi.cookies = ptr_to_u64(cookies);
> > +			goto again;
> > +		}
> 
> The above doesn't look correct.
> Please de-llm your code before posting.

ugh will repost, sry

jirka

Re: [PATCHv3 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Jiri Olsa]

On Mon, Jun 29, 2026 at 04:05:27PM +0000, bot+bpf-ci@kernel.org wrote:
> > diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> > index bdcd717b0348..eb0ba9d71692 100644
> > --- a/tools/bpf/bpftool/link.c
> > +++ b/tools/bpf/bpftool/link.c
> > @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
> >  	return (__u64 *) u64_to_ptr(val);
> >  }
> >
> > +static __u32 *u64_to_u32_arr(__u64 val)
> > +{
> > +	return (__u32 *)u64_to_ptr(val);
> > +}
> > +
> > +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
> > +{
> > +	struct kernel_sym *sym;
> > +
> > +	if (!addr)
> > +		return NULL;
> > +
> > +	sym = kernel_syms_search(&dd, addr);
> > +	if (!sym && is_ibt_enabled && addr >= 4)
> > +		sym = kernel_syms_search(&dd, addr - 4);
> 
> When is_ibt_enabled is set, can addr ever be resolved by the first
> kernel_syms_search(&dd, addr) call, or does a match always require the
> addr - 4 adjustment?
> 
> If a match is only ever possible at addr - 4, would an if/else be clearer
> here than trying addr first and then falling back to addr - 4?

I see trace-able functions without endbr instruction at the entry
on kernel with IBT enabled, like:

	# cat available_filter_functions_addrs | grep get_next_root
	ffffffff81211a70 get_next_root
	# cat /proc/kallsyms | grep get_next_root
	ffffffff81211a70 t get_next_root

and having adjusted functions with endbr instruction as well:

	# cat available_filter_functions_addrs | grep ksys_read
	ffffffff816f6404 ksys_readahead
	ffffffff8182bf14 ksys_read
	# cat /proc/kallsyms | grep ksys_read
	ffffffff816f6400 T ksys_readahead
	ffffffff8182bf10 T ksys_read

so we need the check as is

jirka