[PATCHv2 bpf-next 0/3] bpf: tracing_multi link info support

[PATCHv2 bpf-next 0/3] bpf: tracing_multi link info support

[Jiri Olsa]

hi,
adding link info support for tracing_multi link.

Adding the kernel support with related selftests and bpftool support
to display the tracing_multi link info, the output looks like:

  # bpftool link
  ...
  27: tracing_multi  prog 93
          attach_type trace_fentry_multi  obj_id 1  count 3
          btf_id           addr             cookie           func [module]
          91984            ffffffff824f4a24 a                bpf_fentry_test1
          91986            ffffffff824f6a84 1e               bpf_fentry_test2
          91987            ffffffff824f6a94 14               bpf_fentry_test3
          pids test_progs(1462)

thanks,
jirka


v1: https://lore.kernel.org/bpf/20260621204524.61067-1-jolsa@kernel.org/

v2 changes:
- fix kernel symbol search logic [sashiko]
- typo fixes [bots]


---
Jiri Olsa (3):
      bpf: Add tracing_multi link info support
      selftests/bpf: Add tracing_multi link info tests
      bpftool: Add tracing_multi link info output

 include/uapi/linux/bpf.h                                |   9 +++
 kernel/trace/bpf_trace.c                                |  55 +++++++++++++++++
 tools/bpf/bpftool/link.c                                | 137 +++++++++++++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h                          |   9 +++
 tools/testing/selftests/bpf/prog_tests/fill_link_info.c | 242 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/testing/selftests/bpf/progs/test_fill_link_info.c |   6 ++
 6 files changed, 458 insertions(+)

[PATCHv2 bpf-next 1/3] bpf: Add tracing_multi link info support

[Jiri Olsa]

Adding BPF_OBJ_GET_INFO_BY_FD support for tracing_multi links.

We expose following tracing_multi link data:
- attach_type of the program
- number of ids
- array of BTF ids
- array of its related kernel addresses
- array of cookies

The change follows the kprobe_multi and uprobe_multi link-info convention
of optional output arrays with an in/out count,

On top of standard tracing link data we also expose addresses, because they
are useful info for user (especially when the attachment was done via pattern).
This data is hidden when kallsyms does not allow exposing kernel pointer values.

Assisted-by: Codex:GPT-5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 include/uapi/linux/bpf.h       |  9 ++++++
 kernel/trace/bpf_trace.c       | 55 ++++++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h |  9 ++++++
 3 files changed, 73 insertions(+)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 89b36de5fdbb..80423ccc6b57 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -6839,6 +6839,15 @@ struct bpf_link_info {
 			__u32 flags;
 			__u32 pid;
 		} uprobe_multi;
+		struct {
+			__u32 attach_type;
+			__u32 count; /* in/out: tracing_multi target count */
+			__u32 obj_id;
+			__u32 :32;
+			__aligned_u64 ids;
+			__aligned_u64 addrs;
+			__aligned_u64 cookies;
+		} tracing_multi;
 		struct {
 			__u32 type; /* enum bpf_perf_event_type */
 			__u32 :32;
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 82f8feea6931..fdab4ec5ad4f 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -3699,6 +3699,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link)
 	kvfree(tr_link);
 }
 
+static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link,
+						 struct bpf_link_info *info)
+{
+	u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies);
+	u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs);
+	u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids);
+	struct bpf_tracing_multi_link *tr_link;
+	u32 ucount = info->tracing_multi.count;
+	bool has_cookies, show_addrs;
+	int err = 0;
+
+	if ((uids || ucookies || uaddrs) && !ucount)
+		return -EINVAL;
+
+	tr_link = container_of(link, struct bpf_tracing_multi_link, link);
+
+	info->tracing_multi.attach_type = tr_link->link.attach_type;
+	info->tracing_multi.count = tr_link->nodes_cnt;
+	info->tracing_multi.obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf);
+
+	if (!uids && !ucookies && !uaddrs)
+		return 0;
+
+	if (ucount < tr_link->nodes_cnt)
+		err = -ENOSPC;
+	else
+		ucount = tr_link->nodes_cnt;
+
+	has_cookies = !!tr_link->cookies;
+	show_addrs = kallsyms_show_value(current_cred());
+
+	for (int i = 0; i < ucount; i++) {
+		struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i];
+		u64 addr, cookie;
+		u32 id;
+
+		bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id);
+
+		addr = show_addrs ? mnode->trampoline->ip : 0;
+		cookie = has_cookies ? tr_link->cookies[i] : 0;
+
+		if (uids && put_user(id, uids + i))
+			return -EFAULT;
+		if (uaddrs && put_user(addr, uaddrs + i))
+			return -EFAULT;
+		if (ucookies && put_user(cookie, ucookies + i))
+			return -EFAULT;
+
+		cond_resched();
+	}
+
+	return err;
+}
+
 #ifdef CONFIG_PROC_FS
 static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
 					  struct seq_file *seq)
@@ -3729,6 +3783,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
 static const struct bpf_link_ops bpf_tracing_multi_link_lops = {
 	.release = bpf_tracing_multi_link_release,
 	.dealloc_deferred = bpf_tracing_multi_link_dealloc,
+	.fill_link_info = bpf_tracing_multi_link_fill_link_info,
 #ifdef CONFIG_PROC_FS
 	.show_fdinfo = bpf_tracing_multi_show_fdinfo,
 #endif
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 89b36de5fdbb..80423ccc6b57 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -6839,6 +6839,15 @@ struct bpf_link_info {
 			__u32 flags;
 			__u32 pid;
 		} uprobe_multi;
+		struct {
+			__u32 attach_type;
+			__u32 count; /* in/out: tracing_multi target count */
+			__u32 obj_id;
+			__u32 :32;
+			__aligned_u64 ids;
+			__aligned_u64 addrs;
+			__aligned_u64 cookies;
+		} tracing_multi;
 		struct {
 			__u32 type; /* enum bpf_perf_event_type */
 			__u32 :32;
-- 
2.54.0

Re: [PATCHv2 bpf-next 1/3] bpf: Add tracing_multi link info support

[Leon Hwang]

On 2026/6/23 22:24, Jiri Olsa wrote:
> Adding BPF_OBJ_GET_INFO_BY_FD support for tracing_multi links.
> 
> We expose following tracing_multi link data:
> - attach_type of the program
> - number of ids
> - array of BTF ids
> - array of its related kernel addresses
> - array of cookies
> 
> The change follows the kprobe_multi and uprobe_multi link-info convention
> of optional output arrays with an in/out count,
> 
> On top of standard tracing link data we also expose addresses, because they
> are useful info for user (especially when the attachment was done via pattern).
> This data is hidden when kallsyms does not allow exposing kernel pointer values.
> 
> Assisted-by: Codex:GPT-5
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
lgtm,

Acked-by: Leon Hwang <leon.hwang@linux.dev>

Re: [PATCHv2 bpf-next 1/3] bpf: Add tracing_multi link info support

[Andrii Nakryiko]

On Tue, Jun 23, 2026 at 7:24 AM Jiri Olsa <jolsa@kernel.org> wrote:
>
> Adding BPF_OBJ_GET_INFO_BY_FD support for tracing_multi links.
>
> We expose following tracing_multi link data:
> - attach_type of the program
> - number of ids
> - array of BTF ids
> - array of its related kernel addresses
> - array of cookies
>
> The change follows the kprobe_multi and uprobe_multi link-info convention
> of optional output arrays with an in/out count,
>
> On top of standard tracing link data we also expose addresses, because they
> are useful info for user (especially when the attachment was done via pattern).
> This data is hidden when kallsyms does not allow exposing kernel pointer values.
>
> Assisted-by: Codex:GPT-5
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---
>  include/uapi/linux/bpf.h       |  9 ++++++
>  kernel/trace/bpf_trace.c       | 55 ++++++++++++++++++++++++++++++++++
>  tools/include/uapi/linux/bpf.h |  9 ++++++
>  3 files changed, 73 insertions(+)
>
> diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
> index 89b36de5fdbb..80423ccc6b57 100644
> --- a/include/uapi/linux/bpf.h
> +++ b/include/uapi/linux/bpf.h
> @@ -6839,6 +6839,15 @@ struct bpf_link_info {
>                         __u32 flags;
>                         __u32 pid;
>                 } uprobe_multi;
> +               struct {
> +                       __u32 attach_type;
> +                       __u32 count; /* in/out: tracing_multi target count */
> +                       __u32 obj_id;

we have attach_btf_obj_id in bpf_prog_info, but target_obj_id in
bpf_link_info for tracing programs. The latter, though, is actually
not always BTF object ID (it can be prog_id for PROG_EXT)...

So, me personally, I was confused what "obj_id" here mean, it would be
more obvious if it was called "btf_obj_id", tbh. But that's a minor
nit/confusion, not feeling strongly.

everything else LGTM

Acked-by: Andrii Nakryiko <andrii@kernel.org>


> +                       __u32 :32;
> +                       __aligned_u64 ids;
> +                       __aligned_u64 addrs;
> +                       __aligned_u64 cookies;
> +               } tracing_multi;
>                 struct {
>                         __u32 type; /* enum bpf_perf_event_type */
>                         __u32 :32;

[...]

Re: [PATCHv2 bpf-next 1/3] bpf: Add tracing_multi link info support

[Jiri Olsa]

On Tue, Jun 23, 2026 at 02:01:08PM -0700, Andrii Nakryiko wrote:
> On Tue, Jun 23, 2026 at 7:24 AM Jiri Olsa <jolsa@kernel.org> wrote:
> >
> > Adding BPF_OBJ_GET_INFO_BY_FD support for tracing_multi links.
> >
> > We expose following tracing_multi link data:
> > - attach_type of the program
> > - number of ids
> > - array of BTF ids
> > - array of its related kernel addresses
> > - array of cookies
> >
> > The change follows the kprobe_multi and uprobe_multi link-info convention
> > of optional output arrays with an in/out count,
> >
> > On top of standard tracing link data we also expose addresses, because they
> > are useful info for user (especially when the attachment was done via pattern).
> > This data is hidden when kallsyms does not allow exposing kernel pointer values.
> >
> > Assisted-by: Codex:GPT-5
> > Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> > ---
> >  include/uapi/linux/bpf.h       |  9 ++++++
> >  kernel/trace/bpf_trace.c       | 55 ++++++++++++++++++++++++++++++++++
> >  tools/include/uapi/linux/bpf.h |  9 ++++++
> >  3 files changed, 73 insertions(+)
> >
> > diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
> > index 89b36de5fdbb..80423ccc6b57 100644
> > --- a/include/uapi/linux/bpf.h
> > +++ b/include/uapi/linux/bpf.h
> > @@ -6839,6 +6839,15 @@ struct bpf_link_info {
> >                         __u32 flags;
> >                         __u32 pid;
> >                 } uprobe_multi;
> > +               struct {
> > +                       __u32 attach_type;
> > +                       __u32 count; /* in/out: tracing_multi target count */
> > +                       __u32 obj_id;
> 
> we have attach_btf_obj_id in bpf_prog_info, but target_obj_id in
> bpf_link_info for tracing programs. The latter, though, is actually
> not always BTF object ID (it can be prog_id for PROG_EXT)...
> 
> So, me personally, I was confused what "obj_id" here mean, it would be
> more obvious if it was called "btf_obj_id", tbh. But that's a minor
> nit/confusion, not feeling strongly.

ok, btf_obj_id si better choice, will change

> 
> everything else LGTM
> 
> Acked-by: Andrii Nakryiko <andrii@kernel.org>

thanks,
jirka

[PATCHv2 bpf-next 2/3] selftests/bpf: Add tracing_multi link info tests

[Jiri Olsa]

Adding tracing_multi link info tests that follow the kprobe_multi
and uprobe_multi tests logic.

Assisted-by: Codex:GPT-5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 .../selftests/bpf/prog_tests/fill_link_info.c | 242 ++++++++++++++++++
 .../selftests/bpf/progs/test_fill_link_info.c |   6 +
 2 files changed, 248 insertions(+)

diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
index f589eefbf9fb..4d021e9b0129 100644
--- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
+++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
@@ -4,6 +4,7 @@
 #include <string.h>
 #include <linux/bpf.h>
 #include <linux/limits.h>
+#include <bpf/btf.h>
 #include <test_progs.h>
 #include "trace_helpers.h"
 #include "test_fill_link_info.skel.h"
@@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 };
 #define KPROBE_FUNC "bpf_fentry_test1"
 static __u64 kprobe_addr;
 
+static const char * const tmulti_syms[] = {
+	"bpf_fentry_test2",
+	"bpf_fentry_test1",
+	"bpf_fentry_test3",
+};
+
+static __u64 tmulti_cookies[] = { 30, 10, 20 };
+#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms)
+
+struct tmulti_target {
+	const char *name;
+	__u64 addr;
+	__u64 cookie;
+	__u32 id;
+};
+
 #define UPROBE_FILE "/proc/self/exe"
 static ssize_t uprobe_offset;
 /* uprobe attach point */
@@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel,
 	bpf_link__destroy(link);
 }
 
+static int tmulti_target_cmp(const void *a, const void *b)
+{
+	const struct tmulti_target *ta = a;
+	const struct tmulti_target *tb = b;
+
+	return (ta->id > tb->id) - (ta->id < tb->id);
+}
+
+static int setup_tmulti_targets(const struct bpf_program *prog,
+				struct tmulti_target *targets,
+				__u32 *obj_id)
+{
+	struct bpf_prog_info prog_info;
+	__u32 len = sizeof(prog_info);
+	struct btf *btf;
+	int err, i;
+	__s32 id;
+
+	btf = btf__load_vmlinux_btf();
+	if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf"))
+		return -1;
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC);
+		if (!ASSERT_GT(id, 0, "btf__find_by_name_kind"))
+			goto error;
+
+		targets[i].name = tmulti_syms[i];
+		targets[i].addr = ksym_get_addr(tmulti_syms[i]);
+		targets[i].cookie = tmulti_cookies[i];
+		targets[i].id = id;
+	}
+
+	memset(&prog_info, 0, len);
+	err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len);
+	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd"))
+		goto error;
+	if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id"))
+		goto error;
+	*obj_id = prog_info.attach_btf_obj_id;
+
+	/*
+	 * The kernel tracing multi attach sorts ids. We sort as well,
+	 * so we can easily compare ids and cookies later.
+	 */
+	qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp);
+	btf__free(btf);
+	return 0;
+
+error:
+	btf__free(btf);
+	return -1;
+}
+
+static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog,
+					  const struct tmulti_target *targets,
+					  __u32 obj_id, bool has_cookies)
+{
+	enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog);
+	__u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT];
+	__u32 ids[TRACING_MULTI_CNT];
+	struct bpf_link_info info;
+	__u32 len = sizeof(info);
+	int err, i;
+
+	memset(&info, 0, sizeof(info));
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
+		return -1;
+
+	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
+		return -1;
+
+	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");
+	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
+
+	memset(ids, 0, sizeof(ids));
+	memset(cookies, 0, sizeof(cookies));
+	memset(addrs, 0, sizeof(addrs));
+
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.addrs = ptr_to_u64(addrs);
+	info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
+		return -1;
+
+	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
+		return -1;
+
+	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");
+	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
+	ASSERT_EQ(info.tracing_multi.obj_id, obj_id, "tracing_multi.target_obj_ids");
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.target_btf_ids");
+		ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies");
+
+		if (targets[i].addr) {
+			struct ksym *ksym;
+
+			if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs"))
+				return -1;
+			ksym = ksym_search(addrs[i]);
+			if (!ASSERT_OK_PTR(ksym, "ksym_search"))
+				return -1;
+			ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name");
+		} else {
+			ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs");
+		}
+	}
+
+	return 0;
+}
+
+static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets)
+{
+	__u32 ids[TRACING_MULTI_CNT] = {};
+	struct bpf_link_info info;
+	__u32 len = sizeof(info);
+	int err, i;
+
+	/* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count");
+
+	/* Smaller than actual count provided -> ENOSPC */
+	memset(ids, 0, sizeof(ids));
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.count = TRACING_MULTI_CNT - 1;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count");
+	for (i = 0; i < TRACING_MULTI_CNT - 1; i++)
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids");
+	/* check that the last entry is not populated */
+	ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids");
+
+	/* Bigger than actual count provided -> OK */
+	memset(ids, 0, sizeof(ids));
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.count = TRACING_MULTI_CNT + 1;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_OK(err, "tracing_multi.big_count");
+	for (i = 0; i < TRACING_MULTI_CNT; i++)
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids");
+
+	/* Invalid ids pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids");
+
+	/* Invalid cookies pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.cookies = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies");
+
+	/* Invalid addrs pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.addrs = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs");
+}
+
+static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel,
+					      bool has_cookies, bool invalid)
+{
+	LIBBPF_OPTS(bpf_tracing_multi_opts, opts);
+	struct tmulti_target targets[TRACING_MULTI_CNT];
+	__u32 ids[TRACING_MULTI_CNT], obj_id;
+	__u64 cookies[TRACING_MULTI_CNT];
+	struct bpf_link *link;
+	int link_fd, err, i;
+
+#ifndef __x86_64__
+	test__skip();
+	return;
+#endif
+
+	if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &obj_id))
+		return;
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		ids[i] = targets[i].id;
+		cookies[i] = targets[i].cookie;
+	}
+
+	opts.ids = ids;
+	opts.cnt = TRACING_MULTI_CNT;
+	if (has_cookies)
+		opts.cookies = cookies;
+
+	link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi"))
+		return;
+
+	link_fd = bpf_link__fd(link);
+	if (invalid) {
+		verify_tracing_multi_invalid_user_buffer(link_fd, targets);
+	} else {
+		err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run,
+						     targets, obj_id, has_cookies);
+		ASSERT_OK(err, "verify_tracing_multi_link_info");
+	}
+
+	bpf_link__destroy(link);
+}
+
 #define SEC(name) __attribute__((section(name), used))
 
 static short uprobe_link_info_sema_1 SEC(".probes");
@@ -640,6 +875,13 @@ void test_fill_link_info(void)
 	if (test__start_subtest("kprobe_multi_invalid_ubuff"))
 		test_kprobe_multi_fill_link_info(skel, true, true, true);
 
+	if (test__start_subtest("tracing_multi_link_info")) {
+		test_tracing_multi_fill_link_info(skel, false, false);
+		test_tracing_multi_fill_link_info(skel, true, false);
+	}
+	if (test__start_subtest("tracing_multi_invalid_ubuff"))
+		test_tracing_multi_fill_link_info(skel, true, true);
+
 	if (test__start_subtest("uprobe_multi_link_info"))
 		test_uprobe_multi_fill_link_info(skel, false, false);
 	if (test__start_subtest("uretprobe_multi_link_info"))
diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c
index 137bd6292163..c85081538e93 100644
--- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c
+++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c
@@ -58,4 +58,10 @@ int BPF_PROG(umulti_run)
 	return 0;
 }
 
+SEC("fentry.multi")
+int BPF_PROG(tmulti_run)
+{
+	return 0;
+}
+
 char _license[] SEC("license") = "GPL";
-- 
2.54.0

Re: [PATCHv2 bpf-next 2/3] selftests/bpf: Add tracing_multi link info tests

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Medium] The patch adds selftests for `tracing_multi` link info (`BPF_LINK_TYPE_TRACING_MULTI`), but the required kernel-side implementation is entirely missing.
- [Low] The patch references `info.tracing_multi` inside `struct bpf_link_info`, but this field does not exist in the UAPI headers.
--

commit 23bb30d6d6d285433f08e70bf1a166ecfd023a7d
Author: Jiri Olsa <jolsa@kernel.org>

selftests/bpf: Add tracing_multi link info tests
    
Adding tracing_multi link info tests that follow the kprobe_multi
and uprobe_multi tests logic.

> diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
> index f589eefbf9fbd..4d021e9b0129c 100644
> --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
> +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
[ ... ]
> +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog,
> +					  const struct tmulti_target *targets,
> +					  __u32 obj_id, bool has_cookies)
> +{
[ ... ]
> +	err = bpf_link_get_info_by_fd(fd, &info, &len);
> +	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
> +		return -1;
> +
> +	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
> +		return -1;
> +
> +	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");

[Severity: Low]
Does the tracing_multi field exist in struct bpf_link_info within the UAPI
headers?

It doesn't seem to be present in include/uapi/linux/bpf.h yet, which would
cause a build failure here when accessing info.tracing_multi.attach_type.

> +	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");

[Severity: Medium]
Is the kernel-side implementation for filling this link info missing?

Without a .fill_link_info callback defined in bpf_tracing_multi_link_lops in
kernel/trace/bpf_trace.c, bpf_link_get_info_by_fd() will leave the
tracing_multi struct zero-initialized, causing this assertion to fail.

[ ... ]
> +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets)
> +{
> +	__u32 ids[TRACING_MULTI_CNT] = {};
> +	struct bpf_link_info info;
> +	__u32 len = sizeof(info);
> +	int err, i;
> +
> +	/* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */
> +	memset(&info, 0, sizeof(info));
> +	info.tracing_multi.ids = ptr_to_u64(ids);
> +	err = bpf_link_get_info_by_fd(fd, &info, &len);
> +	ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count");

[Severity: Medium]
Will this return the expected -EINVAL error?

Since the kernel doesn't appear to implement .fill_link_info for this link
type yet, the validation that would return -EINVAL won't execute, causing
this assertion to fail.

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260623142417.275892-3-jolsa@kernel.org?part=1

[PATCHv2 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Jiri Olsa]

Adding bpftool support to show tracing_multi link details,
the new output looks like:

  # bpftool link
  ...
  27: tracing_multi  prog 93
          attach_type trace_fentry_multi  obj_id 1  count 3
          btf_id           addr             cookie           func [module]
          91984            ffffffff824f4a24 a                bpf_fentry_test1
          91986            ffffffff824f6a84 1e               bpf_fentry_test2
          91987            ffffffff824f6a94 14               bpf_fentry_test3
          pids test_progs(1462)

Assisted-by: Codex:GPT-5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 tools/bpf/bpftool/link.c | 137 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index bdcd717b0348..db780ee39820 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
 	return (__u64 *) u64_to_ptr(val);
 }
 
+static __u32 *u64_to_u32_arr(__u64 val)
+{
+	return (__u32 *)u64_to_ptr(val);
+}
+
+static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
+{
+	struct kernel_sym *sym;
+
+	if (!addr)
+		return NULL;
+
+	sym = kernel_syms_search(&dd, addr);
+	if (!sym && is_ibt_enabled && addr >= 4)
+		sym = kernel_syms_search(&dd, addr - 4);
+
+	return sym;
+}
+
 static void
 show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
@@ -403,6 +422,47 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
 	jsonw_end_array(json_wtr);
 }
 
+static void
+show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+	bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+	__u32 i;
+
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	show_symbol = !!dd.sym_count;
+
+	show_link_attach_type_json(info->tracing_multi.attach_type, wtr);
+	jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count);
+	jsonw_uint_field(wtr, "obj_id", info->tracing_multi.obj_id);
+	jsonw_name(wtr, "funcs");
+
+	jsonw_start_array(wtr);
+
+	for (i = 0; i < info->tracing_multi.count; i++) {
+		__u64 addr = u64_to_arr(info->tracing_multi.addrs)[i];
+		struct kernel_sym *sym;
+
+		sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+		jsonw_start_object(wtr);
+		jsonw_uint_field(wtr, "id", u64_to_u32_arr(info->tracing_multi.ids)[i]);
+		jsonw_uint_field(wtr, "addr", addr);
+		if (sym) {
+			jsonw_string_field(wtr, "func", sym->name);
+			if (sym->module[0] == '\0') {
+				jsonw_name(wtr, "module");
+				jsonw_null(wtr);
+			} else {
+				jsonw_string_field(wtr, "module", sym->module);
+			}
+		}
+		jsonw_uint_field(wtr, "cookie", u64_to_arr(info->tracing_multi.cookies)[i]);
+		jsonw_end_object(wtr);
+	}
+	jsonw_end_array(wtr);
+}
+
 static void
 show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
@@ -589,6 +649,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_UPROBE_MULTI:
 		show_uprobe_multi_json(info, json_wtr);
 		break;
+	case BPF_LINK_TYPE_TRACING_MULTI:
+		show_tracing_multi_json(info, json_wtr);
+		break;
 	case BPF_LINK_TYPE_PERF_EVENT:
 		switch (info->perf_event.type) {
 		case BPF_PERF_EVENT_EVENT:
@@ -833,6 +896,42 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
 	}
 }
 
+static void show_tracing_multi_plain(struct bpf_link_info *info)
+{
+	bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+	__u32 i;
+
+	if (!info->tracing_multi.count)
+		return;
+
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	show_symbol = !!dd.sym_count;
+
+	printf("\n\t");
+	show_link_attach_type_plain(info->tracing_multi.attach_type);
+	printf("obj_id %u  ", info->tracing_multi.obj_id);
+	printf("count %u  ", info->tracing_multi.count);
+
+	printf("\n\t%-16s %-16s %-16s %s",
+	       "btf_id", "addr", "cookie", "func [module]");
+	for (i = 0; i < info->tracing_multi.count; i++) {
+		__u64 addr = u64_to_arr(info->tracing_multi.addrs)[i];
+		struct kernel_sym *sym;
+
+		sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+		printf("\n\t%-16u %016llx %-16llx",
+		       u64_to_u32_arr(info->tracing_multi.ids)[i],
+		       addr, u64_to_arr(info->tracing_multi.cookies)[i]);
+		if (sym) {
+			printf(" %s", sym->name);
+			if (sym->module[0] != '\0')
+				printf(" [%s]", sym->module);
+		}
+	}
+}
+
 static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
 {
 	const char *buf;
@@ -989,6 +1088,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_UPROBE_MULTI:
 		show_uprobe_multi_plain(info);
 		break;
+	case BPF_LINK_TYPE_TRACING_MULTI:
+		show_tracing_multi_plain(info);
+		break;
 	case BPF_LINK_TYPE_PERF_EVENT:
 		switch (info->perf_event.type) {
 		case BPF_PERF_EVENT_EVENT:
@@ -1029,6 +1131,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 static int do_show_link(int fd)
 {
 	__u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL;
+	__u32 *ids = NULL;
 	struct bpf_link_info info;
 	__u32 len = sizeof(info);
 	char path_buf[PATH_MAX];
@@ -1114,6 +1217,39 @@ static int do_show_link(int fd)
 			goto again;
 		}
 	}
+	if (info.type == BPF_LINK_TYPE_TRACING_MULTI &&
+	    !info.tracing_multi.ids) {
+		count = info.tracing_multi.count;
+		if (count) {
+			ids = calloc(count, sizeof(__u32));
+			if (!ids) {
+				p_err("mem alloc failed");
+				close(fd);
+				return -ENOMEM;
+			}
+			info.tracing_multi.ids = ptr_to_u64(ids);
+
+			addrs = calloc(count, sizeof(__u64));
+			if (!addrs) {
+				p_err("mem alloc failed");
+				free(ids);
+				close(fd);
+				return -ENOMEM;
+			}
+			info.tracing_multi.addrs = ptr_to_u64(addrs);
+
+			cookies = calloc(count, sizeof(__u64));
+			if (!cookies) {
+				p_err("mem alloc failed");
+				free(addrs);
+				free(ids);
+				close(fd);
+				return -ENOMEM;
+			}
+			info.tracing_multi.cookies = ptr_to_u64(cookies);
+			goto again;
+		}
+	}
 	if (info.type == BPF_LINK_TYPE_PERF_EVENT) {
 		switch (info.perf_event.type) {
 		case BPF_PERF_EVENT_TRACEPOINT:
@@ -1153,6 +1289,7 @@ static int do_show_link(int fd)
 	free(cookies);
 	free(offsets);
 	free(addrs);
+	free(ids);
 	close(fd);
 	return 0;
 }
-- 
2.54.0

Re: [PATCHv2 bpf-next 3/3] bpftool: Add tracing_multi link info output

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Medium] The patch references non-existent `tracing_multi` fields in `struct bpf_link_info` causing a build failure, and the corresponding kernel `.fill_link_info` callback is missing.
- [Medium] Dynamically allocated memory for `ids`, `addrs`, and `cookies` arrays is leaked if the subsequent `bpf_link_get_info_by_fd()` call fails.
--

commit 1c150871ddc705d0719c83498c9983e353211882
Author: Jiri Olsa <jolsa@kernel.org>

bpftool: Add tracing_multi link info output

This patch adds support to bpftool to display tracing_multi link details.
It updates both the JSON and plain text output formats to show attached
functions, addresses, and cookies.

> diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> index bdcd717b0348f..db780ee398206 100644
> --- a/tools/bpf/bpftool/link.c
> +++ b/tools/bpf/bpftool/link.c
[ ... ]
> @@ -1114,6 +1217,39 @@ static int do_show_link(int fd)
>  			goto again;
>  		}
>  	}
> +	if (info.type == BPF_LINK_TYPE_TRACING_MULTI &&
> +	    !info.tracing_multi.ids) {

[Severity: Medium]
Will this break the build? It appears the tracing_multi struct is missing
from the bpf_link_info union in include/uapi/linux/bpf.h.

Also, it seems the kernel side might be missing the .fill_link_info
callback in bpf_tracing_multi_link_lops (in kernel/trace/bpf_trace.c)
to export this data to userspace.

> +		count = info.tracing_multi.count;
> +		if (count) {
> +			ids = calloc(count, sizeof(__u32));
> +			if (!ids) {
> +				p_err("mem alloc failed");
> +				close(fd);
> +				return -ENOMEM;
> +			}
> +			info.tracing_multi.ids = ptr_to_u64(ids);
> +
> +			addrs = calloc(count, sizeof(__u64));
> +			if (!addrs) {
> +				p_err("mem alloc failed");
> +				free(ids);
> +				close(fd);
> +				return -ENOMEM;
> +			}
> +			info.tracing_multi.addrs = ptr_to_u64(addrs);
> +
> +			cookies = calloc(count, sizeof(__u64));
> +			if (!cookies) {
> +				p_err("mem alloc failed");
> +				free(addrs);
> +				free(ids);
> +				close(fd);
> +				return -ENOMEM;
> +			}
> +			info.tracing_multi.cookies = ptr_to_u64(cookies);
> +			goto again;

[Severity: Medium]
If the execution jumps to again: and the subsequent call to
bpf_link_get_info_by_fd() fails, does this leak the memory allocated
for ids, addrs, and cookies?

Looking at the error path earlier in do_show_link():

again:
	err = bpf_link_get_info_by_fd(fd, &info, &len);
	if (err) {
		p_err("can't get link info: %s",
		      strerror(errno));
		close(fd);
		return err;
	}

It returns immediately without freeing these arrays if the second
syscall fails (e.g., due to a concurrent increase in function count
or link destruction).

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260623142417.275892-4-jolsa@kernel.org?part=1

Re: [PATCHv2 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Andrii Nakryiko]

On Tue, Jun 23, 2026 at 7:24 AM Jiri Olsa <jolsa@kernel.org> wrote:
>
> Adding bpftool support to show tracing_multi link details,
> the new output looks like:
>
>   # bpftool link
>   ...
>   27: tracing_multi  prog 93
>           attach_type trace_fentry_multi  obj_id 1  count 3
>           btf_id           addr             cookie           func [module]
>           91984            ffffffff824f4a24 a                bpf_fentry_test1
>           91986            ffffffff824f6a84 1e               bpf_fentry_test2
>           91987            ffffffff824f6a94 14               bpf_fentry_test3
>           pids test_progs(1462)
>
> Assisted-by: Codex:GPT-5
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---
>  tools/bpf/bpftool/link.c | 137 +++++++++++++++++++++++++++++++++++++++
>  1 file changed, 137 insertions(+)
>
> diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> index bdcd717b0348..db780ee39820 100644
> --- a/tools/bpf/bpftool/link.c
> +++ b/tools/bpf/bpftool/link.c
> @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
>         return (__u64 *) u64_to_ptr(val);
>  }
>
> +static __u32 *u64_to_u32_arr(__u64 val)
> +{
> +       return (__u32 *)u64_to_ptr(val);
> +}
> +
> +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
> +{
> +       struct kernel_sym *sym;
> +
> +       if (!addr)
> +               return NULL;
> +
> +       sym = kernel_syms_search(&dd, addr);
> +       if (!sym && is_ibt_enabled && addr >= 4)
> +               sym = kernel_syms_search(&dd, addr - 4);

if is_ibt_enabled, can we match addr with kernel_syms_search at all?
I.e., can addr sometimes be resolvable without - 4 adjustment? If not,
why is this not if/else and instead "try this, if not, try that"?

> +
> +       return sym;
> +}
> +
>  static void
>  show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
>  {
> @@ -403,6 +422,47 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
>         jsonw_end_array(json_wtr);
>  }
>
> +static void
> +show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
> +{
> +       bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
> +       __u32 i;
> +
> +       if (!dd.sym_count)
> +               kernel_syms_load(&dd);
> +       show_symbol = !!dd.sym_count;
> +
> +       show_link_attach_type_json(info->tracing_multi.attach_type, wtr);
> +       jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count);
> +       jsonw_uint_field(wtr, "obj_id", info->tracing_multi.obj_id);
> +       jsonw_name(wtr, "funcs");
> +
> +       jsonw_start_array(wtr);
> +
> +       for (i = 0; i < info->tracing_multi.count; i++) {
> +               __u64 addr = u64_to_arr(info->tracing_multi.addrs)[i];
> +               struct kernel_sym *sym;
> +
> +               sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
> +
> +               jsonw_start_object(wtr);
> +               jsonw_uint_field(wtr, "id", u64_to_u32_arr(info->tracing_multi.ids)[i]);

nit, here and for cookies, do the cast outside of the loop into local
variable? seems cleaner as an approach

> +               jsonw_uint_field(wtr, "addr", addr);
> +               if (sym) {
> +                       jsonw_string_field(wtr, "func", sym->name);
> +                       if (sym->module[0] == '\0') {
> +                               jsonw_name(wtr, "module");
> +                               jsonw_null(wtr);

just curious, is it cleaner to have "module: null" instead of just not
emitting "module:" field at all?

> +                       } else {
> +                               jsonw_string_field(wtr, "module", sym->module);
> +                       }
> +               }
> +               jsonw_uint_field(wtr, "cookie", u64_to_arr(info->tracing_multi.cookies)[i]);
> +               jsonw_end_object(wtr);
> +       }
> +       jsonw_end_array(wtr);
> +}
> +
>  static void
>  show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
>  {
> @@ -589,6 +649,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
>         case BPF_LINK_TYPE_UPROBE_MULTI:
>                 show_uprobe_multi_json(info, json_wtr);
>                 break;
> +       case BPF_LINK_TYPE_TRACING_MULTI:
> +               show_tracing_multi_json(info, json_wtr);
> +               break;
>         case BPF_LINK_TYPE_PERF_EVENT:
>                 switch (info->perf_event.type) {
>                 case BPF_PERF_EVENT_EVENT:
> @@ -833,6 +896,42 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
>         }
>  }
>
> +static void show_tracing_multi_plain(struct bpf_link_info *info)
> +{
> +       bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
> +       __u32 i;
> +
> +       if (!info->tracing_multi.count)
> +               return;
> +
> +       if (!dd.sym_count)
> +               kernel_syms_load(&dd);
> +       show_symbol = !!dd.sym_count;
> +
> +       printf("\n\t");
> +       show_link_attach_type_plain(info->tracing_multi.attach_type);
> +       printf("obj_id %u  ", info->tracing_multi.obj_id);
> +       printf("count %u  ", info->tracing_multi.count);
> +
> +       printf("\n\t%-16s %-16s %-16s %s",
> +              "btf_id", "addr", "cookie", "func [module]");
> +       for (i = 0; i < info->tracing_multi.count; i++) {
> +               __u64 addr = u64_to_arr(info->tracing_multi.addrs)[i];
> +               struct kernel_sym *sym;
> +
> +               sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
> +
> +               printf("\n\t%-16u %016llx %-16llx",
> +                      u64_to_u32_arr(info->tracing_multi.ids)[i],
> +                      addr, u64_to_arr(info->tracing_multi.cookies)[i]);

As I mentioned, it surprised me that we emit cookie as hex. Kernel
doesn't in fdinfo output. And at least in my experience cookie is some
small compact index into some additional lookup array, so decimal
makes most sense. I see now that we have hex for uprobe_multi, but
decimal for perf link, so ugh, goodbye consistency. But still, decimal
will be much more useful in practice, IMO.

> +               if (sym) {
> +                       printf(" %s", sym->name);
> +                       if (sym->module[0] != '\0')
> +                               printf(" [%s]", sym->module);
> +               }
> +       }
> +}
> +
>  static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
>  {
>         const char *buf;
> @@ -989,6 +1088,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
>         case BPF_LINK_TYPE_UPROBE_MULTI:
>                 show_uprobe_multi_plain(info);
>                 break;
> +       case BPF_LINK_TYPE_TRACING_MULTI:
> +               show_tracing_multi_plain(info);
> +               break;
>         case BPF_LINK_TYPE_PERF_EVENT:
>                 switch (info->perf_event.type) {
>                 case BPF_PERF_EVENT_EVENT:
> @@ -1029,6 +1131,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
>  static int do_show_link(int fd)
>  {
>         __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL;
> +       __u32 *ids = NULL;
>         struct bpf_link_info info;
>         __u32 len = sizeof(info);
>         char path_buf[PATH_MAX];
> @@ -1114,6 +1217,39 @@ static int do_show_link(int fd)
>                         goto again;
>                 }
>         }
> +       if (info.type == BPF_LINK_TYPE_TRACING_MULTI &&
> +           !info.tracing_multi.ids) {

why wrap?

> +               count = info.tracing_multi.count;
> +               if (count) {
> +                       ids = calloc(count, sizeof(__u32));
> +                       if (!ids) {
> +                               p_err("mem alloc failed");
> +                               close(fd);
> +                               return -ENOMEM;
> +                       }
> +                       info.tracing_multi.ids = ptr_to_u64(ids);
> +
> +                       addrs = calloc(count, sizeof(__u64));
> +                       if (!addrs) {
> +                               p_err("mem alloc failed");
> +                               free(ids);
> +                               close(fd);
> +                               return -ENOMEM;
> +                       }
> +                       info.tracing_multi.addrs = ptr_to_u64(addrs);
> +
> +                       cookies = calloc(count, sizeof(__u64));
> +                       if (!cookies) {
> +                               p_err("mem alloc failed");
> +                               free(addrs);
> +                               free(ids);
> +                               close(fd);
> +                               return -ENOMEM;
> +                       }

ugh... do three calloc()s in a row, check that all succeeded, if not -
free and close(fd)? shorter and just as good

> +                       info.tracing_multi.cookies = ptr_to_u64(cookies);
> +                       goto again;
> +               }
> +       }
>         if (info.type == BPF_LINK_TYPE_PERF_EVENT) {
>                 switch (info.perf_event.type) {
>                 case BPF_PERF_EVENT_TRACEPOINT:
> @@ -1153,6 +1289,7 @@ static int do_show_link(int fd)
>         free(cookies);
>         free(offsets);
>         free(addrs);
> +       free(ids);
>         close(fd);
>         return 0;
>  }
> --
> 2.54.0
>

Re: [PATCHv2 bpf-next 3/3] bpftool: Add tracing_multi link info output

[Jiri Olsa]

On Tue, Jun 23, 2026 at 02:11:34PM -0700, Andrii Nakryiko wrote:
> On Tue, Jun 23, 2026 at 7:24 AM Jiri Olsa <jolsa@kernel.org> wrote:
> >
> > Adding bpftool support to show tracing_multi link details,
> > the new output looks like:
> >
> >   # bpftool link
> >   ...
> >   27: tracing_multi  prog 93
> >           attach_type trace_fentry_multi  obj_id 1  count 3
> >           btf_id           addr             cookie           func [module]
> >           91984            ffffffff824f4a24 a                bpf_fentry_test1
> >           91986            ffffffff824f6a84 1e               bpf_fentry_test2
> >           91987            ffffffff824f6a94 14               bpf_fentry_test3
> >           pids test_progs(1462)
> >
> > Assisted-by: Codex:GPT-5
> > Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> > ---
> >  tools/bpf/bpftool/link.c | 137 +++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 137 insertions(+)
> >
> > diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> > index bdcd717b0348..db780ee39820 100644
> > --- a/tools/bpf/bpftool/link.c
> > +++ b/tools/bpf/bpftool/link.c
> > @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
> >         return (__u64 *) u64_to_ptr(val);
> >  }
> >
> > +static __u32 *u64_to_u32_arr(__u64 val)
> > +{
> > +       return (__u32 *)u64_to_ptr(val);
> > +}
> > +
> > +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
> > +{
> > +       struct kernel_sym *sym;
> > +
> > +       if (!addr)
> > +               return NULL;
> > +
> > +       sym = kernel_syms_search(&dd, addr);
> > +       if (!sym && is_ibt_enabled && addr >= 4)
> > +               sym = kernel_syms_search(&dd, addr - 4);
> 
> if is_ibt_enabled, can we match addr with kernel_syms_search at all?
> I.e., can addr sometimes be resolvable without - 4 adjustment? If not,
> why is this not if/else and instead "try this, if not, try that"?

I followed the symbol_matches_target logic and assumed we could have functions
with and without endbr instruction at the entry.. but I can't find any example
of such traceable function

> 
> > +
> > +       return sym;
> > +}
> > +
> >  static void
> >  show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
> >  {
> > @@ -403,6 +422,47 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
> >         jsonw_end_array(json_wtr);
> >  }
> >
> > +static void
> > +show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
> > +{
> > +       bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
> > +       __u32 i;
> > +
> > +       if (!dd.sym_count)
> > +               kernel_syms_load(&dd);
> > +       show_symbol = !!dd.sym_count;
> > +
> > +       show_link_attach_type_json(info->tracing_multi.attach_type, wtr);
> > +       jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count);
> > +       jsonw_uint_field(wtr, "obj_id", info->tracing_multi.obj_id);
> > +       jsonw_name(wtr, "funcs");
> > +
> > +       jsonw_start_array(wtr);
> > +
> > +       for (i = 0; i < info->tracing_multi.count; i++) {
> > +               __u64 addr = u64_to_arr(info->tracing_multi.addrs)[i];
> > +               struct kernel_sym *sym;
> > +
> > +               sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
> > +
> > +               jsonw_start_object(wtr);
> > +               jsonw_uint_field(wtr, "id", u64_to_u32_arr(info->tracing_multi.ids)[i]);
> 
> nit, here and for cookies, do the cast outside of the loop into local
> variable? seems cleaner as an approach

ok

> 
> > +               jsonw_uint_field(wtr, "addr", addr);
> > +               if (sym) {
> > +                       jsonw_string_field(wtr, "func", sym->name);
> > +                       if (sym->module[0] == '\0') {
> > +                               jsonw_name(wtr, "module");
> > +                               jsonw_null(wtr);
> 
> just curious, is it cleaner to have "module: null" instead of just not
> emitting "module:" field at all?

I followed what we do for kprobe_multi, not sure what's preffered,
but I think it's better to be consistent.. at least somewhere ;-)

> 
> > +                       } else {
> > +                               jsonw_string_field(wtr, "module", sym->module);
> > +                       }
> > +               }
> > +               jsonw_uint_field(wtr, "cookie", u64_to_arr(info->tracing_multi.cookies)[i]);
> > +               jsonw_end_object(wtr);
> > +       }
> > +       jsonw_end_array(wtr);
> > +}
> > +
> >  static void
> >  show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
> >  {
> > @@ -589,6 +649,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
> >         case BPF_LINK_TYPE_UPROBE_MULTI:
> >                 show_uprobe_multi_json(info, json_wtr);
> >                 break;
> > +       case BPF_LINK_TYPE_TRACING_MULTI:
> > +               show_tracing_multi_json(info, json_wtr);
> > +               break;
> >         case BPF_LINK_TYPE_PERF_EVENT:
> >                 switch (info->perf_event.type) {
> >                 case BPF_PERF_EVENT_EVENT:
> > @@ -833,6 +896,42 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
> >         }
> >  }
> >
> > +static void show_tracing_multi_plain(struct bpf_link_info *info)
> > +{
> > +       bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
> > +       __u32 i;
> > +
> > +       if (!info->tracing_multi.count)
> > +               return;
> > +
> > +       if (!dd.sym_count)
> > +               kernel_syms_load(&dd);
> > +       show_symbol = !!dd.sym_count;
> > +
> > +       printf("\n\t");
> > +       show_link_attach_type_plain(info->tracing_multi.attach_type);
> > +       printf("obj_id %u  ", info->tracing_multi.obj_id);
> > +       printf("count %u  ", info->tracing_multi.count);
> > +
> > +       printf("\n\t%-16s %-16s %-16s %s",
> > +              "btf_id", "addr", "cookie", "func [module]");
> > +       for (i = 0; i < info->tracing_multi.count; i++) {
> > +               __u64 addr = u64_to_arr(info->tracing_multi.addrs)[i];
> > +               struct kernel_sym *sym;
> > +
> > +               sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
> > +
> > +               printf("\n\t%-16u %016llx %-16llx",
> > +                      u64_to_u32_arr(info->tracing_multi.ids)[i],
> > +                      addr, u64_to_arr(info->tracing_multi.cookies)[i]);
> 
> As I mentioned, it surprised me that we emit cookie as hex. Kernel
> doesn't in fdinfo output. And at least in my experience cookie is some
> small compact index into some additional lookup array, so decimal
> makes most sense. I see now that we have hex for uprobe_multi, but
> decimal for perf link, so ugh, goodbye consistency. But still, decimal
> will be much more useful in practice, IMO.

agreed

> 
> > +               if (sym) {
> > +                       printf(" %s", sym->name);
> > +                       if (sym->module[0] != '\0')
> > +                               printf(" [%s]", sym->module);
> > +               }
> > +       }
> > +}
> > +
> >  static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
> >  {
> >         const char *buf;
> > @@ -989,6 +1088,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
> >         case BPF_LINK_TYPE_UPROBE_MULTI:
> >                 show_uprobe_multi_plain(info);
> >                 break;
> > +       case BPF_LINK_TYPE_TRACING_MULTI:
> > +               show_tracing_multi_plain(info);
> > +               break;
> >         case BPF_LINK_TYPE_PERF_EVENT:
> >                 switch (info->perf_event.type) {
> >                 case BPF_PERF_EVENT_EVENT:
> > @@ -1029,6 +1131,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
> >  static int do_show_link(int fd)
> >  {
> >         __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL;
> > +       __u32 *ids = NULL;
> >         struct bpf_link_info info;
> >         __u32 len = sizeof(info);
> >         char path_buf[PATH_MAX];
> > @@ -1114,6 +1217,39 @@ static int do_show_link(int fd)
> >                         goto again;
> >                 }
> >         }
> > +       if (info.type == BPF_LINK_TYPE_TRACING_MULTI &&
> > +           !info.tracing_multi.ids) {
> 
> why wrap?

I followed the previous code style

> 
> > +               count = info.tracing_multi.count;
> > +               if (count) {
> > +                       ids = calloc(count, sizeof(__u32));
> > +                       if (!ids) {
> > +                               p_err("mem alloc failed");
> > +                               close(fd);
> > +                               return -ENOMEM;
> > +                       }
> > +                       info.tracing_multi.ids = ptr_to_u64(ids);
> > +
> > +                       addrs = calloc(count, sizeof(__u64));
> > +                       if (!addrs) {
> > +                               p_err("mem alloc failed");
> > +                               free(ids);
> > +                               close(fd);
> > +                               return -ENOMEM;
> > +                       }
> > +                       info.tracing_multi.addrs = ptr_to_u64(addrs);
> > +
> > +                       cookies = calloc(count, sizeof(__u64));
> > +                       if (!cookies) {
> > +                               p_err("mem alloc failed");
> > +                               free(addrs);
> > +                               free(ids);
> > +                               close(fd);
> > +                               return -ENOMEM;
> > +                       }
> 
> ugh... do three calloc()s in a row, check that all succeeded, if not -
> free and close(fd)? shorter and just as good

ah right.. I guess I followed the previous kprobe/uprobe code style,
but as you said, we can simplify the error path, will change

thanks,
jirka

Re: [PATCHv2 bpf-next 0/3] bpf: tracing_multi link info support

[Andrii Nakryiko]

On Tue, Jun 23, 2026 at 7:24 AM Jiri Olsa <jolsa@kernel.org> wrote:
>
> hi,
> adding link info support for tracing_multi link.
>
> Adding the kernel support with related selftests and bpftool support
> to display the tracing_multi link info, the output looks like:
>
>   # bpftool link
>   ...
>   27: tracing_multi  prog 93
>           attach_type trace_fentry_multi  obj_id 1  count 3
>           btf_id           addr             cookie           func [module]
>           91984            ffffffff824f4a24 a                bpf_fentry_test1
>           91986            ffffffff824f6a84 1e               bpf_fentry_test2
>           91987            ffffffff824f6a94 14               bpf_fentry_test3

kernel's bpf_tracing_link_show_fdinfo() emits cookie as decimal
number, why go with hex here?

>           pids test_progs(1462)
>
> thanks,
> jirka
>
>
> v1: https://lore.kernel.org/bpf/20260621204524.61067-1-jolsa@kernel.org/
>
> v2 changes:
> - fix kernel symbol search logic [sashiko]
> - typo fixes [bots]
>
>
> ---
> Jiri Olsa (3):
>       bpf: Add tracing_multi link info support
>       selftests/bpf: Add tracing_multi link info tests
>       bpftool: Add tracing_multi link info output
>
>  include/uapi/linux/bpf.h                                |   9 +++
>  kernel/trace/bpf_trace.c                                |  55 +++++++++++++++++
>  tools/bpf/bpftool/link.c                                | 137 +++++++++++++++++++++++++++++++++++++++++
>  tools/include/uapi/linux/bpf.h                          |   9 +++
>  tools/testing/selftests/bpf/prog_tests/fill_link_info.c | 242 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tools/testing/selftests/bpf/progs/test_fill_link_info.c |   6 ++
>  6 files changed, 458 insertions(+)

Re: [PATCHv2 bpf-next 0/3] bpf: tracing_multi link info support

[Jiri Olsa]

On Tue, Jun 23, 2026 at 01:51:29PM -0700, Andrii Nakryiko wrote:
> On Tue, Jun 23, 2026 at 7:24 AM Jiri Olsa <jolsa@kernel.org> wrote:
> >
> > hi,
> > adding link info support for tracing_multi link.
> >
> > Adding the kernel support with related selftests and bpftool support
> > to display the tracing_multi link info, the output looks like:
> >
> >   # bpftool link
> >   ...
> >   27: tracing_multi  prog 93
> >           attach_type trace_fentry_multi  obj_id 1  count 3
> >           btf_id           addr             cookie           func [module]
> >           91984            ffffffff824f4a24 a                bpf_fentry_test1
> >           91986            ffffffff824f6a84 1e               bpf_fentry_test2
> >           91987            ffffffff824f6a94 14               bpf_fentry_test3
> 
> kernel's bpf_tracing_link_show_fdinfo() emits cookie as decimal
> number, why go with hex here?

true, will switch to dec, thanks

jirka

> 
> >           pids test_progs(1462)
> >
> > thanks,
> > jirka
> >
> >
> > v1: https://lore.kernel.org/bpf/20260621204524.61067-1-jolsa@kernel.org/
> >
> > v2 changes:
> > - fix kernel symbol search logic [sashiko]
> > - typo fixes [bots]
> >
> >
> > ---
> > Jiri Olsa (3):
> >       bpf: Add tracing_multi link info support
> >       selftests/bpf: Add tracing_multi link info tests
> >       bpftool: Add tracing_multi link info output
> >
> >  include/uapi/linux/bpf.h                                |   9 +++
> >  kernel/trace/bpf_trace.c                                |  55 +++++++++++++++++
> >  tools/bpf/bpftool/link.c                                | 137 +++++++++++++++++++++++++++++++++++++++++
> >  tools/include/uapi/linux/bpf.h                          |   9 +++
> >  tools/testing/selftests/bpf/prog_tests/fill_link_info.c | 242 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> >  tools/testing/selftests/bpf/progs/test_fill_link_info.c |   6 ++
> >  6 files changed, 458 insertions(+)