[RFC/PATCH bpf-next 12/20] libbpf: Add uprobe multi link support to bpf_program__attach_usdt

[Jiri Olsa <jolsa@kernel.org>]

Adding uprobe_multi bool to struct bpf_usdt_opts. If it's true
the usdt_manager_attach_usdt will use uprobe_multi link to attach
to usdt probes.

The bpf program for usdt probe needs to have BPF_TRACE_UPROBE_MULTI
set as expected_attach_type.

Because current uprobe is implemented through perf event interface,
it allows the pid filter for uprobes. This is not the case for
uprobe_multi link, so the pid filter is not allowed for that.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 tools/lib/bpf/libbpf.c          |   9 ++-
 tools/lib/bpf/libbpf.h          |   2 +
 tools/lib/bpf/libbpf_internal.h |   2 +-
 tools/lib/bpf/usdt.c            | 127 ++++++++++++++++++++++++--------
 4 files changed, 105 insertions(+), 35 deletions(-)

diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 70353aaac86e..25d32aa605e8 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -6817,7 +6817,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
 	if (!insns || !insns_cnt)
 		return -EINVAL;
 
-	load_attr.expected_attach_type = prog->expected_attach_type;
 	if (kernel_supports(obj, FEAT_PROG_NAME))
 		prog_name = prog->name;
 	load_attr.attach_prog_fd = prog->attach_prog_fd;
@@ -6853,6 +6852,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
 		insns_cnt = prog->insns_cnt;
 	}
 
+	/* allow prog_prepare_load_fn to change expected_attach_type */
+	load_attr.expected_attach_type = prog->expected_attach_type;
+
 	if (obj->gen_loader) {
 		bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
 				   license, insns, insns_cnt, &load_attr,
@@ -11730,6 +11732,7 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
 	struct bpf_object *obj = prog->obj;
 	struct bpf_link *link;
 	__u64 usdt_cookie;
+	bool uprobe_multi;
 	int err;
 
 	if (!OPTS_VALID(opts, bpf_uprobe_opts))
@@ -11766,8 +11769,10 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
 	}
 
 	usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
+	uprobe_multi = OPTS_GET(opts, uprobe_multi, 0);
 	link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
-					usdt_provider, usdt_name, usdt_cookie);
+					usdt_provider, usdt_name, usdt_cookie,
+					uprobe_multi);
 	err = libbpf_get_error(link);
 	if (err)
 		return libbpf_err_ptr(err);
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 921ab2a94cec..025feb21c2ec 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -673,6 +673,8 @@ struct bpf_usdt_opts {
 	size_t sz;
 	/* custom user-provided value accessible through usdt_cookie() */
 	__u64 usdt_cookie;
+	/* use uprobe_multi link */
+	bool uprobe_multi;
 	size_t :0;
 };
 #define bpf_usdt_opts__last_field usdt_cookie
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index e4d05662a96c..5d5f61d0bcfb 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -567,7 +567,7 @@ struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man,
 					   const struct bpf_program *prog,
 					   pid_t pid, const char *path,
 					   const char *usdt_provider, const char *usdt_name,
-					   __u64 usdt_cookie);
+					   __u64 usdt_cookie, bool uprobe_multi);
 
 static inline bool is_pow_of_2(size_t x)
 {
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
index b8402e3f9eb2..f55dbd47d29e 100644
--- a/tools/lib/bpf/usdt.c
+++ b/tools/lib/bpf/usdt.c
@@ -803,11 +803,20 @@ struct bpf_link_usdt {
 	size_t spec_cnt;
 	int *spec_ids;
 
+	bool has_uprobe_multi;
+
 	size_t uprobe_cnt;
 	struct {
 		long abs_ip;
 		struct bpf_link *link;
 	} *uprobes;
+	struct {
+		char **paths;
+		unsigned long *offsets;
+		unsigned long *ref_ctr_offsets;
+		__u64 *cookies;
+		struct bpf_link *link;
+	} uprobe_multi;
 };
 
 static int bpf_link_usdt_detach(struct bpf_link *link)
@@ -816,19 +825,23 @@ static int bpf_link_usdt_detach(struct bpf_link *link)
 	struct usdt_manager *man = usdt_link->usdt_man;
 	int i;
 
-	for (i = 0; i < usdt_link->uprobe_cnt; i++) {
-		/* detach underlying uprobe link */
-		bpf_link__destroy(usdt_link->uprobes[i].link);
-		/* there is no need to update specs map because it will be
-		 * unconditionally overwritten on subsequent USDT attaches,
-		 * but if BPF cookies are not used we need to remove entry
-		 * from ip_to_spec_id map, otherwise we'll run into false
-		 * conflicting IP errors
-		 */
-		if (!man->has_bpf_cookie) {
-			/* not much we can do about errors here */
-			(void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map),
-						  &usdt_link->uprobes[i].abs_ip);
+	if (usdt_link->has_uprobe_multi) {
+		bpf_link__destroy(usdt_link->uprobe_multi.link);
+	} else {
+		for (i = 0; i < usdt_link->uprobe_cnt; i++) {
+			/* detach underlying uprobe link */
+			bpf_link__destroy(usdt_link->uprobes[i].link);
+			/* there is no need to update specs map because it will be
+			 * unconditionally overwritten on subsequent USDT attaches,
+			 * but if BPF cookies are not used we need to remove entry
+			 * from ip_to_spec_id map, otherwise we'll run into false
+			 * conflicting IP errors
+			 */
+			if (!man->has_bpf_cookie) {
+				/* not much we can do about errors here */
+				(void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map),
+							  &usdt_link->uprobes[i].abs_ip);
+			}
 		}
 	}
 
@@ -868,9 +881,16 @@ static void bpf_link_usdt_dealloc(struct bpf_link *link)
 {
 	struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link);
 
-	free(usdt_link->spec_ids);
-	free(usdt_link->uprobes);
-	free(usdt_link);
+	if (usdt_link->has_uprobe_multi) {
+		free(usdt_link->uprobe_multi.paths);
+		free(usdt_link->uprobe_multi.offsets);
+		free(usdt_link->uprobe_multi.ref_ctr_offsets);
+		free(usdt_link->uprobe_multi.cookies);
+	} else {
+		free(usdt_link->spec_ids);
+		free(usdt_link->uprobes);
+		free(usdt_link);
+	}
 }
 
 static size_t specs_hash_fn(long key, void *ctx)
@@ -941,16 +961,22 @@ static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash
 struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog,
 					  pid_t pid, const char *path,
 					  const char *usdt_provider, const char *usdt_name,
-					  __u64 usdt_cookie)
+					  __u64 usdt_cookie, bool uprobe_multi)
 {
+	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi);
 	int i, fd, err, spec_map_fd, ip_map_fd;
 	LIBBPF_OPTS(bpf_uprobe_opts, opts);
 	struct hashmap *specs_hash = NULL;
 	struct bpf_link_usdt *link = NULL;
 	struct usdt_target *targets = NULL;
+	struct bpf_link *uprobe_link;
 	size_t target_cnt;
 	Elf *elf;
 
+	/* The uprobe_multi link does not have pid filter. */
+	if (uprobe_multi && pid >= 0)
+		return libbpf_err_ptr(-EINVAL);
+
 	spec_map_fd = bpf_map__fd(man->specs_map);
 	ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map);
 
@@ -1001,19 +1027,32 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
 		goto err_out;
 	}
 
+	link->has_uprobe_multi = uprobe_multi;
 	link->usdt_man = man;
 	link->link.detach = &bpf_link_usdt_detach;
 	link->link.dealloc = &bpf_link_usdt_dealloc;
 
-	link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
-	if (!link->uprobes) {
-		err = -ENOMEM;
-		goto err_out;
+	if (uprobe_multi) {
+		link->uprobe_multi.paths = calloc(target_cnt, sizeof(*link->uprobe_multi.paths));
+		link->uprobe_multi.offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.offsets));
+		link->uprobe_multi.ref_ctr_offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.ref_ctr_offsets));
+		link->uprobe_multi.cookies = calloc(target_cnt, sizeof(*link->uprobe_multi.cookies));
+
+		if (!link->uprobe_multi.paths || !link->uprobe_multi.offsets ||
+		    !link->uprobe_multi.ref_ctr_offsets || !link->uprobe_multi.cookies) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+	} else {
+		link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
+		if (!link->uprobes) {
+			err = -ENOMEM;
+			goto err_out;
+		}
 	}
 
 	for (i = 0; i < target_cnt; i++) {
 		struct usdt_target *target = &targets[i];
-		struct bpf_link *uprobe_link;
 		bool is_new;
 		int spec_id;
 
@@ -1049,20 +1088,44 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
 			goto err_out;
 		}
 
-		opts.ref_ctr_offset = target->sema_off;
-		opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
-		uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
-							      target->rel_ip, &opts);
+		if (uprobe_multi) {
+			link->uprobe_multi.paths[i] = (char *) path;
+			link->uprobe_multi.offsets[i] = target->rel_ip;
+			link->uprobe_multi.ref_ctr_offsets[i] = target->sema_off;
+			link->uprobe_multi.cookies[i] = spec_id;
+		} else {
+			opts.ref_ctr_offset = target->sema_off;
+			opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
+			uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
+								      target->rel_ip, &opts);
+			err = libbpf_get_error(uprobe_link);
+			if (err) {
+				pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
+					i, usdt_provider, usdt_name, path, err);
+				goto err_out;
+			}
+
+			link->uprobes[i].link = uprobe_link;
+			link->uprobes[i].abs_ip = target->abs_ip;
+			link->uprobe_cnt++;
+		}
+	}
+
+	if (uprobe_multi) {
+		opts_multi.cnt = target_cnt;
+		opts_multi.paths = (const char **) link->uprobe_multi.paths;
+		opts_multi.offsets = link->uprobe_multi.offsets;
+		opts_multi.ref_ctr_offsets = link->uprobe_multi.ref_ctr_offsets;
+		opts_multi.cookies = link->uprobe_multi.cookies;
+
+		uprobe_link = bpf_program__attach_uprobe_multi_opts(prog, NULL, NULL, &opts_multi);
 		err = libbpf_get_error(uprobe_link);
 		if (err) {
-			pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
-				i, usdt_provider, usdt_name, path, err);
+			pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",
+				usdt_provider, usdt_name, path, err);
 			goto err_out;
 		}
-
-		link->uprobes[i].link = uprobe_link;
-		link->uprobes[i].abs_ip = target->abs_ip;
-		link->uprobe_cnt++;
+		link->uprobe_multi.link = uprobe_link;
 	}
 
 	free(targets);
-- 
2.40.0