From: Jiri Olsa <jolsa@redhat.com>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>
Cc: netdev@vger.kernel.org, bpf@vger.kernel.org,
Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>,
Yonghong Song <yhs@fb.com>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@chromium.org>
Subject: [PATCH bpf-next 10/29] bpf: Add bpf_trampoline_id object
Date: Thu, 18 Nov 2021 12:24:36 +0100 [thread overview]
Message-ID: <20211118112455.475349-11-jolsa@kernel.org> (raw)
In-Reply-To: <20211118112455.475349-1-jolsa@kernel.org>
Replacing the bpf_trampoline's key with struct bpf_tramp_id object,
that currently holds only obj_id/btf_id, so same data as key.
Having the key in the struct will allow us to add more ids (functions)
to single trampoline in following patches.
No functional change is intended.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
include/linux/bpf.h | 18 +++++++++--
include/linux/bpf_verifier.h | 19 ------------
kernel/bpf/syscall.c | 30 +++++++++++++-----
kernel/bpf/trampoline.c | 59 +++++++++++++++++++++++++++++++-----
kernel/bpf/verifier.c | 14 ++++++---
5 files changed, 99 insertions(+), 41 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 35f484f323f3..2ce8b1c49af7 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -671,13 +671,18 @@ struct bpf_tramp_image {
};
};
+struct bpf_tramp_id {
+ u32 obj_id;
+ u32 btf_id;
+};
+
struct bpf_trampoline {
/* hlist for trampoline_table */
struct hlist_node hlist;
/* serializes access to fields of this trampoline */
struct mutex mutex;
refcount_t refcnt;
- u64 key;
+ struct bpf_tramp_id *id;
struct {
struct btf_func_model model;
void *addr;
@@ -732,9 +737,16 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
return bpf_func(ctx, insnsi);
}
#ifdef CONFIG_BPF_JIT
+struct bpf_tramp_id *bpf_tramp_id_alloc(void);
+void bpf_tramp_id_free(struct bpf_tramp_id *id);
+bool bpf_tramp_id_is_empty(struct bpf_tramp_id *id);
+int bpf_tramp_id_is_equal(struct bpf_tramp_id *a, struct bpf_tramp_id *b);
+void bpf_tramp_id_init(struct bpf_tramp_id *id,
+ const struct bpf_prog *tgt_prog,
+ struct btf *btf, u32 btf_id);
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-struct bpf_trampoline *bpf_trampoline_get(u64 key,
+struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);
#define BPF_DISPATCHER_INIT(_name) { \
@@ -792,7 +804,7 @@ static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
{
return -ENOTSUPP;
}
-static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
+static inline struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info)
{
return ERR_PTR(-EOPNOTSUPP);
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b561c0b08e68..0c01a50180a0 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -503,25 +503,6 @@ int check_ctx_reg(struct bpf_verifier_env *env,
int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
u32 regno, u32 mem_size);
-/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
-static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
- struct btf *btf, u32 btf_id)
-{
- if (tgt_prog)
- return ((u64)tgt_prog->aux->id << 32) | btf_id;
- else
- return ((u64)btf_obj_id(btf) << 32) | 0x80000000 | btf_id;
-}
-
-/* unpack the IDs from the key as constructed above */
-static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id)
-{
- if (obj_id)
- *obj_id = key >> 32;
- if (btf_id)
- *btf_id = key & 0x7FFFFFFF;
-}
-
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 648155f0a4b1..f99ea3237f9c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2703,9 +2703,8 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
struct bpf_trampoline *tr = link_trampoline(tr_link);
info->tracing.attach_type = tr_link->attach_type;
- bpf_trampoline_unpack_key(tr->key,
- &info->tracing.target_obj_id,
- &info->tracing.target_btf_id);
+ info->tracing.target_obj_id = tr->id->obj_id;
+ info->tracing.target_btf_id = tr->id->btf_id;
return 0;
}
@@ -2726,7 +2725,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
struct bpf_prog *tgt_prog = NULL;
struct bpf_trampoline *tr = NULL;
struct bpf_tracing_link *link;
- u64 key = 0;
+ struct bpf_tramp_id *id = NULL;
int err;
switch (prog->type) {
@@ -2767,6 +2766,12 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
goto out_put_prog;
}
+ id = bpf_tramp_id_alloc();
+ if (!id) {
+ err = -ENOMEM;
+ goto out_put_prog;
+ }
+
tgt_prog = bpf_prog_get(tgt_prog_fd);
if (IS_ERR(tgt_prog)) {
err = PTR_ERR(tgt_prog);
@@ -2774,7 +2779,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
goto out_put_prog;
}
- key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
+ bpf_tramp_id_init(id, tgt_prog, NULL, btf_id);
}
link = kzalloc(sizeof(*link), GFP_USER);
@@ -2823,12 +2828,20 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
err = -EINVAL;
goto out_unlock;
}
+
+ id = bpf_tramp_id_alloc();
+ if (!id) {
+ err = -ENOMEM;
+ goto out_unlock;
+ }
+
btf_id = prog->aux->attach_btf_id;
- key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
+ bpf_tramp_id_init(id, NULL, prog->aux->attach_btf, btf_id);
}
if (!prog->aux->dst_trampoline ||
- (key && key != prog->aux->dst_trampoline->key)) {
+ (!bpf_tramp_id_is_empty(id) &&
+ bpf_tramp_id_is_equal(id, prog->aux->dst_trampoline->id))) {
/* If there is no saved target, or the specified target is
* different from the destination specified at load time, we
* need a new trampoline and a check for compatibility
@@ -2840,7 +2853,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
if (err)
goto out_unlock;
- tr = bpf_trampoline_get(key, &tgt_info);
+ tr = bpf_trampoline_get(id, &tgt_info);
if (!tr) {
err = -ENOMEM;
goto out_unlock;
@@ -2900,6 +2913,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
out_put_prog:
if (tgt_prog_fd && tgt_prog)
bpf_prog_put(tgt_prog);
+ bpf_tramp_id_free(id);
return err;
}
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index e98de5e73ba5..ae2573c36653 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -59,16 +59,57 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym)
PAGE_SIZE, true, ksym->name);
}
-static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
+static u64 bpf_tramp_id_key(struct bpf_tramp_id *id)
+{
+ return ((u64) id->obj_id << 32) | id->btf_id;
+}
+
+bool bpf_tramp_id_is_empty(struct bpf_tramp_id *id)
+{
+ return !id || (!id->obj_id && !id->btf_id);
+}
+
+int bpf_tramp_id_is_equal(struct bpf_tramp_id *a,
+ struct bpf_tramp_id *b)
+{
+ return !memcmp(a, b, sizeof(*a));
+}
+
+struct bpf_tramp_id *bpf_tramp_id_alloc(void)
+{
+ struct bpf_tramp_id *id;
+
+ return kzalloc(sizeof(*id), GFP_KERNEL);
+}
+
+void bpf_tramp_id_init(struct bpf_tramp_id *id,
+ const struct bpf_prog *tgt_prog,
+ struct btf *btf, u32 btf_id)
+{
+ if (tgt_prog)
+ id->obj_id = tgt_prog->aux->id;
+ else
+ id->obj_id = btf_obj_id(btf);
+ id->btf_id = btf_id;
+}
+
+void bpf_tramp_id_free(struct bpf_tramp_id *id)
+{
+ kfree(id);
+}
+
+static struct bpf_trampoline *bpf_trampoline_lookup(struct bpf_tramp_id *id)
{
struct bpf_trampoline *tr;
struct hlist_head *head;
+ u64 key;
int i;
+ key = bpf_tramp_id_key(id);
mutex_lock(&trampoline_mutex);
head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
hlist_for_each_entry(tr, head, hlist) {
- if (tr->key == key) {
+ if (bpf_tramp_id_is_equal(tr->id, id)) {
refcount_inc(&tr->refcnt);
goto out;
}
@@ -77,7 +118,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
if (!tr)
goto out;
- tr->key = key;
+ tr->id = id;
INIT_HLIST_NODE(&tr->hlist);
hlist_add_head(&tr->hlist, head);
refcount_set(&tr->refcnt, 1);
@@ -291,12 +332,14 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
}
-static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
+static struct bpf_tramp_image*
+bpf_tramp_image_alloc(struct bpf_tramp_id *id, u32 idx)
{
struct bpf_tramp_image *im;
struct bpf_ksym *ksym;
void *image;
int err = -ENOMEM;
+ u64 key;
im = kzalloc(sizeof(*im), GFP_KERNEL);
if (!im)
@@ -317,6 +360,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
ksym = &im->ksym;
INIT_LIST_HEAD_RCU(&ksym->lnode);
+ key = bpf_tramp_id_key(id);
snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu_%u", key, idx);
bpf_image_ksym_add(image, ksym);
return im;
@@ -351,7 +395,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
goto out;
}
- im = bpf_tramp_image_alloc(tr->key, tr->selector);
+ im = bpf_tramp_image_alloc(tr->id, tr->selector);
if (IS_ERR(im)) {
err = PTR_ERR(im);
goto out;
@@ -482,12 +526,12 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
return err;
}
-struct bpf_trampoline *bpf_trampoline_get(u64 key,
+struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info)
{
struct bpf_trampoline *tr;
- tr = bpf_trampoline_lookup(key);
+ tr = bpf_trampoline_lookup(id);
if (!tr)
return NULL;
@@ -521,6 +565,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
* multiple rcu callbacks.
*/
hlist_del(&tr->hlist);
+ bpf_tramp_id_free(tr->id);
kfree(tr);
out:
mutex_unlock(&trampoline_mutex);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index af6a39bbb0dc..a1e4389b0e9e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -13929,8 +13929,8 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
struct bpf_attach_target_info tgt_info = {};
u32 btf_id = prog->aux->attach_btf_id;
struct bpf_trampoline *tr;
+ struct bpf_tramp_id *id;
int ret;
- u64 key;
if (prog->type == BPF_PROG_TYPE_SYSCALL) {
if (prog->aux->sleepable)
@@ -13995,11 +13995,17 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
return -EINVAL;
}
- key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
- tr = bpf_trampoline_get(key, &tgt_info);
- if (!tr)
+ id = bpf_tramp_id_alloc();
+ if (!id)
return -ENOMEM;
+ bpf_tramp_id_init(id, tgt_prog, prog->aux->attach_btf, btf_id);
+ tr = bpf_trampoline_get(id, &tgt_info);
+ if (!tr) {
+ bpf_tramp_id_free(id);
+ return -ENOMEM;
+ }
+
prog->aux->dst_trampoline = tr;
return 0;
}
--
2.31.1
next prev parent reply other threads:[~2021-11-18 11:27 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-18 11:24 [RFC bpf-next v5 00/29] bpf: Add batch support for attaching trampolines Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 01/29] ftrace: Use direct_ops hash in unregister_ftrace_direct Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 02/29] ftrace: Add cleanup to unregister_ftrace_direct_multi Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 03/29] ftrace: Add ftrace_set_filter_ips function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 04/29] bpf: Factor bpf_check_attach_target function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 05/29] bpf: Add bpf_check_attach_model function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 06/29] bpf: Add bpf_arg/bpf_ret_value helpers for tracing programs Jiri Olsa
2021-11-24 21:43 ` Andrii Nakryiko
2021-11-25 16:14 ` Alexei Starovoitov
2021-11-28 18:07 ` Jiri Olsa
2021-11-28 18:06 ` Jiri Olsa
2021-12-01 7:13 ` Andrii Nakryiko
2021-12-01 17:37 ` Alexei Starovoitov
2021-12-01 17:59 ` Andrii Nakryiko
2021-12-01 20:36 ` Alexei Starovoitov
2021-12-01 21:16 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 07/29] bpf, x64: Allow to use caller address from stack Jiri Olsa
2021-11-19 4:14 ` Alexei Starovoitov
2021-11-19 21:46 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 08/29] bpf: Keep active attached trampoline in bpf_prog Jiri Olsa
2021-11-24 21:48 ` Andrii Nakryiko
2021-11-28 17:24 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 09/29] bpf: Add support to load multi func tracing program Jiri Olsa
2021-11-19 4:11 ` Alexei Starovoitov
2021-11-22 20:15 ` Jiri Olsa
2021-11-24 21:51 ` Andrii Nakryiko
2021-11-28 17:41 ` Jiri Olsa
2021-12-01 7:17 ` Andrii Nakryiko
2021-12-01 21:20 ` Jiri Olsa
2021-11-18 11:24 ` Jiri Olsa [this message]
2021-11-18 11:24 ` [PATCH bpf-next 11/29] bpf: Add addr to bpf_trampoline_id object Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 12/29] bpf: Add struct bpf_tramp_node layer Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 13/29] bpf: Add bpf_tramp_attach layer for trampoline attachment Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 14/29] bpf: Add support to store multiple ids in bpf_tramp_id object Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 15/29] bpf: Add support to store multiple addrs " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 16/29] bpf: Add bpf_tramp_id_single function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 17/29] bpf: Resolve id in bpf_tramp_id_single Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 18/29] bpf: Add refcount_t to struct bpf_tramp_id Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 19/29] bpf: Add support to attach trampolines with multiple IDs Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 20/29] bpf: Add support for tracing multi link Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 21/29] libbpf: Add btf__find_by_glob_kind function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 22/29] libbpf: Add support to link multi func tracing program Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 23/29] selftests/bpf: Add bpf_arg/bpf_ret_value test Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 24/29] selftests/bpf: Add fentry multi func test Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 25/29] selftests/bpf: Add fexit " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 26/29] selftests/bpf: Add fentry/fexit " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 27/29] selftests/bpf: Add mixed " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 28/29] selftests/bpf: Add ret_mod " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 29/29] selftests/bpf: Add attach " Jiri Olsa
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211118112455.475349-11-jolsa@kernel.org \
--to=jolsa@redhat.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@chromium.org \
--cc=netdev@vger.kernel.org \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.