From: Jiri Olsa <jolsa@kernel.org>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>
Cc: bpf@vger.kernel.org, Martin KaFai Lau <kafai@fb.com>,
Song Liu <songliubraving@fb.com>, Yonghong Song <yhs@fb.com>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@chromium.org>,
Stanislav Fomichev <sdf@google.com>, Hao Luo <haoluo@google.com>
Subject: [RFC PATCH bpf-next 07/17] bpf: Add support to postpone trampoline update
Date: Mon, 8 Aug 2022 16:06:16 +0200 [thread overview]
Message-ID: <20220808140626.422731-8-jolsa@kernel.org> (raw)
In-Reply-To: <20220808140626.422731-1-jolsa@kernel.org>
Adding support to postpone the trampoline update and record
it to the update list. If the update list is provided, all
the reg/unreg/modify functions only add trampoline to the
update list and stores the requested update information/data
to the trampoline.
It will bed used in following changes where we need to do the
actual trampoline update at the end of the attachment.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
include/linux/bpf.h | 31 ++++++++++++-----
kernel/bpf/trampoline.c | 76 +++++++++++++++++++++++++++++------------
2 files changed, 78 insertions(+), 29 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a5738d57f6bd..a23ff5b8d14c 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -768,6 +768,28 @@ struct btf_func_model {
*/
#define BPF_MAX_TRAMP_LINKS 38
+enum bpf_tramp_update_action {
+ BPF_TRAMP_UPDATE_REG,
+ BPF_TRAMP_UPDATE_UNREG,
+ BPF_TRAMP_UPDATE_MODIFY,
+};
+
+enum bpf_tramp_prog_type {
+ BPF_TRAMP_FENTRY,
+ BPF_TRAMP_FEXIT,
+ BPF_TRAMP_MODIFY_RETURN,
+ BPF_TRAMP_MAX,
+ BPF_TRAMP_REPLACE, /* more than MAX */
+};
+
+struct bpf_tramp_update {
+ enum bpf_tramp_update_action action;
+ struct bpf_tramp_image *im;
+ enum bpf_tramp_prog_type kind;
+ struct bpf_prog_array *old_array;
+ struct list_head list;
+};
+
struct bpf_tramp_prog {
struct bpf_prog *prog;
u64 cookie;
@@ -827,14 +849,6 @@ struct bpf_ksym {
bool prog;
};
-enum bpf_tramp_prog_type {
- BPF_TRAMP_FENTRY,
- BPF_TRAMP_FEXIT,
- BPF_TRAMP_MODIFY_RETURN,
- BPF_TRAMP_MAX,
- BPF_TRAMP_REPLACE, /* more than MAX */
-};
-
struct bpf_tramp_image {
void *image;
struct bpf_ksym ksym;
@@ -886,6 +900,7 @@ struct bpf_trampoline {
u64 selector;
struct module *mod;
struct bpf_shim_tramp_link *shim_link;
+ struct bpf_tramp_update update;
};
struct bpf_attach_target_info {
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index d28070247fa3..e926692ded85 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -31,7 +31,8 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
static DEFINE_MUTEX(trampoline_mutex);
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
+ struct list_head *upd);
static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
{
@@ -87,13 +88,13 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd
if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
!(tr->flags & BPF_TRAMP_F_ORIG_STACK))
- ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+ ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, NULL);
break;
case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
- ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+ ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, NULL);
break;
default:
ret = -EINVAL;
@@ -244,12 +245,20 @@ static void bpf_trampoline_module_put(struct bpf_trampoline *tr)
tr->mod = NULL;
}
-static int unregister_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im)
+static int unregister_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
+ struct list_head *upd)
{
void *old_addr = im->image;
void *ip = tr->func.addr;
int ret;
+ if (upd) {
+ tr->update.action = BPF_TRAMP_UPDATE_UNREG;
+ tr->update.im = NULL;
+ list_add_tail(&tr->update.list, upd);
+ return 0;
+ }
+
if (tr->func.ftrace_managed)
ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr);
else
@@ -257,17 +266,24 @@ static int unregister_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *
if (!ret)
bpf_trampoline_module_put(tr);
+
return ret;
}
static int modify_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
- bool lock_direct_mutex)
+ bool lock_direct_mutex, struct list_head *upd)
{
void *old_addr = tr->cur_image->image;
void *new_addr = im->image;
void *ip = tr->func.addr;
int ret;
+ if (upd) {
+ tr->update.action = BPF_TRAMP_UPDATE_MODIFY;
+ tr->update.im = im;
+ list_add_tail(&tr->update.list, upd);
+ return 0;
+ }
if (tr->func.ftrace_managed) {
if (lock_direct_mutex)
ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr);
@@ -280,7 +296,8 @@ static int modify_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
}
/* first time registering */
-static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im)
+static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
+ struct list_head *upd)
{
void *new_addr = im->image;
void *ip = tr->func.addr;
@@ -294,6 +311,15 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im
tr->func.ftrace_managed = true;
}
+ if (upd) {
+ if (ip && !tr->func.ftrace_managed)
+ return -ENOTSUPP;
+ tr->update.action = BPF_TRAMP_UPDATE_REG;
+ tr->update.im = im;
+ list_add_tail(&tr->update.list, upd);
+ return 0;
+ }
+
if (bpf_trampoline_module_get(tr))
return -ENOENT;
@@ -477,7 +503,8 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
return ERR_PTR(err);
}
-static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
+ struct list_head *upd)
{
struct bpf_tramp_image *im;
struct bpf_tramp_progs *tprogs;
@@ -490,10 +517,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
return PTR_ERR(tprogs);
if (total == 0) {
- err = unregister_fentry(tr, tr->cur_image);
- bpf_tramp_image_put(tr->cur_image);
- tr->cur_image = NULL;
- tr->selector = 0;
+ err = unregister_fentry(tr, tr->cur_image, upd);
+ if (!upd) {
+ bpf_tramp_image_put(tr->cur_image);
+ tr->cur_image = NULL;
+ tr->selector = 0;
+ }
goto out;
}
@@ -536,10 +565,10 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
WARN_ON(!tr->cur_image && tr->selector);
if (tr->cur_image)
/* progs already running at this address */
- err = modify_fentry(tr, im, lock_direct_mutex);
+ err = modify_fentry(tr, im, lock_direct_mutex, upd);
else
/* first time registering */
- err = register_fentry(tr, im);
+ err = register_fentry(tr, im, upd);
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
if (err == -EAGAIN) {
@@ -553,7 +582,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
goto again;
}
#endif
- if (err)
+ if (err || upd)
goto out;
if (tr->cur_image)
@@ -592,7 +621,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
}
}
-static int __bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr)
+static int __bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr,
+ struct list_head *upd)
{
struct bpf_prog_array *old_array, *new_array;
const struct bpf_prog_array_item *item;
@@ -638,11 +668,14 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_tram
return -ENOMEM;
tr->progs_array[kind] = new_array;
tr->progs_cnt[kind]++;
- err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
+ err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, upd);
if (err) {
tr->progs_array[kind] = old_array;
tr->progs_cnt[kind]--;
bpf_prog_array_free(new_array);
+ } else if (upd) {
+ tr->update.kind = kind;
+ tr->update.old_array = old_array;
} else {
bpf_prog_array_free(old_array);
}
@@ -654,12 +687,13 @@ int bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *t
int err;
mutex_lock(&tr->mutex);
- err = __bpf_trampoline_link_prog(tp, tr);
+ err = __bpf_trampoline_link_prog(tp, tr, NULL);
mutex_unlock(&tr->mutex);
return err;
}
-static int __bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr)
+static int __bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr,
+ struct list_head *upd)
{
struct bpf_prog_array *old_array, *new_array;
enum bpf_tramp_prog_type kind;
@@ -683,7 +717,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_tr
tr->progs_cnt[kind]--;
tr->progs_array[kind] = new_array;
bpf_prog_array_free(old_array);
- return bpf_trampoline_update(tr, true /* lock_direct_mutex */);
+ return bpf_trampoline_update(tr, true /* lock_direct_mutex */, upd);
}
/* bpf_trampoline_unlink_prog() should never fail. */
@@ -692,7 +726,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline
int err;
mutex_lock(&tr->mutex);
- err = __bpf_trampoline_unlink_prog(tp, tr);
+ err = __bpf_trampoline_unlink_prog(tp, tr, NULL);
mutex_unlock(&tr->mutex);
return err;
}
@@ -804,7 +838,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
goto err;
}
- err = __bpf_trampoline_link_prog(&shim_link->tp, tr);
+ err = __bpf_trampoline_link_prog(&shim_link->tp, tr, NULL);
if (err)
goto err;
--
2.37.1
next prev parent reply other threads:[~2022-08-08 14:07 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-08-08 14:06 [RFC PATCH bpf-next 00/17] bpf: Add tracing multi link Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 01/17] bpf: Link shimlink directly in trampoline Jiri Olsa
2022-08-08 17:40 ` Song Liu
2022-08-08 17:58 ` Stanislav Fomichev
2022-08-09 15:36 ` Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 02/17] bpf: Replace bpf_tramp_links with bpf_tramp_progs Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 03/17] bpf: Store trampoline progs in arrays Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 04/17] bpf: Add multi tracing attach types Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 05/17] bpf: Add bpf_tramp_id object Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 06/17] bpf: Pass image struct to reg/unreg/modify fentry functions Jiri Olsa
2022-08-08 14:06 ` Jiri Olsa [this message]
2022-08-08 14:06 ` [RFC PATCH bpf-next 08/17] bpf: Factor bpf_trampoline_lookup function Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 09/17] bpf: Factor bpf_trampoline_put function Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 10/17] bpf: Add support to attach program to multiple trampolines Jiri Olsa
2022-08-24 1:22 ` Alexei Starovoitov
2022-08-25 16:08 ` Jiri Olsa
2022-08-25 17:43 ` Alexei Starovoitov
2022-08-26 2:35 ` Andrii Nakryiko
2022-08-26 14:20 ` Jiri Olsa
2022-08-27 5:15 ` Andrii Nakryiko
2022-08-27 12:16 ` Jiri Olsa
2022-08-26 4:37 ` Song Liu
2022-08-08 14:06 ` [RFC PATCH bpf-next 11/17] bpf: Add support to create tracing multi link Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 12/17] libbpf: Add btf__find_by_glob_kind function Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 13/17] libbpf: Add support to create tracing multi link Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 14/17] selftests/bpf: Add fentry tracing multi func test Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 15/17] selftests/bpf: Add fexit " Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 16/17] selftests/bpf: Add fentry/fexit " Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 17/17] selftests/bpf: Add mixed " Jiri Olsa
2022-08-08 17:50 ` [RFC PATCH bpf-next 00/17] bpf: Add tracing multi link Song Liu
2022-08-08 20:35 ` Jiri Olsa
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220808140626.422731-8-jolsa@kernel.org \
--to=jolsa@kernel.org \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=haoluo@google.com \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@chromium.org \
--cc=sdf@google.com \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox