From: Jiri Olsa <jolsa@kernel.org>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>
Cc: bpf@vger.kernel.org, Martin KaFai Lau <kafai@fb.com>,
Song Liu <songliubraving@fb.com>, Yonghong Song <yhs@fb.com>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@chromium.org>,
Stanislav Fomichev <sdf@google.com>, Hao Luo <haoluo@google.com>
Subject: [RFC PATCH bpf-next 07/17] bpf: Add support to postpone trampoline update
Date: Mon, 8 Aug 2022 16:06:16 +0200 [thread overview]
Message-ID: <20220808140626.422731-8-jolsa@kernel.org> (raw)
In-Reply-To: <20220808140626.422731-1-jolsa@kernel.org>
Adding support to postpone the trampoline update and record
it to the update list. If the update list is provided, all
the reg/unreg/modify functions only add trampoline to the
update list and stores the requested update information/data
to the trampoline.
It will bed used in following changes where we need to do the
actual trampoline update at the end of the attachment.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
include/linux/bpf.h | 31 ++++++++++++-----
kernel/bpf/trampoline.c | 76 +++++++++++++++++++++++++++++------------
2 files changed, 78 insertions(+), 29 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a5738d57f6bd..a23ff5b8d14c 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -768,6 +768,28 @@ struct btf_func_model {
*/
#define BPF_MAX_TRAMP_LINKS 38
+enum bpf_tramp_update_action {
+ BPF_TRAMP_UPDATE_REG,
+ BPF_TRAMP_UPDATE_UNREG,
+ BPF_TRAMP_UPDATE_MODIFY,
+};
+
+enum bpf_tramp_prog_type {
+ BPF_TRAMP_FENTRY,
+ BPF_TRAMP_FEXIT,
+ BPF_TRAMP_MODIFY_RETURN,
+ BPF_TRAMP_MAX,
+ BPF_TRAMP_REPLACE, /* more than MAX */
+};
+
+struct bpf_tramp_update {
+ enum bpf_tramp_update_action action;
+ struct bpf_tramp_image *im;
+ enum bpf_tramp_prog_type kind;
+ struct bpf_prog_array *old_array;
+ struct list_head list;
+};
+
struct bpf_tramp_prog {
struct bpf_prog *prog;
u64 cookie;
@@ -827,14 +849,6 @@ struct bpf_ksym {
bool prog;
};
-enum bpf_tramp_prog_type {
- BPF_TRAMP_FENTRY,
- BPF_TRAMP_FEXIT,
- BPF_TRAMP_MODIFY_RETURN,
- BPF_TRAMP_MAX,
- BPF_TRAMP_REPLACE, /* more than MAX */
-};
-
struct bpf_tramp_image {
void *image;
struct bpf_ksym ksym;
@@ -886,6 +900,7 @@ struct bpf_trampoline {
u64 selector;
struct module *mod;
struct bpf_shim_tramp_link *shim_link;
+ struct bpf_tramp_update update;
};
struct bpf_attach_target_info {
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index d28070247fa3..e926692ded85 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -31,7 +31,8 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
static DEFINE_MUTEX(trampoline_mutex);
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
+ struct list_head *upd);
static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
{
@@ -87,13 +88,13 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd
if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
!(tr->flags & BPF_TRAMP_F_ORIG_STACK))
- ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+ ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, NULL);
break;
case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
- ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+ ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, NULL);
break;
default:
ret = -EINVAL;
@@ -244,12 +245,20 @@ static void bpf_trampoline_module_put(struct bpf_trampoline *tr)
tr->mod = NULL;
}
-static int unregister_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im)
+static int unregister_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
+ struct list_head *upd)
{
void *old_addr = im->image;
void *ip = tr->func.addr;
int ret;
+ if (upd) {
+ tr->update.action = BPF_TRAMP_UPDATE_UNREG;
+ tr->update.im = NULL;
+ list_add_tail(&tr->update.list, upd);
+ return 0;
+ }
+
if (tr->func.ftrace_managed)
ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr);
else
@@ -257,17 +266,24 @@ static int unregister_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *
if (!ret)
bpf_trampoline_module_put(tr);
+
return ret;
}
static int modify_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
- bool lock_direct_mutex)
+ bool lock_direct_mutex, struct list_head *upd)
{
void *old_addr = tr->cur_image->image;
void *new_addr = im->image;
void *ip = tr->func.addr;
int ret;
+ if (upd) {
+ tr->update.action = BPF_TRAMP_UPDATE_MODIFY;
+ tr->update.im = im;
+ list_add_tail(&tr->update.list, upd);
+ return 0;
+ }
if (tr->func.ftrace_managed) {
if (lock_direct_mutex)
ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr);
@@ -280,7 +296,8 @@ static int modify_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
}
/* first time registering */
-static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im)
+static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
+ struct list_head *upd)
{
void *new_addr = im->image;
void *ip = tr->func.addr;
@@ -294,6 +311,15 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im
tr->func.ftrace_managed = true;
}
+ if (upd) {
+ if (ip && !tr->func.ftrace_managed)
+ return -ENOTSUPP;
+ tr->update.action = BPF_TRAMP_UPDATE_REG;
+ tr->update.im = im;
+ list_add_tail(&tr->update.list, upd);
+ return 0;
+ }
+
if (bpf_trampoline_module_get(tr))
return -ENOENT;
@@ -477,7 +503,8 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
return ERR_PTR(err);
}
-static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
+ struct list_head *upd)
{
struct bpf_tramp_image *im;
struct bpf_tramp_progs *tprogs;
@@ -490,10 +517,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
return PTR_ERR(tprogs);
if (total == 0) {
- err = unregister_fentry(tr, tr->cur_image);
- bpf_tramp_image_put(tr->cur_image);
- tr->cur_image = NULL;
- tr->selector = 0;
+ err = unregister_fentry(tr, tr->cur_image, upd);
+ if (!upd) {
+ bpf_tramp_image_put(tr->cur_image);
+ tr->cur_image = NULL;
+ tr->selector = 0;
+ }
goto out;
}
@@ -536,10 +565,10 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
WARN_ON(!tr->cur_image && tr->selector);
if (tr->cur_image)
/* progs already running at this address */
- err = modify_fentry(tr, im, lock_direct_mutex);
+ err = modify_fentry(tr, im, lock_direct_mutex, upd);
else
/* first time registering */
- err = register_fentry(tr, im);
+ err = register_fentry(tr, im, upd);
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
if (err == -EAGAIN) {
@@ -553,7 +582,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
goto again;
}
#endif
- if (err)
+ if (err || upd)
goto out;
if (tr->cur_image)
@@ -592,7 +621,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
}
}
-static int __bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr)
+static int __bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr,
+ struct list_head *upd)
{
struct bpf_prog_array *old_array, *new_array;
const struct bpf_prog_array_item *item;
@@ -638,11 +668,14 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_tram
return -ENOMEM;
tr->progs_array[kind] = new_array;
tr->progs_cnt[kind]++;
- err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
+ err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, upd);
if (err) {
tr->progs_array[kind] = old_array;
tr->progs_cnt[kind]--;
bpf_prog_array_free(new_array);
+ } else if (upd) {
+ tr->update.kind = kind;
+ tr->update.old_array = old_array;
} else {
bpf_prog_array_free(old_array);
}
@@ -654,12 +687,13 @@ int bpf_trampoline_link_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *t
int err;
mutex_lock(&tr->mutex);
- err = __bpf_trampoline_link_prog(tp, tr);
+ err = __bpf_trampoline_link_prog(tp, tr, NULL);
mutex_unlock(&tr->mutex);
return err;
}
-static int __bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr)
+static int __bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline *tr,
+ struct list_head *upd)
{
struct bpf_prog_array *old_array, *new_array;
enum bpf_tramp_prog_type kind;
@@ -683,7 +717,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_tr
tr->progs_cnt[kind]--;
tr->progs_array[kind] = new_array;
bpf_prog_array_free(old_array);
- return bpf_trampoline_update(tr, true /* lock_direct_mutex */);
+ return bpf_trampoline_update(tr, true /* lock_direct_mutex */, upd);
}
/* bpf_trampoline_unlink_prog() should never fail. */
@@ -692,7 +726,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_prog *tp, struct bpf_trampoline
int err;
mutex_lock(&tr->mutex);
- err = __bpf_trampoline_unlink_prog(tp, tr);
+ err = __bpf_trampoline_unlink_prog(tp, tr, NULL);
mutex_unlock(&tr->mutex);
return err;
}
@@ -804,7 +838,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
goto err;
}
- err = __bpf_trampoline_link_prog(&shim_link->tp, tr);
+ err = __bpf_trampoline_link_prog(&shim_link->tp, tr, NULL);
if (err)
goto err;
--
2.37.1
next prev parent reply other threads:[~2022-08-08 14:07 UTC|newest]
Thread overview: 33+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-08-08 14:06 [RFC PATCH bpf-next 00/17] bpf: Add tracing multi link Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 01/17] bpf: Link shimlink directly in trampoline Jiri Olsa
2022-08-08 17:40 ` Song Liu
2022-08-08 17:58 ` Stanislav Fomichev
2022-08-09 15:36 ` Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 02/17] bpf: Replace bpf_tramp_links with bpf_tramp_progs Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 03/17] bpf: Store trampoline progs in arrays Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 04/17] bpf: Add multi tracing attach types Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 05/17] bpf: Add bpf_tramp_id object Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 06/17] bpf: Pass image struct to reg/unreg/modify fentry functions Jiri Olsa
2022-08-08 14:06 ` Jiri Olsa [this message]
2022-08-08 14:06 ` [RFC PATCH bpf-next 08/17] bpf: Factor bpf_trampoline_lookup function Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 09/17] bpf: Factor bpf_trampoline_put function Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 10/17] bpf: Add support to attach program to multiple trampolines Jiri Olsa
2022-08-24 1:22 ` Alexei Starovoitov
2022-08-25 16:08 ` Jiri Olsa
2022-08-25 17:43 ` Alexei Starovoitov
2022-08-26 2:35 ` Andrii Nakryiko
2022-08-26 14:20 ` Jiri Olsa
2022-08-27 5:15 ` Andrii Nakryiko
2022-08-27 12:16 ` Jiri Olsa
2022-08-26 4:37 ` Song Liu
2022-08-08 14:06 ` [RFC PATCH bpf-next 11/17] bpf: Add support to create tracing multi link Jiri Olsa
2022-08-09 1:18 ` kernel test robot
2022-08-09 2:50 ` kernel test robot
2022-08-08 14:06 ` [RFC PATCH bpf-next 12/17] libbpf: Add btf__find_by_glob_kind function Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 13/17] libbpf: Add support to create tracing multi link Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 14/17] selftests/bpf: Add fentry tracing multi func test Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 15/17] selftests/bpf: Add fexit " Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 16/17] selftests/bpf: Add fentry/fexit " Jiri Olsa
2022-08-08 14:06 ` [RFC PATCH bpf-next 17/17] selftests/bpf: Add mixed " Jiri Olsa
2022-08-08 17:50 ` [RFC PATCH bpf-next 00/17] bpf: Add tracing multi link Song Liu
2022-08-08 20:35 ` Jiri Olsa
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220808140626.422731-8-jolsa@kernel.org \
--to=jolsa@kernel.org \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=haoluo@google.com \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@chromium.org \
--cc=sdf@google.com \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.