From: Yonghong Song <yonghong.song@linux.dev>
To: bpf@vger.kernel.org
Cc: Alexei Starovoitov <ast@kernel.org>,
Andrii Nakryiko <andrii@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Eduard Zingerman <eddyz87@gmail.com>,
kernel-team@fb.com
Subject: [PATCH bpf-next v5 05/21] bpf: Prepare for an exception cleanup table before the CFG walk
Date: Tue, 22 Sep 2026 21:59:12 -0700 [thread overview]
Message-ID: <20260923045912.2417059-1-yonghong.song@linux.dev> (raw)
In-Reply-To: <20260923045846.2414643-1-yonghong.song@linux.dev>
Record in insn_aux_data what the later passes need from the cleanup table:
throw_call for every bpf_throw() and resume_call for every
bpf_unwind_resume() -- the two calls a JIT lowers its own way rather than
as calls -- and cleanup_pad, the landing pad a frame resumes at, for every
call within the [begin_off, end_off) range of a cleanup record. Subsequent
commits consume all three.
Marking the two calls here, rather than recognising them in the JIT, is
what makes the recognition exact: by the time a JIT runs,
bpf_fixup_kfunc_call() has rewritten every other kfunc's imm into an offset
from __bpf_call_base, and a BTF id compared against one of those offsets
could match an unrelated call.
bpf_prepare_cleanup_exceptions() runs before bpf_check_cfg(), whose walk
consumes what it produces. It refuses a table on an offloaded program, on
one whose JIT cannot dispatch landing pads or was not asked to compile it,
and on one that also installs an exception callback -- two different
answers to what runs on the way out. What survives is marked jit_required:
the interpreter cannot dispatch a pad. bpf_jit_supports_cleanup_pads() is
weak here and says no; the arch patches provide the real ones.
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
include/linux/bpf_verifier.h | 2 ++
include/linux/filter.h | 1 +
kernel/bpf/core.c | 5 +++
kernel/bpf/exception.c | 62 ++++++++++++++++++++++++++++++++++++
kernel/bpf/exception.h | 1 +
kernel/bpf/fixups.c | 6 ++++
kernel/bpf/verifier.c | 6 ++++
7 files changed, 83 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 666c73308b32..e12ec91b8150 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -692,6 +692,8 @@ struct bpf_insn_aux_data {
*/
u64 fastcall_spills_num:3;
u64 arg_prog:4;
+ u64 throw_call:1; /* call to bpf_throw() */
+ u64 resume_call:1; /* call to bpf_unwind_resume() */
/* below flags are initialized once */
u64 jmp_point:1;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 422284b4fa96..2582a7606e46 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1242,6 +1242,7 @@ bool bpf_jit_supports_stack_args(void);
bool bpf_jit_supports_arena_args(void);
bool bpf_jit_supports_far_kfunc_call(void);
bool bpf_jit_supports_exceptions(void);
+bool bpf_jit_supports_cleanup_pads(void);
bool bpf_jit_supports_ptr_xchg(void);
bool bpf_jit_supports_arena(void);
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 227211166dcc..a379cd1ec4c6 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3475,6 +3475,11 @@ void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp,
{
}
+bool __weak bpf_jit_supports_cleanup_pads(void)
+{
+ return false;
+}
+
bool __weak bpf_jit_supports_timed_may_goto(void)
{
return false;
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
index 6989f31aa6c9..244ea1b5417c 100644
--- a/kernel/bpf/exception.c
+++ b/kernel/bpf/exception.c
@@ -7,8 +7,70 @@
#include <linux/filter.h>
#include "exception.h"
+#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
+
BTF_ID_LIST_SINGLE(bpf_unwind_resume_id, func, bpf_unwind_resume)
+static void mark_kfunc_sites(struct bpf_verifier_env *env)
+{
+ u32 i;
+
+ for (i = 0; i < env->prog->len; i++) {
+ struct bpf_insn *insn = &env->prog->insnsi[i];
+
+ if (bpf_is_throw_kfunc(insn))
+ env->insn_aux_data[i].throw_call = true;
+ else if (bpf_is_unwind_resume_kfunc(insn))
+ env->insn_aux_data[i].resume_call = true;
+ }
+}
+
+static void mark_call_sites(struct bpf_verifier_env *env)
+{
+ u32 i, j;
+
+ for (i = 0; i < env->cleanup_info_cnt; i++) {
+ struct bpf_cleanup_info *rec = &env->cleanup_info[i];
+
+ for (j = rec->begin_off; j < rec->end_off; j++) {
+ struct bpf_insn *insn = &env->prog->insnsi[j];
+
+ if (!bpf_pseudo_call(insn) && !bpf_is_throw_kfunc(insn))
+ continue;
+ env->insn_aux_data[j].cleanup_pad = rec->landing_pad_off + 1;
+ }
+ }
+}
+
+int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env)
+{
+ if (!env->cleanup_info_cnt)
+ return 0;
+
+ if (bpf_prog_is_offloaded(env->prog->aux)) {
+ verbose(env,
+ "exception cleanup is not supported for offloaded programs\n");
+ return -EINVAL;
+ }
+
+ if (!bpf_jit_supports_cleanup_pads() || !env->prog->jit_requested) {
+ verbose(env,
+ "exception cleanup needs a JIT that can dispatch landing pads\n");
+ return -EOPNOTSUPP;
+ }
+ env->prog->jit_required = 1;
+
+ if (env->exception_callback_subprog) {
+ verbose(env,
+ "exception cleanup table cannot be combined with an exception callback\n");
+ return -EINVAL;
+ }
+
+ mark_kfunc_sites(env);
+ mark_call_sites(env);
+ return 0;
+}
+
bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
{
return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
index 634cb2c15ffc..b96b2c429e22 100644
--- a/kernel/bpf/exception.h
+++ b/kernel/bpf/exception.h
@@ -7,6 +7,7 @@
struct bpf_verifier_env;
+int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env);
int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx);
#endif /* _LINUX_BPF_EXCEPTION_H */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index e3459123803c..fed5b66ed3b7 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -256,6 +256,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
data[i].non_stack_access =
data[off + cnt - 1].non_stack_access;
data[off + cnt - 1].non_stack_access = false;
+ data[i].throw_call =
+ data[off + cnt - 1].throw_call;
+ data[off + cnt - 1].throw_call = false;
+ data[i].resume_call =
+ data[off + cnt - 1].resume_call;
+ data[off + cnt - 1].resume_call = false;
data[i].cleanup_pad = data[off + cnt - 1].cleanup_pad;
data[off + cnt - 1].cleanup_pad = 0;
} else if (bpf_is_mem_insn(insn + i)) {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c65103152320..8163c75ec3fa 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -37,6 +37,7 @@
#include "diagnostics.h"
#include "disasm.h"
+#include "exception.h"
static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
@@ -21792,6 +21793,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
+ /* The CFG needs an edge from a call in a cleanup range to its pad. */
+ ret = bpf_prepare_cleanup_exceptions(env);
+ if (ret < 0)
+ goto skip_full_check;
+
/* Validate instructions and resolve the program's referenced resources. */
ret = check_and_resolve_insns(env);
if (ret < 0)
--
2.53.0-Meta
next prev parent reply other threads:[~2026-09-23 4:59 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-23 4:58 [PATCH bpf-next v5 00/21] bpf: Run exception cleanup landing pads when bpf_throw() unwinds Yonghong Song
2026-09-23 4:58 ` [PATCH bpf-next v5 01/21] bpf: Pack bpf_insn_aux_data flags into bit fields Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 19:16 ` Eduard Zingerman
2026-09-23 4:58 ` [PATCH bpf-next v5 02/21] bpf: Accept the compiler's exception cleanup table at program load Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 03/21] bpf: Add the bpf_unwind_resume() kfunc Yonghong Song
2026-09-23 6:13 ` Kumar Kartikeya Dwivedi
2026-09-23 4:59 ` [PATCH bpf-next v5 04/21] bpf: Add lookups for exception cleanup resumes and landing pads Yonghong Song
2026-09-23 4:59 ` Yonghong Song [this message]
2026-09-23 5:40 ` [PATCH bpf-next v5 05/21] bpf: Prepare for an exception cleanup table before the CFG walk bot+bpf-ci
2026-09-23 4:59 ` [PATCH bpf-next v5 06/21] bpf: Make exception landing pads reachable in the CFG Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 19:25 ` Eduard Zingerman
2026-09-23 4:59 ` [PATCH bpf-next v5 07/21] bpf: Explore the landing pads no call site reaches Yonghong Song
2026-09-23 16:26 ` Eduard Zingerman
2026-09-23 17:27 ` Yonghong Song
2026-09-23 18:06 ` Eduard Zingerman
2026-09-23 18:09 ` Yonghong Song
2026-09-23 21:28 ` Alexei Starovoitov
2026-09-23 22:19 ` Eduard Zingerman
2026-09-23 23:20 ` Alexei Starovoitov
2026-09-24 0:13 ` Kumar Kartikeya Dwivedi
2026-09-24 1:53 ` Alexei Starovoitov
2026-09-24 6:12 ` Kumar Kartikeya Dwivedi
2026-09-23 23:09 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 08/21] bpf: Walk the exception unwind in the verifier Yonghong Song
2026-09-23 16:34 ` Eduard Zingerman
2026-09-23 18:01 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 09/21] bpf: Refuse exception cleanup shapes bpf_throw() cannot dispatch Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 10/21] bpf: Refuse a private stack for a program with an exception cleanup table Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 11/21] bpf: Dispatch exception cleanup pads from bpf_throw() Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 12/21] bpf, x86: Dispatch exception cleanup pads at run time Yonghong Song
2026-09-23 6:45 ` Eduard Zingerman
2026-09-23 20:45 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 13/21] bpf, arm64: " Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 14/21] libbpf: Resolve the compiler's _Unwind_Resume to the kernel's kfunc Yonghong Song
2026-09-23 8:36 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 15/21] libbpf: Add cleanup_info to bpf_prog_load_opts Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 16/21] libbpf: Collect .bpf_cleanup records and pass them to the kernel Yonghong Song
2026-09-23 5:52 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 17/21] libbpf: Carry the exception cleanup table through the light skeleton Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 18/21] libbpf: Let the static linker carry .bpf_cleanup relocations Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 19/21] selftests/bpf: Add an end-to-end .bpf_cleanup exception test Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 20/21] selftests/bpf: Cover the exception cleanup shapes the chain does not reach Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 7:50 ` Eduard Zingerman
2026-09-23 5:00 ` [PATCH bpf-next v5 21/21] selftests/bpf: Load an exception cleanup program from a light skeleton Yonghong Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260923045912.2417059-1-yonghong.song@linux.dev \
--to=yonghong.song@linux.dev \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=kernel-team@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.