From: Yonghong Song <yonghong.song@linux.dev>
To: bpf@vger.kernel.org
Cc: Alexei Starovoitov <ast@kernel.org>,
Andrii Nakryiko <andrii@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Eduard Zingerman <eddyz87@gmail.com>,
kernel-team@fb.com
Subject: [PATCH bpf-next v5 05/21] bpf: Prepare for an exception cleanup table before the CFG walk
Date: Tue, 22 Sep 2026 21:59:12 -0700 [thread overview]
Message-ID: <20260923045912.2417059-1-yonghong.song@linux.dev> (raw)
In-Reply-To: <20260923045846.2414643-1-yonghong.song@linux.dev>
Record in insn_aux_data what the later passes need from the cleanup table:
throw_call for every bpf_throw() and resume_call for every
bpf_unwind_resume() -- the two calls a JIT lowers its own way rather than
as calls -- and cleanup_pad, the landing pad a frame resumes at, for every
call within the [begin_off, end_off) range of a cleanup record. Subsequent
commits consume all three.
Marking the two calls here, rather than recognising them in the JIT, is
what makes the recognition exact: by the time a JIT runs,
bpf_fixup_kfunc_call() has rewritten every other kfunc's imm into an offset
from __bpf_call_base, and a BTF id compared against one of those offsets
could match an unrelated call.
bpf_prepare_cleanup_exceptions() runs before bpf_check_cfg(), whose walk
consumes what it produces. It refuses a table on an offloaded program, on
one whose JIT cannot dispatch landing pads or was not asked to compile it,
and on one that also installs an exception callback -- two different
answers to what runs on the way out. What survives is marked jit_required:
the interpreter cannot dispatch a pad. bpf_jit_supports_cleanup_pads() is
weak here and says no; the arch patches provide the real ones.
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
include/linux/bpf_verifier.h | 2 ++
include/linux/filter.h | 1 +
kernel/bpf/core.c | 5 +++
kernel/bpf/exception.c | 62 ++++++++++++++++++++++++++++++++++++
kernel/bpf/exception.h | 1 +
kernel/bpf/fixups.c | 6 ++++
kernel/bpf/verifier.c | 6 ++++
7 files changed, 83 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 666c73308b32..e12ec91b8150 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -692,6 +692,8 @@ struct bpf_insn_aux_data {
*/
u64 fastcall_spills_num:3;
u64 arg_prog:4;
+ u64 throw_call:1; /* call to bpf_throw() */
+ u64 resume_call:1; /* call to bpf_unwind_resume() */
/* below flags are initialized once */
u64 jmp_point:1;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 422284b4fa96..2582a7606e46 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1242,6 +1242,7 @@ bool bpf_jit_supports_stack_args(void);
bool bpf_jit_supports_arena_args(void);
bool bpf_jit_supports_far_kfunc_call(void);
bool bpf_jit_supports_exceptions(void);
+bool bpf_jit_supports_cleanup_pads(void);
bool bpf_jit_supports_ptr_xchg(void);
bool bpf_jit_supports_arena(void);
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 227211166dcc..a379cd1ec4c6 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3475,6 +3475,11 @@ void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp,
{
}
+bool __weak bpf_jit_supports_cleanup_pads(void)
+{
+ return false;
+}
+
bool __weak bpf_jit_supports_timed_may_goto(void)
{
return false;
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
index 6989f31aa6c9..244ea1b5417c 100644
--- a/kernel/bpf/exception.c
+++ b/kernel/bpf/exception.c
@@ -7,8 +7,70 @@
#include <linux/filter.h>
#include "exception.h"
+#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
+
BTF_ID_LIST_SINGLE(bpf_unwind_resume_id, func, bpf_unwind_resume)
+static void mark_kfunc_sites(struct bpf_verifier_env *env)
+{
+ u32 i;
+
+ for (i = 0; i < env->prog->len; i++) {
+ struct bpf_insn *insn = &env->prog->insnsi[i];
+
+ if (bpf_is_throw_kfunc(insn))
+ env->insn_aux_data[i].throw_call = true;
+ else if (bpf_is_unwind_resume_kfunc(insn))
+ env->insn_aux_data[i].resume_call = true;
+ }
+}
+
+static void mark_call_sites(struct bpf_verifier_env *env)
+{
+ u32 i, j;
+
+ for (i = 0; i < env->cleanup_info_cnt; i++) {
+ struct bpf_cleanup_info *rec = &env->cleanup_info[i];
+
+ for (j = rec->begin_off; j < rec->end_off; j++) {
+ struct bpf_insn *insn = &env->prog->insnsi[j];
+
+ if (!bpf_pseudo_call(insn) && !bpf_is_throw_kfunc(insn))
+ continue;
+ env->insn_aux_data[j].cleanup_pad = rec->landing_pad_off + 1;
+ }
+ }
+}
+
+int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env)
+{
+ if (!env->cleanup_info_cnt)
+ return 0;
+
+ if (bpf_prog_is_offloaded(env->prog->aux)) {
+ verbose(env,
+ "exception cleanup is not supported for offloaded programs\n");
+ return -EINVAL;
+ }
+
+ if (!bpf_jit_supports_cleanup_pads() || !env->prog->jit_requested) {
+ verbose(env,
+ "exception cleanup needs a JIT that can dispatch landing pads\n");
+ return -EOPNOTSUPP;
+ }
+ env->prog->jit_required = 1;
+
+ if (env->exception_callback_subprog) {
+ verbose(env,
+ "exception cleanup table cannot be combined with an exception callback\n");
+ return -EINVAL;
+ }
+
+ mark_kfunc_sites(env);
+ mark_call_sites(env);
+ return 0;
+}
+
bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
{
return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
index 634cb2c15ffc..b96b2c429e22 100644
--- a/kernel/bpf/exception.h
+++ b/kernel/bpf/exception.h
@@ -7,6 +7,7 @@
struct bpf_verifier_env;
+int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env);
int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx);
#endif /* _LINUX_BPF_EXCEPTION_H */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index e3459123803c..fed5b66ed3b7 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -256,6 +256,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
data[i].non_stack_access =
data[off + cnt - 1].non_stack_access;
data[off + cnt - 1].non_stack_access = false;
+ data[i].throw_call =
+ data[off + cnt - 1].throw_call;
+ data[off + cnt - 1].throw_call = false;
+ data[i].resume_call =
+ data[off + cnt - 1].resume_call;
+ data[off + cnt - 1].resume_call = false;
data[i].cleanup_pad = data[off + cnt - 1].cleanup_pad;
data[off + cnt - 1].cleanup_pad = 0;
} else if (bpf_is_mem_insn(insn + i)) {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c65103152320..8163c75ec3fa 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -37,6 +37,7 @@
#include "diagnostics.h"
#include "disasm.h"
+#include "exception.h"
static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
@@ -21792,6 +21793,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
+ /* The CFG needs an edge from a call in a cleanup range to its pad. */
+ ret = bpf_prepare_cleanup_exceptions(env);
+ if (ret < 0)
+ goto skip_full_check;
+
/* Validate instructions and resolve the program's referenced resources. */
ret = check_and_resolve_insns(env);
if (ret < 0)
--
2.53.0-Meta
next prev parent reply other threads:[~2026-09-23 4:59 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-23 4:58 [PATCH bpf-next v5 00/21] bpf: Run exception cleanup landing pads when bpf_throw() unwinds Yonghong Song
2026-09-23 4:58 ` [PATCH bpf-next v5 01/21] bpf: Pack bpf_insn_aux_data flags into bit fields Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 19:16 ` Eduard Zingerman
2026-09-23 4:58 ` [PATCH bpf-next v5 02/21] bpf: Accept the compiler's exception cleanup table at program load Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 03/21] bpf: Add the bpf_unwind_resume() kfunc Yonghong Song
2026-09-23 6:13 ` Kumar Kartikeya Dwivedi
2026-09-23 4:59 ` [PATCH bpf-next v5 04/21] bpf: Add lookups for exception cleanup resumes and landing pads Yonghong Song
2026-09-23 4:59 ` Yonghong Song [this message]
2026-09-23 5:40 ` [PATCH bpf-next v5 05/21] bpf: Prepare for an exception cleanup table before the CFG walk bot+bpf-ci
2026-09-23 4:59 ` [PATCH bpf-next v5 06/21] bpf: Make exception landing pads reachable in the CFG Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 19:25 ` Eduard Zingerman
2026-09-23 4:59 ` [PATCH bpf-next v5 07/21] bpf: Explore the landing pads no call site reaches Yonghong Song
2026-09-23 16:26 ` Eduard Zingerman
2026-09-23 17:27 ` Yonghong Song
2026-09-23 18:06 ` Eduard Zingerman
2026-09-23 18:09 ` Yonghong Song
2026-09-23 21:28 ` Alexei Starovoitov
2026-09-23 22:19 ` Eduard Zingerman
2026-09-23 23:20 ` Alexei Starovoitov
2026-09-24 0:13 ` Kumar Kartikeya Dwivedi
2026-09-24 1:53 ` Alexei Starovoitov
2026-09-24 6:12 ` Kumar Kartikeya Dwivedi
2026-09-23 23:09 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 08/21] bpf: Walk the exception unwind in the verifier Yonghong Song
2026-09-23 16:34 ` Eduard Zingerman
2026-09-23 18:01 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 09/21] bpf: Refuse exception cleanup shapes bpf_throw() cannot dispatch Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 10/21] bpf: Refuse a private stack for a program with an exception cleanup table Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 11/21] bpf: Dispatch exception cleanup pads from bpf_throw() Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 12/21] bpf, x86: Dispatch exception cleanup pads at run time Yonghong Song
2026-09-23 6:45 ` Eduard Zingerman
2026-09-23 20:45 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 13/21] bpf, arm64: " Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 14/21] libbpf: Resolve the compiler's _Unwind_Resume to the kernel's kfunc Yonghong Song
2026-09-23 8:36 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 15/21] libbpf: Add cleanup_info to bpf_prog_load_opts Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 16/21] libbpf: Collect .bpf_cleanup records and pass them to the kernel Yonghong Song
2026-09-23 5:52 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 17/21] libbpf: Carry the exception cleanup table through the light skeleton Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 18/21] libbpf: Let the static linker carry .bpf_cleanup relocations Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 19/21] selftests/bpf: Add an end-to-end .bpf_cleanup exception test Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 20/21] selftests/bpf: Cover the exception cleanup shapes the chain does not reach Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 7:50 ` Eduard Zingerman
2026-09-23 5:00 ` [PATCH bpf-next v5 21/21] selftests/bpf: Load an exception cleanup program from a light skeleton Yonghong Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260923045912.2417059-1-yonghong.song@linux.dev \
--to=yonghong.song@linux.dev \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=kernel-team@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox