From: Yonghong Song <yonghong.song@linux.dev>
To: bpf@vger.kernel.org
Cc: Alexei Starovoitov <ast@kernel.org>,
Andrii Nakryiko <andrii@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Eduard Zingerman <eddyz87@gmail.com>,
kernel-team@fb.com
Subject: [PATCH bpf-next v5 02/21] bpf: Accept the compiler's exception cleanup table at program load
Date: Tue, 22 Sep 2026 21:58:56 -0700 [thread overview]
Message-ID: <20260923045856.2415318-1-yonghong.song@linux.dev> (raw)
In-Reply-To: <20260923045846.2414643-1-yonghong.song@linux.dev>
LLVM 23 added exception handling support for BPF with the .bpf_cleanup
section ([1]). Rust code compiled with panic=unwind runs cleanup code
(Drop glue) when bpf_throw() fires, and the LLVM BPF backend emits that
section from the landing pads the frontend produced. Plain C cannot
generate .bpf_cleanup unless inline asm is used. The Rust compiler does not
*properly* support BPF exception handling yet, but the kernel can support
the table today, and inline assembly is enough to test it.
Add the UAPI to carry the .bpf_cleanup table into the kernel. BPF_PROG_LOAD
grows cleanup_info, cleanup_info_cnt and cleanup_info_rec_size, and struct
bpf_cleanup_info describes one record as a triple of instruction indices:
the half-open call-site range [begin_off, end_off) and the landing_pad_off
the frame resumes at. The table arrives sorted by begin_off, with disjoint
ranges and each record's three offsets inside one subprogram;
check_cleanup_info() holds it to that at load time. Nothing reads it yet;
the patches that follow -- the CFG walk, the unwind walk and the JITs --
are its consumers.
Link: https://github.com/llvm/llvm-project/pull/192164 [1]
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
include/linux/bpf_verifier.h | 2 +
include/uapi/linux/bpf.h | 9 +++
kernel/bpf/check_btf.c | 134 +++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 2 +-
kernel/bpf/verifier.c | 1 +
tools/include/uapi/linux/bpf.h | 9 +++
6 files changed, 156 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 15bc61de61ef..2b6b480c059e 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -988,6 +988,8 @@ struct bpf_verifier_env {
struct arg_track **callsite_at_stack;
u32 pass_cnt; /* number of times do_check() was called */
u32 subprog_cnt;
+ struct bpf_cleanup_info *cleanup_info;
+ u32 cleanup_info_cnt;
/* number of instructions analyzed by the verifier */
u32 prev_insn_processed, insn_processed;
/* number of jmps, calls, exits analyzed so far */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 0aaa54359aeb..646aa8758d48 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1669,6 +1669,9 @@ union bpf_attr {
* verification.
*/
__s32 keyring_id;
+ __aligned_u64 cleanup_info; /* exception cleanup table */
+ __u32 cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+ __u32 cleanup_info_cnt; /* number of bpf_cleanup_info records */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7599,6 +7602,12 @@ struct bpf_line_info {
__u32 line_col;
};
+struct bpf_cleanup_info {
+ __u32 begin_off;
+ __u32 end_off;
+ __u32 landing_pad_off;
+};
+
struct bpf_spin_lock {
__u32 val;
};
diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c
index 0e8b3ccc7a5b..bc569fa5f97f 100644
--- a/kernel/bpf/check_btf.c
+++ b/kernel/bpf/check_btf.c
@@ -407,6 +407,136 @@ static int check_core_relo(struct bpf_verifier_env *env,
return err;
}
+#define MIN_BPF_CLEANUP_INFO_SIZE 12
+#define MAX_CLEANUP_INFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
+
+static int check_cleanup_info(struct bpf_verifier_env *env,
+ const union bpf_attr *attr,
+ bpfptr_t uattr)
+{
+ u32 krec_size = sizeof(struct bpf_cleanup_info);
+ u32 i, nrec, urec_size, min_size, prev_end = 0;
+ struct bpf_cleanup_info *krecord;
+ bpfptr_t urecord;
+ int ret = -EINVAL;
+
+ nrec = attr->cleanup_info_cnt;
+ if (!nrec)
+ return 0;
+ if (nrec > env->prog->len) {
+ verbose(env, "cleanup info has %u records for %u instructions\n",
+ nrec, env->prog->len);
+ return -EINVAL;
+ }
+
+ urec_size = attr->cleanup_info_rec_size;
+ if (urec_size < MIN_BPF_CLEANUP_INFO_SIZE ||
+ urec_size > MAX_CLEANUP_INFO_REC_SIZE ||
+ urec_size % sizeof(u32)) {
+ verbose(env, "invalid cleanup info rec size %u\n", urec_size);
+ return -EINVAL;
+ }
+
+ krecord = kvcalloc(nrec, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+ if (!krecord)
+ return -ENOMEM;
+
+ min_size = min_t(u32, krec_size, urec_size);
+ urecord = make_bpfptr(attr->cleanup_info, uattr.is_kernel);
+ for (i = 0; i < nrec; i++) {
+ struct bpf_subprog_info *sb, *se, *sl;
+ struct bpf_cleanup_info *rec = &krecord[i];
+
+ ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
+ if (ret) {
+ if (ret == -E2BIG) {
+ verbose(env, "nonzero tailing record in cleanup info\n");
+ if (copy_to_bpfptr_offset(uattr,
+ offsetof(union bpf_attr,
+ cleanup_info_rec_size),
+ &min_size, sizeof(min_size)))
+ ret = -EFAULT;
+ }
+ goto err_free;
+ }
+
+ if (copy_from_bpfptr(rec, urecord, min_size)) {
+ ret = -EFAULT;
+ goto err_free;
+ }
+ bpfptr_add(&urecord, urec_size);
+
+ ret = -EINVAL;
+ if (rec->begin_off >= rec->end_off) {
+ verbose(env, "cleanup_info[%u]: begin %u >= end %u\n",
+ i, rec->begin_off, rec->end_off);
+ goto err_free;
+ }
+ if (i && rec->begin_off < prev_end) {
+ verbose(env,
+ "cleanup_info[%u]: range [%u,%u) is unsorted or overlaps the previous record\n",
+ i, rec->begin_off, rec->end_off);
+ goto err_free;
+ }
+ prev_end = rec->end_off;
+
+ sb = bpf_find_containing_subprog(env, rec->begin_off);
+ se = bpf_find_containing_subprog(env, rec->end_off - 1);
+ sl = bpf_find_containing_subprog(env, rec->landing_pad_off);
+ if (!sb || !se || !sl) {
+ verbose(env, "cleanup_info[%u]: offset out of range\n", i);
+ goto err_free;
+ }
+ if (sb != se || sb != sl) {
+ verbose(env,
+ "cleanup_info[%u]: range/landing pad span multiple subprogs\n",
+ i);
+ goto err_free;
+ }
+ /*
+ * A zero opcode is the second half of a 16-byte insn, not an
+ * insn. end_off is exclusive, so it may be one past the last.
+ */
+ if (!env->prog->insnsi[rec->begin_off].code ||
+ !env->prog->insnsi[rec->landing_pad_off].code ||
+ (rec->end_off < env->prog->len &&
+ !env->prog->insnsi[rec->end_off].code)) {
+ verbose(env, "cleanup_info[%u]: points at invalid insn\n", i);
+ goto err_free;
+ }
+ }
+
+ /* Reject a landing pad inside any call-site range, its own included. */
+ ret = -EINVAL;
+ for (i = 0; i < nrec; i++) {
+ u32 pad = krecord[i].landing_pad_off;
+ u32 l = 0, r = nrec;
+
+ while (l < r) {
+ u32 m = l + (r - l) / 2;
+
+ if (pad < krecord[m].begin_off) {
+ r = m;
+ } else if (pad >= krecord[m].end_off) {
+ l = m + 1;
+ } else {
+ verbose(env,
+ "cleanup_info[%u]: landing pad %u is inside the call-site range of cleanup_info[%u]\n",
+ i, pad, m);
+ goto err_free;
+ }
+ }
+ }
+
+ env->cleanup_info = krecord;
+ env->cleanup_info_cnt = nrec;
+ return 0;
+
+err_free:
+ kvfree(krecord);
+ return ret;
+}
+
int bpf_prepare_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
@@ -441,6 +571,10 @@ int bpf_check_btf_info(struct bpf_verifier_env *env,
{
int err;
+ err = check_cleanup_info(env, attr, uattr);
+ if (err)
+ return err;
+
if (!attr->func_info_cnt && !attr->line_info_cnt) {
if (check_abnormal_return(env))
return -EINVAL;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 74496fd716d3..63c84a60c1b7 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2924,7 +2924,7 @@ int __init __used bpf_multi_func(void) { return 0; }
BTF_ID_LIST_GLOBAL_SINGLE(bpf_multi_func_btf_id, func, bpf_multi_func)
/* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD keyring_id
+#define BPF_PROG_LOAD_LAST_FIELD cleanup_info_cnt
static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
{
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a7c9e2d8965d..c65103152320 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21999,6 +21999,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
kvfree(env->succ);
kvfree(env->gotox_tmp_buf);
bpf_diag_free(env);
+ kvfree(env->cleanup_info);
kvfree(env);
return ret;
}
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 0aaa54359aeb..646aa8758d48 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1669,6 +1669,9 @@ union bpf_attr {
* verification.
*/
__s32 keyring_id;
+ __aligned_u64 cleanup_info; /* exception cleanup table */
+ __u32 cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+ __u32 cleanup_info_cnt; /* number of bpf_cleanup_info records */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7599,6 +7602,12 @@ struct bpf_line_info {
__u32 line_col;
};
+struct bpf_cleanup_info {
+ __u32 begin_off;
+ __u32 end_off;
+ __u32 landing_pad_off;
+};
+
struct bpf_spin_lock {
__u32 val;
};
--
2.53.0-Meta
next prev parent reply other threads:[~2026-09-23 4:59 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-23 4:58 [PATCH bpf-next v5 00/21] bpf: Run exception cleanup landing pads when bpf_throw() unwinds Yonghong Song
2026-09-23 4:58 ` [PATCH bpf-next v5 01/21] bpf: Pack bpf_insn_aux_data flags into bit fields Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 19:16 ` Eduard Zingerman
2026-09-23 4:58 ` Yonghong Song [this message]
2026-09-23 4:59 ` [PATCH bpf-next v5 03/21] bpf: Add the bpf_unwind_resume() kfunc Yonghong Song
2026-09-23 6:13 ` Kumar Kartikeya Dwivedi
2026-09-23 4:59 ` [PATCH bpf-next v5 04/21] bpf: Add lookups for exception cleanup resumes and landing pads Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 05/21] bpf: Prepare for an exception cleanup table before the CFG walk Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 4:59 ` [PATCH bpf-next v5 06/21] bpf: Make exception landing pads reachable in the CFG Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 19:25 ` Eduard Zingerman
2026-09-23 4:59 ` [PATCH bpf-next v5 07/21] bpf: Explore the landing pads no call site reaches Yonghong Song
2026-09-23 16:26 ` Eduard Zingerman
2026-09-23 17:27 ` Yonghong Song
2026-09-23 18:06 ` Eduard Zingerman
2026-09-23 18:09 ` Yonghong Song
2026-09-23 21:28 ` Alexei Starovoitov
2026-09-23 22:19 ` Eduard Zingerman
2026-09-23 23:20 ` Alexei Starovoitov
2026-09-24 0:13 ` Kumar Kartikeya Dwivedi
2026-09-24 1:53 ` Alexei Starovoitov
2026-09-24 6:12 ` Kumar Kartikeya Dwivedi
2026-09-23 23:09 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 08/21] bpf: Walk the exception unwind in the verifier Yonghong Song
2026-09-23 16:34 ` Eduard Zingerman
2026-09-23 18:01 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 09/21] bpf: Refuse exception cleanup shapes bpf_throw() cannot dispatch Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 10/21] bpf: Refuse a private stack for a program with an exception cleanup table Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 11/21] bpf: Dispatch exception cleanup pads from bpf_throw() Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 12/21] bpf, x86: Dispatch exception cleanup pads at run time Yonghong Song
2026-09-23 6:45 ` Eduard Zingerman
2026-09-23 20:45 ` Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 13/21] bpf, arm64: " Yonghong Song
2026-09-23 4:59 ` [PATCH bpf-next v5 14/21] libbpf: Resolve the compiler's _Unwind_Resume to the kernel's kfunc Yonghong Song
2026-09-23 8:36 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 15/21] libbpf: Add cleanup_info to bpf_prog_load_opts Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 16/21] libbpf: Collect .bpf_cleanup records and pass them to the kernel Yonghong Song
2026-09-23 5:52 ` Kumar Kartikeya Dwivedi
2026-09-23 5:00 ` [PATCH bpf-next v5 17/21] libbpf: Carry the exception cleanup table through the light skeleton Yonghong Song
2026-09-23 5:00 ` [PATCH bpf-next v5 18/21] libbpf: Let the static linker carry .bpf_cleanup relocations Yonghong Song
2026-09-23 5:40 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 19/21] selftests/bpf: Add an end-to-end .bpf_cleanup exception test Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 5:00 ` [PATCH bpf-next v5 20/21] selftests/bpf: Cover the exception cleanup shapes the chain does not reach Yonghong Song
2026-09-23 5:51 ` bot+bpf-ci
2026-09-23 7:50 ` Eduard Zingerman
2026-09-23 5:00 ` [PATCH bpf-next v5 21/21] selftests/bpf: Load an exception cleanup program from a light skeleton Yonghong Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260923045856.2415318-1-yonghong.song@linux.dev \
--to=yonghong.song@linux.dev \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=kernel-team@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox