From: Yonghong Song <yonghong.song@linux.dev>
To: bpf@vger.kernel.org
Cc: Alexei Starovoitov <ast@kernel.org>,
Andrii Nakryiko <andrii@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Eduard Zingerman <eddyz87@gmail.com>,
kernel-team@fb.com
Subject: [PATCH bpf-next v6 02/21] bpf: Accept the compiler's exception cleanup table at program load
Date: Fri, 25 Sep 2026 22:00:16 -0700 [thread overview]
Message-ID: <20260926050016.2214730-1-yonghong.song@linux.dev> (raw)
In-Reply-To: <20260926050006.2213110-1-yonghong.song@linux.dev>
LLVM 23 added exception handling support for BPF with the .bpf_cleanup
section ([1]). Rust code compiled with panic=unwind runs cleanup code
(Drop glue) when an unwind passes through, and the LLVM BPF backend emits
that section from the landing pads the frontend produced. Plain C cannot
generate .bpf_cleanup unless inline asm is used. The Rust compiler does not
*properly* support BPF exception handling yet, but the kernel can support
the table today, and inline assembly is enough to test it.
Add the UAPI to carry the .bpf_cleanup table into the kernel. BPF_PROG_LOAD
grows cleanup_info, cleanup_info_cnt and cleanup_info_rec_size, and struct
bpf_cleanup_info describes one record as a triple of instruction indices:
the half-open call-site range [begin_off, end_off) and the landing_pad_off
the frame resumes at. The table arrives sorted by begin_off, with disjoint
ranges and each record's three offsets inside one subprogram;
check_cleanup_info() holds it to that at load time. Nothing reads it yet;
the patches that follow -- the CFG walk, the unwind walk and the JITs --
are its consumers.
Link: https://github.com/llvm/llvm-project/pull/192164 [1]
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
include/linux/bpf_verifier.h | 2 +
include/uapi/linux/bpf.h | 9 +++
kernel/bpf/check_btf.c | 134 +++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 2 +-
kernel/bpf/verifier.c | 1 +
tools/include/uapi/linux/bpf.h | 9 +++
6 files changed, 156 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index d85cf969bcb0..6ce25c96ebed 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1018,6 +1018,8 @@ struct bpf_verifier_env {
struct spill_snapshot **callsite_at_stack;
u32 pass_cnt; /* number of times do_check() was called */
u32 subprog_cnt;
+ struct bpf_cleanup_info *cleanup_info;
+ u32 cleanup_info_cnt;
/* number of instructions analyzed by the verifier */
u32 prev_insn_processed, insn_processed;
/* number of jmps, calls, exits analyzed so far */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 4687c3310996..aca43f4f927f 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1702,6 +1702,9 @@ union bpf_attr {
* verification.
*/
__s32 keyring_id;
+ __aligned_u64 cleanup_info; /* exception cleanup table */
+ __u32 cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+ __u32 cleanup_info_cnt; /* number of bpf_cleanup_info records */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7638,6 +7641,12 @@ struct bpf_line_info {
__u32 line_col;
};
+struct bpf_cleanup_info {
+ __u32 begin_off;
+ __u32 end_off;
+ __u32 landing_pad_off;
+};
+
struct bpf_spin_lock {
__u32 val;
};
diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c
index 4c1ed842f661..5c62f350d500 100644
--- a/kernel/bpf/check_btf.c
+++ b/kernel/bpf/check_btf.c
@@ -407,6 +407,136 @@ int bpf_check_core_relo(struct bpf_verifier_env *env,
return err;
}
+#define MIN_BPF_CLEANUP_INFO_SIZE 12
+#define MAX_CLEANUP_INFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
+
+static int check_cleanup_info(struct bpf_verifier_env *env,
+ const union bpf_attr *attr,
+ bpfptr_t uattr)
+{
+ u32 krec_size = sizeof(struct bpf_cleanup_info);
+ u32 i, nrec, urec_size, min_size, prev_end = 0;
+ struct bpf_cleanup_info *krecord;
+ bpfptr_t urecord;
+ int ret = -EINVAL;
+
+ nrec = attr->cleanup_info_cnt;
+ if (!nrec)
+ return 0;
+ if (nrec > env->prog->len) {
+ verbose(env, "cleanup info has %u records for %u instructions\n",
+ nrec, env->prog->len);
+ return -EINVAL;
+ }
+
+ urec_size = attr->cleanup_info_rec_size;
+ if (urec_size < MIN_BPF_CLEANUP_INFO_SIZE ||
+ urec_size > MAX_CLEANUP_INFO_REC_SIZE ||
+ urec_size % sizeof(u32)) {
+ verbose(env, "invalid cleanup info rec size %u\n", urec_size);
+ return -EINVAL;
+ }
+
+ krecord = kvcalloc(nrec, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+ if (!krecord)
+ return -ENOMEM;
+
+ min_size = min_t(u32, krec_size, urec_size);
+ urecord = make_bpfptr(attr->cleanup_info, uattr.is_kernel);
+ for (i = 0; i < nrec; i++) {
+ struct bpf_subprog_info *sb, *se, *sl;
+ struct bpf_cleanup_info *rec = &krecord[i];
+
+ ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
+ if (ret) {
+ if (ret == -E2BIG) {
+ verbose(env, "nonzero tailing record in cleanup info\n");
+ if (copy_to_bpfptr_offset(uattr,
+ offsetof(union bpf_attr,
+ cleanup_info_rec_size),
+ &min_size, sizeof(min_size)))
+ ret = -EFAULT;
+ }
+ goto err_free;
+ }
+
+ if (copy_from_bpfptr(rec, urecord, min_size)) {
+ ret = -EFAULT;
+ goto err_free;
+ }
+ bpfptr_add(&urecord, urec_size);
+
+ ret = -EINVAL;
+ if (rec->begin_off >= rec->end_off) {
+ verbose(env, "cleanup_info[%u]: begin %u >= end %u\n",
+ i, rec->begin_off, rec->end_off);
+ goto err_free;
+ }
+ if (i && rec->begin_off < prev_end) {
+ verbose(env,
+ "cleanup_info[%u]: range [%u,%u) is unsorted or overlaps the previous record\n",
+ i, rec->begin_off, rec->end_off);
+ goto err_free;
+ }
+ prev_end = rec->end_off;
+
+ sb = bpf_find_containing_subprog(env, rec->begin_off);
+ se = bpf_find_containing_subprog(env, rec->end_off - 1);
+ sl = bpf_find_containing_subprog(env, rec->landing_pad_off);
+ if (!sb || !se || !sl) {
+ verbose(env, "cleanup_info[%u]: offset out of range\n", i);
+ goto err_free;
+ }
+ if (sb != se || sb != sl) {
+ verbose(env,
+ "cleanup_info[%u]: range/landing pad span multiple subprogs\n",
+ i);
+ goto err_free;
+ }
+ /*
+ * A zero opcode is the second half of a 16-byte insn, not an
+ * insn. end_off is exclusive, so it may be one past the last.
+ */
+ if (!env->prog->insnsi[rec->begin_off].code ||
+ !env->prog->insnsi[rec->landing_pad_off].code ||
+ (rec->end_off < env->prog->len &&
+ !env->prog->insnsi[rec->end_off].code)) {
+ verbose(env, "cleanup_info[%u]: points at invalid insn\n", i);
+ goto err_free;
+ }
+ }
+
+ /* Reject a landing pad inside any call-site range, its own included. */
+ ret = -EINVAL;
+ for (i = 0; i < nrec; i++) {
+ u32 pad = krecord[i].landing_pad_off;
+ u32 l = 0, r = nrec;
+
+ while (l < r) {
+ u32 m = l + (r - l) / 2;
+
+ if (pad < krecord[m].begin_off) {
+ r = m;
+ } else if (pad >= krecord[m].end_off) {
+ l = m + 1;
+ } else {
+ verbose(env,
+ "cleanup_info[%u]: landing pad %u is inside the call-site range of cleanup_info[%u]\n",
+ i, pad, m);
+ goto err_free;
+ }
+ }
+ }
+
+ env->cleanup_info = krecord;
+ env->cleanup_info_cnt = nrec;
+ return 0;
+
+err_free:
+ kvfree(krecord);
+ return ret;
+}
+
int bpf_prepare_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
@@ -441,6 +571,10 @@ int bpf_check_btf_info(struct bpf_verifier_env *env,
{
int err;
+ err = check_cleanup_info(env, attr, uattr);
+ if (err)
+ return err;
+
if (!attr->func_info_cnt && !attr->line_info_cnt) {
if (check_abnormal_return(env))
return -EINVAL;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index ac52f4ae414c..0e14afe3fdc5 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2924,7 +2924,7 @@ int __init __used bpf_multi_func(void) { return 0; }
BTF_ID_LIST_GLOBAL_SINGLE(bpf_multi_func_btf_id, func, bpf_multi_func)
/* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD keyring_id
+#define BPF_PROG_LOAD_LAST_FIELD cleanup_info_cnt
static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
{
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 03dbc0e00398..6d3408f295ed 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22801,6 +22801,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
kvfree(env->callx_edges);
kvfree(env->func_ptrs);
bpf_diag_free(env);
+ kvfree(env->cleanup_info);
kvfree(env);
return ret;
}
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 4687c3310996..aca43f4f927f 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1702,6 +1702,9 @@ union bpf_attr {
* verification.
*/
__s32 keyring_id;
+ __aligned_u64 cleanup_info; /* exception cleanup table */
+ __u32 cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+ __u32 cleanup_info_cnt; /* number of bpf_cleanup_info records */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7638,6 +7641,12 @@ struct bpf_line_info {
__u32 line_col;
};
+struct bpf_cleanup_info {
+ __u32 begin_off;
+ __u32 end_off;
+ __u32 landing_pad_off;
+};
+
struct bpf_spin_lock {
__u32 val;
};
--
2.53.0-Meta
next prev parent reply other threads:[~2026-09-26 5:00 UTC|newest]
Thread overview: 56+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-26 5:00 [PATCH bpf-next v6 00/21] bpf: Run exception cleanup landing pads when bpf_unwind() unwinds Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 01/21] bpf: Pack bpf_insn_aux_data flags into bit fields Yonghong Song
2026-09-26 5:00 ` Yonghong Song [this message]
2026-09-26 5:00 ` [PATCH bpf-next v6 03/21] bpf: Add the bpf_unwind() and bpf_unwind_resume() kfuncs Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 04/21] bpf: Add lookups for exception cleanup resumes and landing pads Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 05/21] bpf: Prepare for an exception cleanup table before the CFG walk Yonghong Song
2026-09-26 5:16 ` sashiko-bot
2026-09-26 23:54 ` Yonghong Song
2026-09-27 20:39 ` bot+bpf-ci
2026-09-28 0:01 ` Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 06/21] bpf: Make exception landing pads reachable in the CFG Yonghong Song
2026-09-26 5:21 ` sashiko-bot
2026-09-27 0:02 ` Yonghong Song
2026-09-27 20:40 ` bot+bpf-ci
2026-09-28 0:12 ` Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 07/21] bpf: Resume a covered call at its landing pad Yonghong Song
2026-09-26 5:15 ` sashiko-bot
2026-09-26 8:21 ` Alexei Starovoitov
2026-09-27 0:04 ` Yonghong Song
2026-09-27 0:41 ` Yonghong Song
2026-09-27 20:40 ` bot+bpf-ci
2026-09-28 0:17 ` Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 08/21] bpf: Refuse a landing pad that does not resume Yonghong Song
2026-09-26 5:17 ` sashiko-bot
2026-09-27 3:06 ` Yonghong Song
2026-09-27 20:40 ` bot+bpf-ci
2026-09-28 0:29 ` Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 09/21] bpf: Refuse a private stack for a program with an exception cleanup table Yonghong Song
2026-09-26 5:00 ` [PATCH bpf-next v6 10/21] bpf: Dispatch cleanup pads by rewriting return addresses Yonghong Song
2026-09-27 20:40 ` bot+bpf-ci
2026-09-28 1:08 ` Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 11/21] bpf, x86: Dispatch exception cleanup pads at run time Yonghong Song
2026-09-26 5:15 ` sashiko-bot
2026-09-27 4:35 ` Yonghong Song
2026-09-27 20:39 ` bot+bpf-ci
2026-09-28 3:10 ` Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 12/21] bpf, arm64: " Yonghong Song
2026-09-26 5:14 ` sashiko-bot
2026-09-27 20:40 ` bot+bpf-ci
2026-09-26 5:01 ` [PATCH bpf-next v6 13/21] libbpf: Resolve the compiler's _Unwind_Resume to the kernel's kfunc Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 14/21] libbpf: Add cleanup_info to bpf_prog_load_opts Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 15/21] libbpf: Collect .bpf_cleanup records and pass them to the kernel Yonghong Song
2026-09-27 20:39 ` bot+bpf-ci
2026-09-28 3:28 ` Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 16/21] libbpf: Carry the exception cleanup table through the light skeleton Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 17/21] libbpf: Let the static linker carry .bpf_cleanup relocations Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 18/21] selftests/bpf: Add an end-to-end .bpf_cleanup exception test Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 19/21] selftests/bpf: Add __set_global() and __ret_global() test tags Yonghong Song
2026-09-26 5:18 ` sashiko-bot
2026-09-27 4:58 ` Yonghong Song
2026-09-27 20:24 ` bot+bpf-ci
2026-09-28 3:36 ` Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 20/21] selftests/bpf: Cover the exception cleanup shapes the chain does not reach Yonghong Song
2026-09-27 20:40 ` bot+bpf-ci
2026-09-28 3:49 ` Yonghong Song
2026-09-26 5:01 ` [PATCH bpf-next v6 21/21] selftests/bpf: Load an exception cleanup program from a light skeleton Yonghong Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260926050016.2214730-1-yonghong.song@linux.dev \
--to=yonghong.song@linux.dev \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=kernel-team@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.