[PATCH bpf-next v9 02/23] bpf: Accept the compiler's exception cleanup table at program load

[Yonghong Song <yonghong.song@linux.dev>]

LLVM 23 added exception handling for BPF with the .bpf_cleanup section
[1]: Rust code compiled with panic=unwind runs cleanup code (Drop glue)
when an unwind passes through, and the BPF backend emits the section
from the landing pads. rustc does not fully support BPF exception
handling yet, and C can produce the section only with inline asm, which
is enough to test it.

Add the UAPI to carry the table to BPF_PROG_LOAD: cleanup_info,
cleanup_info_cnt and cleanup_info_rec_size. A record, struct
bpf_cleanup_info, says that calls in [begin_off, end_off) that unwind
resume at landing_pad_off, all instruction indices.

bpf_exc_check_info() checks the table at load time: records sorted by
begin_off, with non-empty and disjoint ranges, each inside one subprog,
no landing pad inside any range, and no offset naming the second half of
an ld_imm64. Nothing reads the table yet.

Link: https://github.com/llvm/llvm-project/pull/192164 [1]
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf_verifier.h   |   2 +
 include/uapi/linux/bpf.h       |  14 ++++
 kernel/bpf/Makefile            |   2 +-
 kernel/bpf/exception.c         | 137 +++++++++++++++++++++++++++++++++
 kernel/bpf/exception.h         |  15 ++++
 kernel/bpf/syscall.c           |   2 +-
 kernel/bpf/verifier.c          |   6 ++
 tools/include/uapi/linux/bpf.h |  14 ++++
 8 files changed, 190 insertions(+), 2 deletions(-)
 create mode 100644 kernel/bpf/exception.c
 create mode 100644 kernel/bpf/exception.h

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 68636e1f048b..a5f493876993 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1021,6 +1021,8 @@ struct bpf_verifier_env {
 	struct spill_snapshot **callsite_at_stack;
 	u32 pass_cnt; /* number of times do_check() was called */
 	u32 subprog_cnt;
+	struct bpf_cleanup_info *cleanup_info;
+	u32 cleanup_info_cnt;
 	/* number of instructions analyzed by the verifier */
 	u32 prev_insn_processed, insn_processed;
 	/* number of jmps, calls, exits analyzed so far */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index e0ed44b1bbcb..15dfba087201 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1708,6 +1708,9 @@ union bpf_attr {
 		 * verification.
 		 */
 		__s32		keyring_id;
+		__aligned_u64	cleanup_info;	/* exception cleanup table */
+		__u32		cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+		__u32		cleanup_info_cnt; /* number of bpf_cleanup_info records */
 	};
 
 	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7644,6 +7647,17 @@ struct bpf_line_info {
 	__u32	line_col;
 };
 
+/*
+ * One record of an exception cleanup table: calls in [begin_off, end_off)
+ * that unwind resume at landing_pad_off. All three are instruction offsets
+ * in the program as loaded.
+ */
+struct bpf_cleanup_info {
+	__u32	begin_off;
+	__u32	end_off;
+	__u32	landing_pad_off;
+};
+
 struct bpf_spin_lock {
 	__u32	val;
 };
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index c1f9b0d3468d..8a6947b3d13a 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -11,7 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_
 obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
 obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
 obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
-obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o
+obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o exception.o
 obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o
 obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
 obj-$(CONFIG_BPF_JIT) += trampoline.o
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
new file mode 100644
index 000000000000..d6b8ca98e71c
--- /dev/null
+++ b/kernel/bpf/exception.c
@@ -0,0 +1,137 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/slab.h>
+#include "exception.h"
+
+#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
+
+#define MIN_BPF_CLEANUP_INFO_SIZE	12
+#define MAX_CLEANUP_INFO_REC_SIZE	252	/* as MAX_FUNCINFO_REC_SIZE */
+
+int bpf_exc_check_info(struct bpf_verifier_env *env, const union bpf_attr *attr,
+		       bpfptr_t uattr)
+{
+	u32 krec_size = sizeof(struct bpf_cleanup_info);
+	u32 i, nrec, urec_size, min_size, prev_end = 0;
+	struct bpf_cleanup_info *krecord;
+	bpfptr_t urecord;
+	int ret = -EINVAL;
+
+	nrec = attr->cleanup_info_cnt;
+	if (!nrec)
+		return 0;
+	if (nrec > env->prog->len) {
+		verbose(env, "cleanup info has %u records for %u instructions\n",
+			nrec, env->prog->len);
+		return -EINVAL;
+	}
+
+	urec_size = attr->cleanup_info_rec_size;
+	if (urec_size < MIN_BPF_CLEANUP_INFO_SIZE ||
+	    urec_size > MAX_CLEANUP_INFO_REC_SIZE ||
+	    urec_size % sizeof(u32)) {
+		verbose(env, "invalid cleanup info rec size %u\n", urec_size);
+		return -EINVAL;
+	}
+
+	krecord = kvcalloc(nrec, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+	if (!krecord)
+		return -ENOMEM;
+
+	min_size = min_t(u32, krec_size, urec_size);
+	urecord = make_bpfptr(attr->cleanup_info, uattr.is_kernel);
+	for (i = 0; i < nrec; i++) {
+		struct bpf_subprog_info *sb, *se, *sl;
+		struct bpf_cleanup_info *rec = &krecord[i];
+
+		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
+		if (ret) {
+			if (ret == -E2BIG) {
+				verbose(env, "nonzero tailing record in cleanup info\n");
+				if (copy_to_bpfptr_offset(uattr,
+							  offsetof(union bpf_attr,
+								   cleanup_info_rec_size),
+							  &min_size, sizeof(min_size)))
+					ret = -EFAULT;
+			}
+			goto err_free;
+		}
+
+		if (copy_from_bpfptr(rec, urecord, min_size)) {
+			ret = -EFAULT;
+			goto err_free;
+		}
+		bpfptr_add(&urecord, urec_size);
+
+		ret = -EINVAL;
+		if (rec->begin_off >= rec->end_off) {
+			verbose(env, "cleanup_info[%u]: begin %u >= end %u\n",
+				i, rec->begin_off, rec->end_off);
+			goto err_free;
+		}
+		if (i && rec->begin_off < prev_end) {
+			verbose(env,
+				"cleanup_info[%u]: range [%u,%u) is unsorted or overlaps the previous record\n",
+				i, rec->begin_off, rec->end_off);
+			goto err_free;
+		}
+		prev_end = rec->end_off;
+
+		sb = bpf_find_containing_subprog(env, rec->begin_off);
+		se = bpf_find_containing_subprog(env, rec->end_off - 1);
+		sl = bpf_find_containing_subprog(env, rec->landing_pad_off);
+		if (!sb || !se || !sl) {
+			verbose(env, "cleanup_info[%u]: offset out of range\n", i);
+			goto err_free;
+		}
+		if (sb != se || sb != sl) {
+			verbose(env,
+				"cleanup_info[%u]: range/landing pad span multiple subprogs\n",
+				i);
+			goto err_free;
+		}
+		/*
+		 * A zero opcode is the second half of a 16-byte insn, not an
+		 * insn. end_off is exclusive, so it may be one past the last.
+		 */
+		if (!env->prog->insnsi[rec->begin_off].code ||
+		    !env->prog->insnsi[rec->landing_pad_off].code ||
+		    (rec->end_off < env->prog->len &&
+		     !env->prog->insnsi[rec->end_off].code)) {
+			verbose(env, "cleanup_info[%u]: points at invalid insn\n", i);
+			goto err_free;
+		}
+	}
+
+	/* Reject a landing pad inside any call-site range, its own included. */
+	ret = -EINVAL;
+	for (i = 0; i < nrec; i++) {
+		u32 pad = krecord[i].landing_pad_off;
+		u32 l = 0, r = nrec;
+
+		while (l < r) {
+			u32 m = l + (r - l) / 2;
+
+			if (pad < krecord[m].begin_off) {
+				r = m;
+			} else if (pad >= krecord[m].end_off) {
+				l = m + 1;
+			} else {
+				verbose(env,
+					"cleanup_info[%u]: landing pad %u is inside the call-site range of cleanup_info[%u]\n",
+					i, pad, m);
+				goto err_free;
+			}
+		}
+	}
+
+	env->cleanup_info = krecord;
+	env->cleanup_info_cnt = nrec;
+	return 0;
+
+err_free:
+	kvfree(krecord);
+	return ret;
+}
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
new file mode 100644
index 000000000000..cf099dcc5b74
--- /dev/null
+++ b/kernel/bpf/exception.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+#ifndef __BPF_EXCEPTION_H
+#define __BPF_EXCEPTION_H
+
+#include <linux/bpfptr.h>
+#include <linux/types.h>
+
+union bpf_attr;
+struct bpf_verifier_env;
+
+int bpf_exc_check_info(struct bpf_verifier_env *env, const union bpf_attr *attr,
+		       bpfptr_t uattr);
+
+#endif /* __BPF_EXCEPTION_H */
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 654f896af265..10e0a693a06d 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2948,7 +2948,7 @@ int __init __used bpf_multi_func(void) { return 0; }
 BTF_ID_LIST_GLOBAL_SINGLE(bpf_multi_func_btf_id, func, bpf_multi_func)
 
 /* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD keyring_id
+#define BPF_PROG_LOAD_LAST_FIELD cleanup_info_cnt
 
 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
 {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 353bde9ae227..60413bf0ad3f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -37,6 +37,7 @@
 
 #include "diagnostics.h"
 #include "disasm.h"
+#include "exception.h"
 
 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
@@ -22706,6 +22707,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	ret = bpf_exc_check_info(env, attr, uattr);
+	if (ret < 0)
+		goto skip_full_check;
+
 	/* Validate instructions and resolve the program's referenced resources. */
 	ret = check_and_resolve_insns(env);
 	if (ret < 0)
@@ -22923,6 +22928,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	kvfree(env->callx_edges);
 	kvfree(env->func_ptrs);
 	bpf_diag_free(env);
+	kvfree(env->cleanup_info);
 	kvfree(env);
 	return ret;
 }
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index e0ed44b1bbcb..15dfba087201 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1708,6 +1708,9 @@ union bpf_attr {
 		 * verification.
 		 */
 		__s32		keyring_id;
+		__aligned_u64	cleanup_info;	/* exception cleanup table */
+		__u32		cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+		__u32		cleanup_info_cnt; /* number of bpf_cleanup_info records */
 	};
 
 	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7644,6 +7647,17 @@ struct bpf_line_info {
 	__u32	line_col;
 };
 
+/*
+ * One record of an exception cleanup table: calls in [begin_off, end_off)
+ * that unwind resume at landing_pad_off. All three are instruction offsets
+ * in the program as loaded.
+ */
+struct bpf_cleanup_info {
+	__u32	begin_off;
+	__u32	end_off;
+	__u32	landing_pad_off;
+};
+
 struct bpf_spin_lock {
 	__u32	val;
 };
-- 
2.53.0-Meta