[PATCH bpf-next v4 01/20] bpf: Accept the compiler's exception cleanup table at program load

[Yonghong Song <yonghong.song@linux.dev>]

LLVM 23 added exception handling support for BPF with the .bpf_cleanup
section ([1]). Rust code compiled with panic=unwind runs cleanup code
(Drop glue) when bpf_throw() fires, and the LLVM BPF backend emits that
section from the landing pads the frontend produced. Plain C cannot
generate .bpf_cleanup unless inline asm is used. The Rust compiler does not
*properly* support BPF exception handling yet, but the kernel can support
the table today, and inline assembly is enough to test it.

Add the UAPI to carry the .bpf_cleanup table into the kernel. BPF_PROG_LOAD
grows cleanup_info, cleanup_info_cnt and cleanup_info_rec_size, and struct
bpf_cleanup_info describes one record as a triple of instruction indices:
the half-open call-site range [begin_off, end_off) and the landing_pad_off
the frame resumes at. check_cleanup_info() validates the table a program is
loaded with, so the rest of the kernel can rely on it.

Link: https://github.com/llvm/llvm-project/pull/192164 [1]
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf_verifier.h   |   2 +
 include/uapi/linux/bpf.h       |   9 ++
 kernel/bpf/check_btf.c         | 151 +++++++++++++++++++++++++++++++++
 kernel/bpf/syscall.c           |   2 +-
 kernel/bpf/verifier.c          |   1 +
 tools/include/uapi/linux/bpf.h |   9 ++
 6 files changed, 173 insertions(+), 1 deletion(-)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 92f528c45605..de2cff5e3eca 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -987,6 +987,8 @@ struct bpf_verifier_env {
 	struct arg_track **callsite_at_stack;
 	u32 pass_cnt; /* number of times do_check() was called */
 	u32 subprog_cnt;
+	struct bpf_cleanup_info *cleanup_info;
+	u32 cleanup_info_cnt;
 	/* number of instructions analyzed by the verifier */
 	u32 prev_insn_processed, insn_processed;
 	/* number of jmps, calls, exits analyzed so far */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 6330b7d745c5..a69cc0427575 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1669,6 +1669,9 @@ union bpf_attr {
 		 * verification.
 		 */
 		__s32		keyring_id;
+		__aligned_u64	cleanup_info;	/* exception cleanup table */
+		__u32		cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+		__u32		cleanup_info_cnt; /* number of bpf_cleanup_info records */
 	};
 
 	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7599,6 +7602,12 @@ struct bpf_line_info {
 	__u32	line_col;
 };
 
+struct bpf_cleanup_info {
+	__u32	begin_off;
+	__u32	end_off;
+	__u32	landing_pad_off;
+};
+
 struct bpf_spin_lock {
 	__u32	val;
 };
diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c
index 0e8b3ccc7a5b..17af0a414d74 100644
--- a/kernel/bpf/check_btf.c
+++ b/kernel/bpf/check_btf.c
@@ -407,6 +407,153 @@ static int check_core_relo(struct bpf_verifier_env *env,
 	return err;
 }
 
+static int cleanup_insn_subprog(struct bpf_verifier_env *env, u32 off)
+{
+	struct bpf_subprog_info *info;
+
+	if (off >= env->prog->len)
+		return -1;
+	info = bpf_find_containing_subprog(env, off);
+	return info ? info - env->subprog_info : -1;
+}
+
+#define MIN_BPF_CLEANUP_INFO_SIZE	12
+#define MAX_CLEANUP_INFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE
+
+static int check_cleanup_info(struct bpf_verifier_env *env,
+			      const union bpf_attr *attr,
+			      bpfptr_t uattr)
+{
+	u32 krec_size = sizeof(struct bpf_cleanup_info);
+	u32 i, nrec, urec_size, min_size, prev_end = 0;
+	struct bpf_cleanup_info *krecord;
+	bpfptr_t urecord;
+	int ret = -EINVAL;
+
+	nrec = attr->cleanup_info_cnt;
+	if (!nrec)
+		return 0;
+	/* Disjoint ranges, so no more records than instructions. */
+	if (nrec > env->prog->len) {
+		verbose(env, "cleanup info has %u records for %u instructions\n",
+			nrec, env->prog->len);
+		return -EINVAL;
+	}
+
+	urec_size = attr->cleanup_info_rec_size;
+	if (urec_size < MIN_BPF_CLEANUP_INFO_SIZE ||
+	    urec_size > MAX_CLEANUP_INFO_REC_SIZE ||
+	    urec_size % sizeof(u32)) {
+		verbose(env, "invalid cleanup info rec size %u\n", urec_size);
+		return -EINVAL;
+	}
+
+	krecord = kvcalloc(nrec, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+	if (!krecord)
+		return -ENOMEM;
+
+	min_size = min_t(u32, krec_size, urec_size);
+	urecord = make_bpfptr(attr->cleanup_info, uattr.is_kernel);
+	for (i = 0; i < nrec; i++) {
+		struct bpf_cleanup_info *rec = &krecord[i];
+		int sb, se, sl;
+
+		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
+		if (ret) {
+			if (ret == -E2BIG) {
+				verbose(env, "nonzero tailing record in cleanup info\n");
+				if (copy_to_bpfptr_offset(uattr,
+							  offsetof(union bpf_attr,
+								   cleanup_info_rec_size),
+							  &min_size, sizeof(min_size)))
+					ret = -EFAULT;
+			}
+			goto err_free;
+		}
+
+		if (copy_from_bpfptr(rec, urecord, min_size)) {
+			ret = -EFAULT;
+			goto err_free;
+		}
+		bpfptr_add(&urecord, urec_size);
+
+		ret = -EINVAL;
+		if (rec->begin_off >= rec->end_off) {
+			verbose(env, "cleanup_info[%u]: begin %u >= end %u\n",
+				i, rec->begin_off, rec->end_off);
+			goto err_free;
+		}
+		if (i && rec->begin_off < prev_end) {
+			verbose(env,
+				"cleanup_info[%u]: range [%u,%u) is unsorted or overlaps the previous record\n",
+				i, rec->begin_off, rec->end_off);
+			goto err_free;
+		}
+		prev_end = rec->end_off;
+
+		sb = cleanup_insn_subprog(env, rec->begin_off);
+		se = cleanup_insn_subprog(env, rec->end_off - 1);
+		sl = cleanup_insn_subprog(env, rec->landing_pad_off);
+		if (sb < 0 || se < 0 || sl < 0) {
+			verbose(env, "cleanup_info[%u]: offset out of range\n", i);
+			goto err_free;
+		}
+		if (sb != se || sb != sl) {
+			verbose(env,
+				"cleanup_info[%u]: range/landing pad span multiple subprogs\n",
+				i);
+			goto err_free;
+		}
+		/*
+		 * The second half of a 16-byte instruction carries a zero
+		 * opcode and is not an instruction of its own, so no offset
+		 * may name one. end_off is exclusive, so it may also be one
+		 * past the last instruction of the program.
+		 */
+		if (!env->prog->insnsi[rec->begin_off].code ||
+		    !env->prog->insnsi[rec->landing_pad_off].code ||
+		    (rec->end_off < env->prog->len &&
+		     !env->prog->insnsi[rec->end_off].code)) {
+			verbose(env, "cleanup_info[%u]: points at invalid insn\n", i);
+			goto err_free;
+		}
+	}
+
+	/*
+	 * Reject a landing pad that lies inside a call-site range, its own
+	 * included: it would be both a pad and a call that unwinds to one, and
+	 * an exception out of it would have nowhere to go.
+	 */
+	ret = -EINVAL;
+	for (i = 0; i < nrec; i++) {
+		u32 pad = krecord[i].landing_pad_off;
+		u32 l = 0, r = nrec;
+
+		while (l < r) {
+			u32 m = l + (r - l) / 2;
+
+			if (pad < krecord[m].begin_off) {
+				r = m;
+			} else if (pad >= krecord[m].end_off) {
+				l = m + 1;
+			} else {
+				verbose(env,
+					"cleanup_info[%u]: landing pad %u is inside the call-site range of cleanup_info[%u]\n",
+					i, pad, m);
+				goto err_free;
+			}
+		}
+	}
+
+	env->cleanup_info = krecord;
+	env->cleanup_info_cnt = nrec;
+	return 0;
+
+err_free:
+	kvfree(krecord);
+	return ret;
+}
+
 int bpf_prepare_btf_info(struct bpf_verifier_env *env,
 			 const union bpf_attr *attr,
 			 bpfptr_t uattr)
@@ -441,6 +588,10 @@ int bpf_check_btf_info(struct bpf_verifier_env *env,
 {
 	int err;
 
+	err = check_cleanup_info(env, attr, uattr);
+	if (err)
+		return err;
+
 	if (!attr->func_info_cnt && !attr->line_info_cnt) {
 		if (check_abnormal_return(env))
 			return -EINVAL;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 113486b15d29..049717dc3b7d 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2915,7 +2915,7 @@ int __init __used bpf_multi_func(void) { return 0; }
 BTF_ID_LIST_GLOBAL_SINGLE(bpf_multi_func_btf_id, func, bpf_multi_func)
 
 /* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD keyring_id
+#define BPF_PROG_LOAD_LAST_FIELD cleanup_info_cnt
 
 static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
 {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index d62c0f74cff5..f6878f903bc1 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21919,6 +21919,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	kvfree(env->succ);
 	kvfree(env->gotox_tmp_buf);
 	bpf_diag_free(env);
+	kvfree(env->cleanup_info);
 	kvfree(env);
 	return ret;
 }
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 6330b7d745c5..a69cc0427575 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1669,6 +1669,9 @@ union bpf_attr {
 		 * verification.
 		 */
 		__s32		keyring_id;
+		__aligned_u64	cleanup_info;	/* exception cleanup table */
+		__u32		cleanup_info_rec_size; /* userspace bpf_cleanup_info size */
+		__u32		cleanup_info_cnt; /* number of bpf_cleanup_info records */
 	};
 
 	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -7599,6 +7602,12 @@ struct bpf_line_info {
 	__u32	line_col;
 };
 
+struct bpf_cleanup_info {
+	__u32	begin_off;
+	__u32	end_off;
+	__u32	landing_pad_off;
+};
+
 struct bpf_spin_lock {
 	__u32	val;
 };
-- 
2.53.0-Meta