[PATCH bpf-next v6 08/21] bpf: Refuse a landing pad that does not resume

[Yonghong Song <yonghong.song@linux.dev>]

A cleanup pad runs drop glue and calls bpf_unwind_resume(), so the frame
returns and the unwind goes on. A catch pad -- std::panic::catch_unwind and
its kin -- runs the same drops and then carries on in its frame, stopping
the unwind there. Only the first is supported: bpf_unwind() rewrites every
frame's return address in one pass, so a frame above a catch pad would
resume at a pad for an unwind that had already been caught.

Nothing in the record says which kind a pad is, but the code does, exactly
as LLVM emits it -- a cleanup pad reaches _Unwind_Resume and a catch pad
reaches a return. bpf_unwind() and the branch pushed at a covered call are
the only ways into a pad, and both mark the frame they enter.
bpf_exc_check_insn() asks that mark about every instruction of a program
that carries a table, and refuses:

 - an exit, which is how a catch pad ends
 - a tail call, which replaces the frame, and a BPF_LD_[ABS|IND], which on
   a failed load leaves through the "r0 = 0; exit" gen_ld_abs() patches in
 - an indirect jump, which nothing a frontend emits in a pad needs
 - a bpf_unwind(), which starts a second walk over frames the first has
   already rewritten, and a call to a global subprogram that might_unwind,
   which is verified on its own and cannot be walked into from here
 - a bpf_unwind_resume() outside a pad, where the fixups would lower it to
   a bare return rather than to a resume
 - an instruction reached both inside and outside a pad, which is a jump
   into a pad: the cleanup would run with nothing to clean up after

The first three are about the pad's own frame, since a subprogram the pad
calls may do any of them and still come back; the unwind is refused
anywhere above a pad, since it never does. do_check() asks before it may
prune the path, so an instruction is in a pad or it is not, never both, and
the mark need not join the comparison in states_equal().

A callback that can unwind is refused here too: an unwind out of one stops
at the helper's own frame, which is C and has no landing pad, so the helper
would carry on as though nothing had happened.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf_verifier.h |  4 ++
 kernel/bpf/exception.c       | 81 ++++++++++++++++++++++++++++++++++++
 kernel/bpf/exception.h       |  3 ++
 kernel/bpf/verifier.c        | 18 ++++++++
 4 files changed, 106 insertions(+)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 0143688896b0..4174c7d0177e 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -339,6 +339,8 @@ struct bpf_func_state {
 	bool in_async_callback_fn;
 	bool in_exception_callback_fn;
 	bool no_stack_arg_load;
+	/* an unwind reached this frame and its landing pad is running */
+	bool in_pad;
 	/* For callback calling functions that limit number of possible
 	 * callback executions (e.g. bpf_loop) keeps track of current
 	 * simulated iteration number.
@@ -698,6 +700,8 @@ struct bpf_insn_aux_data {
 	u64 non_stack_access:1; /* instruction can access non-stack memory */
 	/* true if some jump or call instruction targets this instruction */
 	u64 jump_target:1;
+	u64 in_cleanup_pad:1; /* reached with a landing pad running */
+	u64 outside_cleanup_pad:1; /* ... and the other way round */
 
 	unsigned int orig_idx; /* original instruction index, initialized once */
 	/*
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
index 1ed0370a171b..86fdf847d33e 100644
--- a/kernel/bpf/exception.c
+++ b/kernel/bpf/exception.c
@@ -12,6 +12,15 @@
 BTF_ID_LIST_SINGLE(bpf_unwind_id, func, bpf_unwind)
 BTF_ID_LIST_SINGLE(bpf_unwind_resume_id, func, bpf_unwind_resume)
 
+int bpf_exc_check_callback(struct bpf_verifier_env *env, int subprog)
+{
+	if (!env->subprog_info[subprog].might_unwind)
+		return 0;
+
+	verbose(env, "subprog %d may unwind and is used as a callback\n", subprog);
+	return -EINVAL;
+}
+
 static void mark_call_sites(struct bpf_verifier_env *env)
 {
 	u32 i, j;
@@ -69,6 +78,78 @@ bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
 	       insn->imm == bpf_unwind_resume_id[0];
 }
 
+/* Is an unwind in flight: is this frame a landing pad, or below one? */
+static bool unwinding(const struct bpf_verifier_state *state)
+{
+	u32 i;
+
+	for (i = 0; i <= state->curframe; i++)
+		if (state->frame[i]->in_pad)
+			return true;
+	return false;
+}
+
+int bpf_exc_check_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
+{
+	bool in_pad = cur_func(env)->in_pad;
+	struct bpf_insn_aux_data *aux;
+	u32 i = env->insn_idx;
+	const char *why = NULL;
+
+	if (unwinding(env->cur_state)) {
+		if (bpf_is_unwind_kfunc(insn)) {
+			verbose(env, "insn %u starts a second unwind while one is in flight\n", i);
+			return -EINVAL;
+		}
+		if (bpf_pseudo_call(insn)) {
+			int subprog = bpf_find_subprog(env, i + insn->imm + 1);
+
+			if (subprog >= 0 && bpf_subprog_is_global(env, subprog) &&
+			    env->subprog_info[subprog].might_unwind) {
+				verbose(env,
+					"insn %u calls global subprog %d, which can unwind while an unwind is in flight\n",
+					i, subprog);
+				return -EINVAL;
+			}
+		}
+	}
+
+	aux = &env->insn_aux_data[i];
+
+	if (in_pad ? aux->outside_cleanup_pad : aux->in_cleanup_pad) {
+		verbose(env, "insn %u runs both inside and outside a landing pad\n", i);
+		return -EINVAL;
+	}
+	if (in_pad)
+		aux->in_cleanup_pad = true;
+	else
+		aux->outside_cleanup_pad = true;
+
+	if (!in_pad)
+		return 0;
+
+	if (insn->code == (BPF_JMP | BPF_EXIT)) {
+		verbose(env,
+			"exit at insn %u ends a landing pad: a catch pad is not supported yet, only cleanup pads that resume\n",
+			i);
+		return -EOPNOTSUPP;
+	}
+	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call)
+		why = "is a tail call, which replaces the frame";
+	else if (BPF_CLASS(insn->code) == BPF_LD &&
+		 (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND))
+		why = "is a BPF_LD_[ABS|IND], which can leave through the epilogue";
+	else if (insn->code == (BPF_JMP | BPF_JA | BPF_X) ||
+		 insn->code == (BPF_JMP32 | BPF_JA | BPF_X))
+		why = "is an indirect jump";
+
+	if (!why)
+		return 0;
+
+	verbose(env, "insn %u %s, and is in a landing pad\n", i, why);
+	return -EINVAL;
+}
+
 int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx)
 {
 	u32 pad = env->insn_aux_data[idx].cleanup_pad;
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
index b96b2c429e22..f3aff0fe8ecc 100644
--- a/kernel/bpf/exception.h
+++ b/kernel/bpf/exception.h
@@ -6,8 +6,11 @@
 #include <linux/types.h>
 
 struct bpf_verifier_env;
+struct bpf_insn;
 
 int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env);
 int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx);
+int bpf_exc_check_callback(struct bpf_verifier_env *env, int subprog);
+int bpf_exc_check_insn(struct bpf_verifier_env *env, struct bpf_insn *insn);
 
 #endif /* _LINUX_BPF_EXCEPTION_H */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 77176250f866..cc1ed776da15 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -10985,6 +10985,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 	 * callbacks
 	 */
 	env->subprog_info[subprog].is_cb = true;
+	err = bpf_exc_check_callback(env, subprog);
+	if (err)
+		return err;
+
 	if (bpf_pseudo_kfunc_call(insn) &&
 	    !is_callback_calling_kfunc(insn->imm)) {
 		verifier_bug(env, "kfunc %s#%d not marked as callback-calling",
@@ -19185,6 +19189,7 @@ static int push_cleanup_pad_branch(struct bpf_verifier_env *env, int insn_idx)
 	 */
 	clear_caller_saved_regs(env, frame->regs);
 	mark_reg_unknown(env, frame->regs, BPF_REG_0);
+	frame->in_pad = true;
 	return 0;
 }
 
@@ -19197,6 +19202,7 @@ static int process_bpf_unwind(struct bpf_verifier_env *env, int *insn_idx)
 		return PROCESS_BPF_EXIT;
 	clear_caller_saved_regs(env, frame->regs);
 	mark_reg_unknown(env, frame->regs, BPF_REG_0);
+	frame->in_pad = true;
 	*insn_idx = pad;
 	return INSN_IDX_UPDATED;
 }
@@ -19441,6 +19447,11 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 			if (bpf_is_unwind_kfunc(insn))
 				return process_bpf_unwind(env, &env->insn_idx);
 			if (bpf_is_unwind_resume_kfunc(insn)) {
+				if (!cur_func(env)->in_pad) {
+					verbose(env, "resume at insn %d is not in a landing pad\n",
+						env->insn_idx);
+					return -EINVAL;
+				}
 				/*
 				 * Mark r0 a known zero -- unknown first, as
 				 * the known-zero helper keeps the type it
@@ -19450,6 +19461,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 				 */
 				mark_reg_unknown(env, cur_regs(env), BPF_REG_0);
 				mark_reg_known_zero(env, cur_regs(env), BPF_REG_0);
+				cur_func(env)->in_pad = false;
 				return process_bpf_exit_full(env, do_print_state, false);
 			}
 			if (env->cur_state->active_locks) {
@@ -19578,6 +19590,12 @@ static int do_check(struct bpf_verifier_env *env)
 			}
 		}
 
+		if (unlikely(env->cleanup_info_cnt)) {
+			err = bpf_exc_check_insn(env, insn);
+			if (err)
+				return err;
+		}
+
 		if (bpf_is_prune_point(env, env->insn_idx)) {
 			err = bpf_is_state_visited(env, env->insn_idx);
 			if (err < 0)
-- 
2.53.0-Meta