[PATCH bpf-next v5 09/21] bpf: Refuse exception cleanup shapes bpf_throw() cannot dispatch

[Yonghong Song <yonghong.song@linux.dev>]

The verifier now walks the unwind, so at every instruction it knows whether
an exception is in flight and, if it is, which frame's landing pad it
entered. do_check() asks bpf_exc_check_insn() about each instruction of
a program that carries a table, and it refuses:

  - bpf_throw() in any frame while unwinding: a second unwind over frames
    the first is still discarding. A static callee is walked from the pad,
    so its own throw trips this too; a global one is not walked, which is
    what subprog_info.might_throw is for
  - in the frame whose landing pad the unwind entered, where the code runs
    on the walker's stack: a plain exit, which is what a catch pad ends in,
    a tail call, an indirect jump, an outgoing on-stack call argument, and
    a BPF_LD_[ABS|IND], whose failure path leaves the frame through the
    epilogue
  - an instruction reached both inside and outside a pad: the arm64 JIT
    addresses the frame through a register only a pad entry computes. This
    also turns away a jump into the middle of a pad

bpf_exc_check_callback() refuses a subprogram that may unwind as a
helper callback, whose helper frame would end the walk before it found a
boundary.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf_verifier.h |   2 +
 kernel/bpf/exception.c       | 101 +++++++++++++++++++++++++++++++++++
 kernel/bpf/exception.h       |   3 ++
 kernel/bpf/fixups.c          |   1 +
 kernel/bpf/verifier.c        |  10 ++++
 5 files changed, 117 insertions(+)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 3146f707e030..736304d912af 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -698,6 +698,8 @@ struct bpf_insn_aux_data {
 	u64 arg_prog:4;
 	u64 throw_call:1; /* call to bpf_throw() */
 	u64 resume_call:1; /* call to bpf_unwind_resume() */
+	u64 in_cleanup_pad:1; /* runs with an exception in flight, in the pad's frame */
+	u64 outside_cleanup_pad:1; /* ... and the other way round */
 
 	/* below flags are initialized once */
 	u64 jmp_point:1;
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
index 3270e7ae2e7c..3aafbb612e7f 100644
--- a/kernel/bpf/exception.c
+++ b/kernel/bpf/exception.c
@@ -26,6 +26,15 @@ static void mark_kfunc_sites(struct bpf_verifier_env *env)
 	}
 }
 
+int bpf_exc_check_callback(struct bpf_verifier_env *env, int subprog)
+{
+	if (!env->cleanup_info_cnt || !env->subprog_info[subprog].might_throw)
+		return 0;
+
+	verbose(env, "subprog %d may unwind and is used as a callback\n", subprog);
+	return -EINVAL;
+}
+
 static void mark_call_sites(struct bpf_verifier_env *env)
 {
 	u32 i, j;
@@ -72,6 +81,98 @@ int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env)
 	return 0;
 }
 
+int bpf_exc_check_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
+{
+	struct bpf_verifier_state *state = env->cur_state;
+	struct bpf_insn_aux_data *aux;
+	u32 i = env->insn_idx;
+	bool in_pad;
+
+	aux = &env->insn_aux_data[i];
+	in_pad = state->unwinding && state->curframe == state->unwind_frameno;
+
+	if (in_pad ? aux->outside_cleanup_pad : aux->in_cleanup_pad) {
+		verbose(env,
+			"insn %u runs both inside and outside an exception cleanup landing pad\n",
+			i);
+		return -EINVAL;
+	}
+	if (in_pad)
+		aux->in_cleanup_pad = true;
+	else
+		aux->outside_cleanup_pad = true;
+
+	if (!state->unwinding)
+		return 0;
+
+	if (bpf_is_throw_kfunc(insn)) {
+		verbose(env,
+			"bpf_throw() at insn %u throws while an exception is in flight\n",
+			i);
+		return -EINVAL;
+	}
+	if (bpf_pseudo_call(insn)) {
+		int subprog = bpf_find_subprog(env, i + insn->imm + 1);
+
+		if (subprog >= 0 && bpf_subprog_is_global(env, subprog) &&
+		    env->subprog_info[subprog].might_throw) {
+			verbose(env,
+				"insn %u calls global subprog %d, which can throw while an exception is in flight\n",
+				i, subprog);
+			return -EINVAL;
+		}
+	}
+
+	if (!in_pad)
+		return 0;
+
+	if (insn->code == (BPF_JMP | BPF_EXIT)) {
+		verbose(env,
+			"exit at insn %u ends an exception cleanup landing pad: a catch pad is not supported yet, only cleanup pads that resume\n",
+			i);
+		return -EOPNOTSUPP;
+	}
+	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call) {
+		verbose(env,
+			"bpf_tail_call() at insn %u is in an exception cleanup landing pad\n",
+			i);
+		return -EINVAL;
+	}
+	if (insn->code == (BPF_JMP | BPF_JA | BPF_X) ||
+	    insn->code == (BPF_JMP32 | BPF_JA | BPF_X)) {
+		verbose(env,
+			"indirect jump at insn %u is in an exception cleanup landing pad\n",
+			i);
+		return -EINVAL;
+	}
+	/* A BPF_LD_[ABS|IND] can leave the frame through its epilogue. */
+	if (BPF_CLASS(insn->code) == BPF_LD &&
+	    (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)) {
+		verbose(env,
+			"BPF_LD_[ABS|IND] at insn %u is in an exception cleanup landing pad\n",
+			i);
+		return -EINVAL;
+	}
+	if (is_stack_arg_st(insn) || is_stack_arg_stx(insn)) {
+		verbose(env,
+			"insn %u stages an on-stack call argument in an exception cleanup landing pad\n",
+			i);
+		return -EINVAL;
+	}
+	if (bpf_pseudo_kfunc_call(insn)) {
+		struct bpf_call_summary cs;
+
+		if (bpf_get_call_summary(env, insn, &cs) &&
+		    cs.arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) {
+			verbose(env,
+				"insn %u calls a kfunc with an on-stack argument in an exception cleanup landing pad\n",
+				i);
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
 bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
 {
 	return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
index b96b2c429e22..0872a3f70583 100644
--- a/kernel/bpf/exception.h
+++ b/kernel/bpf/exception.h
@@ -5,9 +5,12 @@
 
 #include <linux/types.h>
 
+struct bpf_insn;
 struct bpf_verifier_env;
 
 int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env);
+int bpf_exc_check_insn(struct bpf_verifier_env *env, struct bpf_insn *insn);
+int bpf_exc_check_callback(struct bpf_verifier_env *env, int subprog);
 int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx);
 
 #endif /* _LINUX_BPF_EXCEPTION_H */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index fed5b66ed3b7..53a737c88dcc 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -252,6 +252,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
 		/* Expand insni[off]'s seen count to the patched range. */
 		data[i].seen = old_seen;
 		data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0;
+		data[i].in_cleanup_pad = data[off + cnt - 1].in_cleanup_pad;
 		if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) {
 			data[i].non_stack_access =
 				data[off + cnt - 1].non_stack_access;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 53edce3d64ea..7b74dd9b9f48 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -10549,6 +10549,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 	 * callbacks
 	 */
 	env->subprog_info[subprog].is_cb = true;
+	err = bpf_exc_check_callback(env, subprog);
+	if (err)
+		return err;
+
 	if (bpf_pseudo_kfunc_call(insn) &&
 	    !is_callback_calling_kfunc(insn->imm)) {
 		verifier_bug(env, "kfunc %s#%d not marked as callback-calling",
@@ -19041,6 +19045,12 @@ static int do_check(struct bpf_verifier_env *env)
 				return err;
 		}
 
+		if (unlikely(env->cleanup_info_cnt)) {
+			err = bpf_exc_check_insn(env, insn);
+			if (err)
+				return err;
+		}
+
 		sanitize_mark_insn_seen(env);
 		prev_insn_idx = env->insn_idx;
 
-- 
2.53.0-Meta