[PATCH bpf-next v9 08/23] bpf: Refuse a landing pad that does not resume

[Yonghong Song <yonghong.song@linux.dev>]

A cleanup pad runs drop glue and calls bpf_unwind_resume(), so its frame
returns and the unwind goes on. A catch pad carries on in its frame
instead. Only the first is supported: bpf_unwind() rewrites every frame's
return address in one pass, so a catch pad's caller would still resume
at a pad.

Nothing in the record says which kind a pad is, but the code does: a
cleanup pad reaches _Unwind_Resume, a catch pad a return. So the unwind
marks the frame whose pad it enters, and refuses in the marked code:

  refused                                  where
  ---------------------------------------  ----------------------------
  an exit: how a catch pad ends            the pad's own frame
  a tail call, a BPF_LD_[ABS|IND]: it      the pad's own frame
    leaves through an exit on a failed
    load
  an indirect jump: nothing a compiler     the pad's own frame
    frontend emits in a pad needs one
  a bpf_unwind(), a call to a global       the pad and its callees
    subprog that might_unwind
  a bpf_unwind_resume() outside a pad      any program

The first three are refused only in the pad's own frame: a subprog the
pad calls may do them and still come back.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf_verifier.h |  1 +
 kernel/bpf/exception.c       | 68 ++++++++++++++++++++++++++++++++++++
 kernel/bpf/exception.h       |  2 ++
 kernel/bpf/states.c          |  3 ++
 kernel/bpf/verifier.c        | 24 ++++++++++++-
 5 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index d3740ad20235..6449e4babc60 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -339,6 +339,7 @@ struct bpf_func_state {
 	bool in_async_callback_fn;
 	bool in_exception_callback_fn;
 	bool no_stack_arg_load;
+	bool in_pad;
 	/* For callback calling functions that limit number of possible
 	 * callback executions (e.g. bpf_loop) keeps track of current
 	 * simulated iteration number.
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
index d85ca358b463..6155f5d73420 100644
--- a/kernel/bpf/exception.c
+++ b/kernel/bpf/exception.c
@@ -141,6 +141,15 @@ int bpf_exc_check_info(struct bpf_verifier_env *env, const union bpf_attr *attr,
 BTF_ID_LIST_SINGLE(bpf_unwind_id, func, bpf_unwind)
 BTF_ID_LIST_SINGLE(bpf_unwind_resume_id, func, bpf_unwind_resume)
 
+int bpf_exc_check_callback(struct bpf_verifier_env *env, int subprog)
+{
+	if (!env->subprog_info[subprog].might_unwind)
+		return 0;
+
+	verbose(env, "subprog %d may unwind and is used as a callback\n", subprog);
+	return -EINVAL;
+}
+
 static int reject_throw(struct bpf_verifier_env *env)
 {
 	u32 i;
@@ -223,6 +232,65 @@ bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
 	       insn->imm == bpf_unwind_resume_id[0];
 }
 
+static bool unwinding(const struct bpf_verifier_state *state)
+{
+	u32 i;
+
+	for (i = 0; i <= state->curframe; i++)
+		if (state->frame[i]->in_pad)
+			return true;
+	return false;
+}
+
+int bpf_exc_check_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)
+{
+	bool in_pad = cur_func(env)->in_pad;
+	u32 i = env->insn_idx;
+	const char *why = NULL;
+
+	if (unwinding(env->cur_state)) {
+		if (bpf_is_unwind_kfunc(insn)) {
+			verbose(env, "insn %u starts a second unwind while one is in flight\n", i);
+			return -EINVAL;
+		}
+		if (bpf_pseudo_call(insn)) {
+			int subprog = bpf_find_subprog(env, i + insn->imm + 1);
+
+			if (subprog >= 0 && bpf_subprog_is_global(env, subprog) &&
+			    env->subprog_info[subprog].might_unwind) {
+				verbose(env,
+					"insn %u calls global subprog %d, which can unwind while an unwind is in flight\n",
+					i, subprog);
+				return -EINVAL;
+			}
+		}
+	}
+
+	if (!in_pad)
+		return 0;
+
+	if (insn->code == (BPF_JMP | BPF_EXIT)) {
+		verbose(env,
+			"exit at insn %u ends a landing pad: a catch pad is not supported yet, only cleanup pads that resume\n",
+			i);
+		return -EINVAL;
+	}
+	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call)
+		why = "is a tail call, which replaces the frame";
+	else if (BPF_CLASS(insn->code) == BPF_LD &&
+		 (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND))
+		why = "is a BPF_LD_[ABS|IND], which can leave through the epilogue";
+	else if (insn->code == (BPF_JMP | BPF_JA | BPF_X) ||
+		 insn->code == (BPF_JMP32 | BPF_JA | BPF_X))
+		why = "is an indirect jump, which nothing a compiler frontend emits in a pad needs";
+
+	if (!why)
+		return 0;
+
+	verbose(env, "insn %u %s, and is in a landing pad\n", i, why);
+	return -EINVAL;
+}
+
 int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx)
 {
 	u32 pad = env->insn_aux_data[idx].cleanup_pad;
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
index 9b75e556c82e..1d7fc795619f 100644
--- a/kernel/bpf/exception.h
+++ b/kernel/bpf/exception.h
@@ -17,5 +17,7 @@ int bpf_exc_check_prog(struct bpf_verifier_env *env);
 int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx);
 bool bpf_is_unwind_kfunc(const struct bpf_insn *insn);
 bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn);
+int bpf_exc_check_callback(struct bpf_verifier_env *env, int subprog);
+int bpf_exc_check_insn(struct bpf_verifier_env *env, struct bpf_insn *insn);
 
 #endif /* __BPF_EXCEPTION_H */
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 4ffa5b7ebf0f..eaf95d6e81bc 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -956,6 +956,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
 	if (!old->no_stack_arg_load && cur->no_stack_arg_load)
 		return false;
 
+	if (old->in_pad != cur->in_pad)
+		return false;
+
 	for (i = 0; i < MAX_BPF_REG; i++)
 		if (((1 << i) & live_regs) &&
 		    !regsafe(env, &old->regs[i], &cur->regs[i],
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1553c7d5bbdb..59b275bfc949 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -10997,6 +10997,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 	 * callbacks
 	 */
 	env->subprog_info[subprog].is_cb = true;
+	err = bpf_exc_check_callback(env, subprog);
+	if (err)
+		return err;
+
 	if (bpf_pseudo_kfunc_call(insn) &&
 	    !is_callback_calling_kfunc(insn->imm)) {
 		verifier_bug(env, "kfunc %s#%d not marked as callback-calling",
@@ -15239,8 +15243,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 
 	if (bpf_is_unwind_kfunc(insn))
 		return process_bpf_unwind(env, insn_idx_p, NULL);
-	if (bpf_is_unwind_resume_kfunc(insn))
+	if (bpf_is_unwind_resume_kfunc(insn)) {
+		if (!cur_func(env)->in_pad) {
+			verbose(env, "resume at insn %d is not in a landing pad\n", insn_idx);
+			return -EINVAL;
+		}
 		return unwind_frames(env, NULL);
+	}
 
 	return 0;
 }
@@ -19247,6 +19256,7 @@ static int unwind_frames(struct bpf_verifier_env *env, bool *do_print_state)
 			return err;
 		clear_caller_saved_regs(env, caller->regs);
 		mark_reg_unknown(env, caller->regs, BPF_REG_0);
+		caller->in_pad = true;
 		env->insn_idx = pad;
 		if (do_print_state)
 			*do_print_state = true;
@@ -19322,6 +19332,7 @@ static int process_global_call_unwind(struct bpf_verifier_env *env,
 	frame = cur_func(env);
 	clear_caller_saved_regs(env, frame->regs);
 	mark_reg_unknown(env, frame->regs, BPF_REG_0);
+	frame->in_pad = true;
 	env->insn_idx = pad;
 	*do_print_state = true;
 	return INSN_IDX_UPDATED;
@@ -19337,6 +19348,7 @@ static int process_bpf_unwind(struct bpf_verifier_env *env, int *insn_idx,
 		return unwind_frames(env, do_print_state);
 	clear_caller_saved_regs(env, frame->regs);
 	mark_reg_unknown(env, frame->regs, BPF_REG_0);
+	frame->in_pad = true;
 	*insn_idx = pad;
 	return INSN_IDX_UPDATED;
 }
@@ -19708,6 +19720,16 @@ static int do_check(struct bpf_verifier_env *env)
 			}
 		}
 
+		if (unlikely(env->cleanup_info_cnt)) {
+			err = bpf_exc_check_insn(env, insn);
+			if (error_recoverable_with_nospec(err) && state->speculative) {
+				insn_aux->nospec = true;
+				goto process_bpf_exit;
+			}
+			if (err)
+				return err;
+		}
+
 		if (bpf_is_prune_point(env, env->insn_idx)) {
 			err = bpf_is_state_visited(env, env->insn_idx);
 			if (err < 0)
-- 
2.53.0-Meta