[PATCH bpf-next v11 12/26] bpf: Keep an epilogue for each subprog an unwind can pass through

[Yonghong Song <yonghong.song@linux.dev>]

A later patch makes bpf_unwind() walk the BPF frames on the stack. A
frame whose call site has a landing pad continues at the pad. Any other
frame is sent to its epilogue: its saved return address is rewritten to
point there, and the frame returns to its caller from it. The x86 JIT
emits a function's epilogue at its first exit insn, so every subprog the
walk can pass through needs an exit that is still there when it is JITed.
There are two ways it can be missing.

First, dead code removal deletes an exit the verifier never reached. In a
subprog whose path to its exit always goes through a call that unwinds,
that is every exit it has.

Second, a compiler emits no exit for a function that never returns. Built
with panic=unwind, rustc compiles

        #[inline(never)]
        fn fail() -> ! { unsafe { bpf_unwind() }; loop {} }

to

        fail:
                call bpf_unwind
        L0:     goto L0

bpf_exc_patch_unwind_calls() later puts 'r0 = 0; exit' after the
bpf_unwind() call, and fail() returns through it, since the walk leaves
the return address of the frame that called bpf_unwind() alone. A
function that calls fail() and then loops the same way, though, has no
exit at all for the walk to send it to.

Add keep_subprog_exits(), which dead code removal calls before removing
anything. For each subprog that can unwind and that the verifier reached,
it marks the last exit as seen, so that the exit is kept. If the subprog
has no exit, it puts one after the subprog's last insn. Only the walk
ever gets there. keep_subprog_exits() runs before the exits after
bpf_unwind() and bpf_unwind_resume() calls are added, so a frame like
fail() gets one too, which is never reached.

The exit is put in by patching the last insn into itself followed by the
exit. The patch leaves most of the last insn's aux data on the exit, so
it is put back on the last insn, its nospec mark and kfunc data
included.

Once the next patch lets a subprog end with a call, that insn may be a
call to a subprog placed after the frame. The exit moves the callee down
by one insn, so adjust the call's offset.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 kernel/bpf/fixups.c | 60 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 57 insertions(+), 3 deletions(-)

diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 158a0e033dbb..e634d838cf5f 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -734,13 +734,67 @@ static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env)
 	}
 }
 
+/*
+ * The walk sends a frame that a callee unwinds through to its epilogue, which
+ * the x86 JIT emits at the first exit, so keep one. A frame that only unwinds
+ * may have none: put one after its last insn.
+ */
+static int keep_subprog_exits(struct bpf_verifier_env *env)
+{
+	struct bpf_insn patch[2] = { {}, BPF_EXIT_INSN() };
+	struct bpf_prog *new_prog;
+	u32 i, j;
+
+	for (i = 0; i < env->subprog_cnt; i++) {
+		u32 start, end;
+		bool found = false;
+
+		if (!env->subprog_info[i].might_unwind)
+			continue;
+		start = env->subprog_info[i].start;
+		if (!env->insn_aux_data[start].seen)
+			continue;
+
+		end = env->subprog_info[i + 1].start;
+		for (j = end; j-- > start; ) {
+			if (env->prog->insnsi[j].code == (BPF_JMP | BPF_EXIT)) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			struct bpf_insn_aux_data aux = env->insn_aux_data[end - 1];
+			struct bpf_insn *last;
+
+			patch[0] = env->prog->insnsi[end - 1];
+			new_prog = bpf_patch_insn_data(env, end - 1, patch, 2);
+			if (!new_prog)
+				return -ENOMEM;
+			env->prog = new_prog;
+			env->insn_aux_data[end - 1] = aux;
+			memset(&env->insn_aux_data[end], 0, sizeof(aux));
+			last = &new_prog->insnsi[end - 1];
+			if (bpf_pseudo_call(last) && last->imm >= 0)
+				last->imm++;
+			j = end;
+		}
+		env->insn_aux_data[j].seen = env->pass_cnt;
+	}
+	return 0;
+}
+
 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env)
 {
-	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
-	int insn_cnt = env->prog->len;
-	int i, err;
+	struct bpf_insn_aux_data *aux_data;
+	int i, insn_cnt, err;
 
 	keep_funcs_with_addr_taken(env);
+	err = keep_subprog_exits(env);
+	if (err)
+		return err;
+
+	aux_data = env->insn_aux_data;
+	insn_cnt = env->prog->len;
 
 	for (i = 0; i < insn_cnt; i++) {
 		int j;
-- 
2.53.0-Meta