[PATCH bpf-next v3 6/7] bpf,x86: Fix exception unwinding with outgoing stack arguments

[Yonghong Song <yonghong.song@linux.dev>]

When a main program with exception_boundary has outgoing stack
arguments (e.g. from calling subprogs with >5 args), bpf_throw() fails
to correctly restore callee-saved registers, causing a kernel crash.

The x86 JIT allocates the outgoing stack arg area below the
callee-saved registers via 'sub rsp, outgoing_rsp' in the prologue.
When bpf_throw() unwinds, it captures the main program's sp (which
includes this outgoing area) and passes it to the exception callback.
The callback gets rsp and rbp, followed by pop_callee_regs, but rsp
points into the outgoing arg area rather than the callee-saved
registers, so the pops restore garbage values. Returning to the
kernel with corrupted callee-saved registers causes a crash.

Fix this by passing the main program's outgoing_rsp as the 4th
argument to the exception callback. The callback adjusts rsp with
'add rsp, rcx' before popping callee-saved registers, correctly
skipping the outgoing arg area. When outgoing_rsp is 0 (the common
case), this is a no-op.

Fixes: 324c3ca6eed6 ("bpf,x86: Implement JIT support for stack arguments")
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 arch/x86/net/bpf_jit_comp.c | 9 ++++++++-
 include/linux/bpf.h         | 3 ++-
 kernel/bpf/fixups.c         | 1 +
 kernel/bpf/helpers.c        | 2 +-
 4 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index ceefefb4da21..f4fdceedaad7 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -557,10 +557,15 @@ static void emit_prologue(u8 **pprog, u8 *ip, u32 stack_depth, bool ebpf_from_cb
 			/* Keep the same instruction layout. */
 			emit_nops(&prog, 3);     /* nop3 */
 	}
-	/* Exception callback receives FP as third parameter */
+	/*
+	 * Exception callback receives:
+	 *   rsi = main program's SP, rdx = main program's FP,
+	 *   rcx = main program's outgoing stack arg area size
+	 */
 	if (is_exception_cb) {
 		EMIT3(0x48, 0x89, 0xF4); /* mov rsp, rsi */
 		EMIT3(0x48, 0x89, 0xD5); /* mov rbp, rdx */
+		EMIT3(0x48, 0x01, 0xCC); /* add rsp, rcx */
 		/* The main frame must have exception_boundary as true, so we
 		 * first restore those callee-saved regs from stack, before
 		 * reusing the stack frame.
@@ -1789,6 +1794,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	 * Arg 6 goes into r9 register, not on stack.
 	 */
 	outgoing_rsp = out_stack_arg_cnt > 1 ? (out_stack_arg_cnt - 1) * 8 : 0;
+	if (bpf_prog->aux->exception_boundary)
+		bpf_prog->aux->stack_arg_adjust = outgoing_rsp;
 	emit_sub_rsp(&prog, outgoing_rsp);
 
 	if (arena_vm_start)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 242f9597d9ab..2a1616c769a9 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1735,7 +1735,8 @@ struct bpf_prog_aux {
 	int cgroup_atype; /* enum cgroup_bpf_attach_type */
 	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
 	char name[BPF_OBJ_NAME_LEN];
-	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
+	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64 stack_arg_adjust, u64);
+	u16 stack_arg_adjust;
 #ifdef CONFIG_SECURITY
 	void *security;
 #endif
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 2cec4e8cd4a0..52aaf2863648 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1265,6 +1265,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 	prog->aux->real_func_cnt = env->subprog_cnt;
 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;
+	prog->aux->stack_arg_adjust = func[0]->aux->stack_arg_adjust;
 	bpf_prog_jit_attempt_done(prog);
 	return 0;
 out_free:
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index baa12b24bb64..53a0430f60b3 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3301,7 +3301,7 @@ __bpf_kfunc void bpf_throw(u64 cookie)
 	 * which skips compiler generated instrumentation to do the same.
 	 */
 	kasan_unpoison_task_stack_below((void *)(long)ctx.sp);
-	ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, 0, 0);
+	ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, ctx.aux->stack_arg_adjust, 0);
 	WARN(1, "A call to BPF exception callback should never return\n");
 }
 
-- 
2.53.0-Meta