[PATCH bpf-next 11/20] bpf, x86: Dispatch exception cleanup pads at run time

[Yonghong Song <yonghong.song@linux.dev>]

Provide the arch half for x86-64: force the full callee-saved spill for a
program carrying a cleanup table so the walker can find a frame's r6-r9 in
its callee's prologue, record where that spill area starts, build the
native cleanup table from the JIT's addrs[], emit a bare return for a pad's
bpf_unwind_resume(), and hand control to a pad from
arch_bpf_run_cleanup_pad().

Support is gated on CONFIG_UNWINDER_ORC, the same requirement
arch_bpf_stack_walk() and therefore bpf_throw() already have here.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 arch/x86/net/Makefile          |  2 +-
 arch/x86/net/bpf_cleanup_pad.S | 74 ++++++++++++++++++++++++++++++++
 arch/x86/net/bpf_jit_comp.c    | 78 ++++++++++++++++++++++++++++------
 3 files changed, 141 insertions(+), 13 deletions(-)
 create mode 100644 arch/x86/net/bpf_cleanup_pad.S

diff --git a/arch/x86/net/Makefile b/arch/x86/net/Makefile
index dddbefc0f439..9d574d972df3 100644
--- a/arch/x86/net/Makefile
+++ b/arch/x86/net/Makefile
@@ -6,5 +6,5 @@
 ifeq ($(CONFIG_X86_32),y)
         obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
 else
-        obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
+        obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o bpf_cleanup_pad.o
 endif
diff --git a/arch/x86/net/bpf_cleanup_pad.S b/arch/x86/net/bpf_cleanup_pad.S
new file mode 100644
index 000000000000..da4b448ecf09
--- /dev/null
+++ b/arch/x86/net/bpf_cleanup_pad.S
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bpf_cleanup_abi.h>
+#include <linux/linkage.h>
+#include <asm/nospec-branch.h>
+
+/*
+ * The x86-64 BPF JIT prologue spills, once bpf_cleanup_force_spill() makes it
+ * unconditional, r12, rbx, r13, r14 and r15 in that order -- so within the
+ * spill area the lowest address holds r15 and the highest r12. The throw-site
+ * spill the JIT emits uses the same layout, so the routine below reads both
+ * the same way:
+ *
+ *	spill_base + 0		BPF r9  (r15)
+ *	spill_base + 8		BPF r8  (r14)
+ *	spill_base + 16		BPF r7  (r13)
+ *	spill_base + 24		BPF r6  (rbx)
+ *	spill_base + 32		r12 (arena base, not a BPF register)
+ */
+
+	.code64
+	.section .text, "ax"
+
+/*
+ * void arch_bpf_run_cleanup_pad(u64 pad, u64 frame_fp, u64 spill_base)
+ *
+ * rdi = native address of the landing pad
+ * rsi = frame pointer of the frame the pad belongs to
+ * rdx = spill area holding that frame's BPF callee-saved registers
+ *
+ * Give the pad the register state of its own frame and call it. The pad ends
+ * in the bare return the JIT emits for its bpf_unwind_resume(), so it comes
+ * back here rather than returning to its frame's caller. It runs on this
+ * stack, far below the frame it is cleaning up after, so nothing it calls can
+ * reach into that frame.
+ */
+SYM_FUNC_START(arch_bpf_run_cleanup_pad)
+	ANNOTATE_NOENDBR
+
+	pushq %rbp
+	pushq %rbx
+	pushq %r12
+	pushq %r13
+	pushq %r14
+	pushq %r15
+	/* Keep the pad's entry rsp congruent to a normal call's. */
+	subq $8, %rsp
+
+	movq 0(%rdx), %r15
+	movq 8(%rdx), %r14
+	movq 16(%rdx), %r13
+	movq 24(%rdx), %rbx
+	movq 32(%rdx), %r12
+	/* rbp is BPF r10, so this is the whole of the pad's frame setup. */
+	movq %rsi, %rbp
+
+	/* CALL_NOSPEC needs the target in a register; rcx is BPF r4, dead. */
+	movq %rdi, %rcx
+
+	/* BPF r0 on the way into a pad, not whatever the kernel left in rax. */
+	movl $BPF_PAD_ENTRY_R0, %eax
+
+	CALL_NOSPEC rcx
+
+	addq $8, %rsp
+	popq %r15
+	popq %r14
+	popq %r13
+	popq %r12
+	popq %rbx
+	popq %rbp
+	RET
+SYM_FUNC_END(arch_bpf_run_cleanup_pad)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index d4a980140b48..9d0dd54773e8 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -357,6 +357,11 @@ struct jit_context {
 /* Number of bytes that will be skipped on tailcall */
 #define X86_TAIL_CALL_OFFSET	(12 + ENDBR_INSN_SIZE)
 
+/* Throw-site spill: r15, r14, r13, rbx, r12 low to high, the layout the
+ * prologue's pushes leave, so arch_bpf_run_cleanup_pad() reads both alike.
+ */
+#define X86_CLEANUP_SPILL_SZ	(5 * 8)
+
 static void push_r9(u8 **pprog)
 {
 	u8 *prog = *pprog;
@@ -832,7 +837,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog,
 	/* Inc tail_call_cnt if the slot is populated. */
 	EMIT4(0x48, 0x83, 0x00, 0x01);            /* add qword ptr [rax], 1 */
 
-	if (bpf_prog->aux->exception_boundary) {
+	if (bpf_prog->aux->exception_boundary || bpf_cleanup_force_spill(bpf_prog)) {
 		pop_callee_regs(&prog, all_callee_regs_used);
 		pop_r12(&prog);
 	} else {
@@ -899,7 +904,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog,
 	/* Inc tail_call_cnt if the slot is populated. */
 	EMIT4(0x48, 0x83, 0x00, 0x01);                /* add qword ptr [rax], 1 */
 
-	if (bpf_prog->aux->exception_boundary) {
+	if (bpf_prog->aux->exception_boundary || bpf_cleanup_force_spill(bpf_prog)) {
 		pop_callee_regs(&prog, all_callee_regs_used);
 		pop_r12(&prog);
 	} else {
@@ -1977,6 +1982,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	u8 *ip, *prog = temp;
 	u32 stack_depth;
 	int callee_saved_size;
+	u32 throw_spill, prologue_depth;
 	s32 outgoing_arg_base;
 	int err;
 
@@ -2015,7 +2021,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 
 	detect_reg_usage(insn, insn_cnt, callee_regs_used);
 
-	emit_prologue(&prog, image, stack_depth,
+	throw_spill = bpf_cleanup_force_spill(bpf_prog) ? X86_CLEANUP_SPILL_SZ : 0;
+	prologue_depth = stack_depth + throw_spill;
+
+	emit_prologue(&prog, image, prologue_depth,
 		      bpf_prog_was_classic(bpf_prog), tail_call_reachable,
 		      bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
 
@@ -2024,7 +2033,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	/* Exception callback will clobber callee regs for its own use, and
 	 * restore the original callee regs from main prog's stack frame.
 	 */
-	if (bpf_prog->aux->exception_boundary) {
+	if (bpf_prog->aux->exception_boundary || bpf_cleanup_force_spill(bpf_prog)) {
 		/* We also need to save r12, which is not mapped to any BPF
 		 * register, as we throw after entry into the kernel, which may
 		 * overwrite r12.
@@ -2039,9 +2048,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 
 	/* Compute callee-saved register area size. */
 	callee_saved_size = 0;
-	if (bpf_prog->aux->exception_boundary || arena_vm_start)
+	if (bpf_prog->aux->exception_boundary || bpf_cleanup_force_spill(bpf_prog) ||
+	    arena_vm_start)
 		callee_saved_size += 8; /* r12 */
-	if (bpf_prog->aux->exception_boundary) {
+	if (bpf_prog->aux->exception_boundary || bpf_cleanup_force_spill(bpf_prog)) {
 		callee_saved_size += 4 * 8; /* rbx, r13, r14, r15 */
 	} else {
 		int j;
@@ -2063,7 +2073,19 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	 * Note that tail_call_reachable is guaranteed to be false when
 	 * stack args exist, so tcc pushes need not be accounted for.
 	 */
-	outgoing_arg_base = -(round_up(stack_depth, 8) + callee_saved_size);
+	outgoing_arg_base = -(round_up(stack_depth, 8) + throw_spill + callee_saved_size);
+
+	/*
+	 * Lowest address of each spill area, as an offset from rbp; see
+	 * bpf_cleanup_pad.S for the layout. The 16 is the tail call counter
+	 * pair emit_prologue_tail_call() pushes above the callee-saved one.
+	 */
+	if (bpf_cleanup_force_spill(bpf_prog)) {
+		bpf_prog->aux->exc->spill_off = -(round_up(stack_depth, 8) + throw_spill +
+						  (tail_call_reachable ? 16 : 0) +
+						  callee_saved_size);
+		bpf_prog->aux->exc->throw_spill_off = -(round_up(stack_depth, 8) + throw_spill);
+	}
 
 	/*
 	 * Allocate outgoing stack arg area for args 7+ only.
@@ -2110,7 +2132,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 				dst_reg = X86_REG_R9;
 		}
 
-		if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1))
+		if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1) ||
+		    bpf_cleanup_insn_is_pad(bpf_prog, i - 1))
 			EMIT_ENDBR();
 
 		ip = image + addrs[i - 1] + (prog - temp);
@@ -2903,9 +2926,27 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 		case BPF_JMP | BPF_CALL: {
 			const struct btf_func_model *fm = NULL;
 
+			if (bpf_cleanup_insn_is_throw(bpf_prog, i - 1)) {
+				/* Spill r6-r9 and r12 where the bpf_throw() walker looks. */
+				s32 off = bpf_prog->aux->exc->throw_spill_off;
+				u8 *spill = prog;
+
+				emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_9, off + 0);
+				emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_8, off + 8);
+				emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_7, off + 16);
+				emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, off + 24);
+				emit_stx(&prog, BPF_DW, BPF_REG_FP, X86_REG_R12, off + 32);
+				ip += prog - spill;
+			}
+
+			if (bpf_is_unwind_resume_kfunc(insn)) {
+				emit_return(&prog, image + addrs[i - 1] + (prog - temp));
+				break;
+			}
+
 			func = (u8 *) __bpf_call_base + imm32;
 			if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
-				LOAD_TAIL_CALL_CNT_PTR(stack_depth);
+				LOAD_TAIL_CALL_CNT_PTR(prologue_depth);
 				ip += 7;
 			}
 			if (!imm32)
@@ -2948,13 +2989,13 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 							  &prog,
 							  ip,
 							  callee_regs_used,
-							  stack_depth,
+							  prologue_depth,
 							  ctx);
 			else
 				emit_bpf_tail_call_indirect(bpf_prog,
 							    &prog,
 							    callee_regs_used,
-							    stack_depth,
+							    prologue_depth,
 							    ip,
 							    ctx);
 			break;
@@ -3215,7 +3256,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 			}
 			/* Deallocate outgoing args 7+ area. */
 			emit_add_rsp(&prog, outgoing_rsp);
-			if (bpf_prog->aux->exception_boundary) {
+			if (bpf_prog->aux->exception_boundary ||
+			    bpf_cleanup_force_spill(bpf_prog)) {
 				pop_callee_regs(&prog, all_callee_regs_used);
 				pop_r12(&prog);
 			} else {
@@ -4385,6 +4427,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 		 */
 		bpf_prog_update_insn_ptrs(prog, addrs, image);
 
+		/*
+		 * Same mapping, consumed by the bpf_throw() frame walker:
+		 * turn the cleanup records into native address ranges now
+		 * that the image is final.
+		 */
+		bpf_cleanup_fill_native_ranges(prog, addrs, image);
+
 		/*
 		 * ctx.prog_offset is used when CFI preambles put code *before*
 		 * the function. See emit_cfi(). For FineIBT specifically this code
@@ -4501,6 +4550,11 @@ bool bpf_jit_supports_exceptions(void)
 	return IS_ENABLED(CONFIG_UNWINDER_ORC);
 }
 
+bool bpf_jit_supports_cleanup_pads(void)
+{
+	return IS_ENABLED(CONFIG_UNWINDER_ORC);
+}
+
 bool bpf_jit_supports_private_stack(void)
 {
 	return true;
-- 
2.53.0-Meta