[PATCH bpf-next v1 4/6] bpf, x86: Add a native bpf_throw() entry that captures R6 to R9

[Kumar Kartikeya Dwivedi <memxor@gmail.com>]

Add bpf_throw_entry, which the verifier calls in place of bpf_throw(). It
saves R6..R9 to the stack in the layout of struct bpf_throw_regs, passes
the cookie from R1 and calls bpf_throw_unwind(), which does not return,
so a ud2 follows the call.

The JIT calls the entry directly through __bpf_call_base + imm, so it
needs no ENDBR. objtool generates ORC for its frame, so the stack walk of
bpf_throw_unwind() passes through the entry to the BPF frames, where the
frame pointer fallback of the ORC unwinder takes over as before.

The entry calls bpf_throw_unwind(), which is built with the syscall, so
it is built with CONFIG_BPF_SYSCALL as well and the JIT returns it only
then. The JIT itself builds with CONFIG_BPF_JIT alone, for cBPF.

Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
---
 arch/x86/net/Makefile       |  1 +
 arch/x86/net/bpf_jit_comp.c |  8 ++++++++
 arch/x86/net/bpf_throw.S    | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 46 insertions(+)
 create mode 100644 arch/x86/net/bpf_throw.S

diff --git a/arch/x86/net/Makefile b/arch/x86/net/Makefile
index dddbefc0f439..daef0aa8b28c 100644
--- a/arch/x86/net/Makefile
+++ b/arch/x86/net/Makefile
@@ -7,4 +7,5 @@ ifeq ($(CONFIG_X86_32),y)
         obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
 else
         obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
+        obj-$(CONFIG_BPF_SYSCALL) += bpf_throw.o
 endif
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 2d7495438d18..1a865313d09e 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -5171,6 +5171,14 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp
 #endif
 }
 
+#ifdef CONFIG_BPF_SYSCALL
+/* arch/x86/net/bpf_throw.S, which calls bpf_throw_unwind(); both need the syscall. */
+void *bpf_arch_throw_entry(void)
+{
+	return (void *)bpf_throw_entry;
+}
+#endif
+
 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
 			       struct bpf_prog *new, struct bpf_prog *old)
 {
diff --git a/arch/x86/net/bpf_throw.S b/arch/x86/net/bpf_throw.S
new file mode 100644
index 000000000000..c70a4002cc8a
--- /dev/null
+++ b/arch/x86/net/bpf_throw.S
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/linkage.h>
+#include <asm/nospec-branch.h>
+
+	.code64
+	.section .text, "ax"
+
+/*
+ * Native entry of the bpf_throw() kfunc. The verifier retargets the call to
+ * bpf_throw() here, so the BPF register file of the throw site is still intact
+ * on entry. Capture its callee-saved registers on the stack as a struct
+ * bpf_throw_regs and hand it to bpf_throw_unwind(), which never returns.
+ */
+SYM_FUNC_START(bpf_throw_entry)
+	ANNOTATE_NOENDBR
+
+	/* Setup frame. */
+	pushq %rbp
+	movq %rsp, %rbp
+
+	/* Save R9..R6 so that [rsp + 8 * i] is saved[i]. */
+	pushq %r15	/* R9 */
+	pushq %r14	/* R8 */
+	pushq %r13	/* R7 */
+	pushq %rbx	/* R6 */
+
+	/* bpf_throw_unwind(regs, cookie); the cookie is R1 of the caller. */
+	movq %rdi, %rsi
+	movq %rsp, %rdi
+
+	/* Emit call depth accounting for call below. */
+	CALL_DEPTH_ACCOUNT
+	call bpf_throw_unwind
+	ud2
+SYM_FUNC_END(bpf_throw_entry)
-- 
2.53.0