[PATCH bpf-next v4 07/13] bpf, x86: Convert struct_ops arena arguments in the trampoline

[Kumar Kartikeya Dwivedi <memxor@gmail.com>]

From: Tejun Heo <tj@kernel.org>

Implement the struct_ops arena argument conversion on x86. save_args()
receives the arena base from bpf_tramp_arena_base() and consults the
btf_func_model argument flags as it copies each native argument into the
BPF ctx, routing a marked argument through RAX:

  movl %esrc, %eax      /* truncate and clear the upper 32 bits */
  subl $base_lo, %eax
  movq %rax, ctx_slot

A nullable argument tests the full 64-bit kernel pointer first:

  movq  %rsrc, %rax
  testq %rax, %rax
  jz    1f
  subl  $base_lo, %eax
1:
  movq  %rax, ctx_slot

The 32-bit subtraction is sufficient since (u32)(kaddr - base) ==
(u32)kaddr - (u32)base, and it clears the upper half as the JITs require
of arena pointer registers. Stack-passed arguments already reload
through RAX, so only the subtraction (and the NULL test) is inserted
there.

Keeping arena and nullable classification in btf_func_model avoids a
parallel trampoline slot bitmap. bpf_tramp_arena_base() returns a base
only for a single-program struct_ops indirect trampoline; other
trampolines pass zero and perform no conversion. The size probe reruns
the same emission with the same model and nodes, so the image size
matches by construction.

With both the kfunc and struct_ops directions implemented, flip
bpf_jit_supports_arena_args() on for x86.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
---
 arch/x86/net/bpf_jit_comp.c | 57 +++++++++++++++++++++++++++++++++----
 include/linux/bpf.h         | 22 ++++----------
 kernel/bpf/bpf_struct_ops.c |  1 -
 kernel/bpf/trampoline.c     | 41 +++++++++++---------------
 4 files changed, 74 insertions(+), 47 deletions(-)

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 817977797e59..cb8ee0f3a642 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3043,8 +3043,35 @@ static int get_nr_used_regs(const struct btf_func_model *m)
 	return nr_used_regs;
 }
 
+/*
+ * Convert an arena kernel address into the arena pointer form on its way
+ * into the BPF ctx, rax = (u32)(src - kern_vm_start). A nullable arg
+ * preserves NULL, tested on the full 64-bit kernel pointer. The 32-bit
+ * subtraction both truncates and clears the upper half, so the stored
+ * value satisfies the JIT invariant for arena pointer registers.
+ */
+static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base_lo)
+{
+	u8 *prog = *pprog;
+
+	if (nullable) {
+		if (src_reg != BPF_REG_0)
+			emit_mov_reg(&prog, true, BPF_REG_0, src_reg);
+		/* test rax, rax; jz over the 5-byte sub */
+		EMIT3(0x48, 0x85, 0xC0);
+		EMIT2(X86_JE, 5);
+	} else if (src_reg != BPF_REG_0) {
+		emit_mov_reg(&prog, false, BPF_REG_0, src_reg);
+	}
+	/* sub eax, base_lo */
+	EMIT1_off32(0x2D, base_lo);
+
+	*pprog = prog;
+}
+
 static void save_args(const struct btf_func_model *m, u8 **prog,
-		      int stack_size, bool for_call_origin, u32 flags)
+		      int stack_size, bool for_call_origin, u32 flags,
+		      u64 arena_base)
 {
 	int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0;
 	bool use_jmp = bpf_trampoline_use_jmp(flags);
@@ -3056,6 +3083,9 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
 	 * mov QWORD PTR [rbp-0x8],rsi
 	 */
 	for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
+		bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
+		bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
+
 		arg_regs = (m->arg_size[i] + 7) / 8;
 
 		/* According to the research of Yonghong, struct members
@@ -3089,6 +3119,9 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
 			for (j = 0; j < arg_regs; j++) {
 				emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP,
 					 nr_stack_slots * 8 + 16 + (!use_jmp) * 8);
+				if (arena_arg)
+					emit_arena_arg_conv(prog, BPF_REG_0, nullable,
+							    (u32)arena_base);
 				emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0,
 					 -stack_size);
 
@@ -3109,9 +3142,13 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
 
 			/* copy the arguments from regs into stack */
 			for (j = 0; j < arg_regs; j++) {
-				emit_stx(prog, BPF_DW, BPF_REG_FP,
-					 nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs,
-					 -stack_size);
+				u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs;
+
+				if (arena_arg) {
+					emit_arena_arg_conv(prog, src, nullable, (u32)arena_base);
+					src = BPF_REG_0;
+				}
+				emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size);
 				stack_size -= 8;
 				nr_regs++;
 			}
@@ -3407,6 +3444,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 	void *orig_call = func_addr;
 	int cookie_off, cookie_cnt;
 	u8 **branches = NULL;
+	u64 arena_base;
 	u64 func_meta;
 	u8 *prog;
 	bool save_ret;
@@ -3419,6 +3457,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 	WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) &&
 		     (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET)));
 
+	arena_base = bpf_tramp_arena_base(m, tnodes, flags);
+
 	for (i = 0; i < m->nr_args; i++)
 		nr_regs += (m->arg_size[i] + 7) / 8 - 1;
 
@@ -3553,7 +3593,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr);
 	}
 
-	save_args(m, &prog, regs_off, false, flags);
+	save_args(m, &prog, regs_off, false, flags, arena_base);
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		/* arg1: mov rdi, im */
@@ -3595,7 +3635,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		restore_regs(m, &prog, regs_off);
-		save_args(m, &prog, arg_stack_off, true, flags);
+		save_args(m, &prog, arg_stack_off, true, flags, 0);
 
 		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
 			/* Before calling the original function, load the
@@ -4096,6 +4136,11 @@ bool bpf_jit_supports_stack_args(void)
 	return true;
 }
 
+bool bpf_jit_supports_arena_args(void)
+{
+	return true;
+}
+
 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 {
 	if (text_poke_copy(dst, src, len) == NULL)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 0d1773af8373..00aaa23b4f7e 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1290,18 +1290,13 @@ struct bpf_tramp_nodes {
 };
 
 /*
- * Which 8-byte ctx slots of a struct_ops trampoline hold arena kernel
- * pointers that save_args() converts to the arena pointer form,
- * ctx[slot] = (u32)(kaddr - kern_vm_start).
+ * The arena base against which a struct_ops trampoline converts the
+ * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF
+ * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline
+ * converts nothing.
  */
-struct bpf_tramp_arena_args {
-	u32 slots;
-	u32 nullable_slots;	/* subset of @slots where NULL is preserved */
-	u64 kern_vm_start;
-};
-
-bool bpf_tramp_collect_arena_args(struct bpf_tramp_nodes *tnodes, u32 flags,
-				  struct bpf_tramp_arena_args *aargs);
+u64 bpf_tramp_arena_base(const struct btf_func_model *m,
+			 struct bpf_tramp_nodes *tnodes, u32 flags);
 
 struct bpf_tramp_run_ctx;
 
@@ -1704,11 +1699,6 @@ struct bpf_ctx_arg_aux {
 	u32 btf_id;
 	u32 ref_id;
 	bool refcounted;
-	/*
-	 * We don't encode NULL-ness in the type for the program, but still need
-	 * to distinguish it for the purposes of telling JITs what sequence to emit.
-	 */
-	bool arena_nullable;
 };
 
 struct btf_mod_pair {
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index ebfb7fff453e..ced7b18246a9 100644
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c
@@ -288,7 +288,6 @@ static int prepare_arg_info(struct btf *btf,
 			 * precision around it, since it has no safety implication.
 			 */
 			info->reg_type = PTR_TO_ARENA;
-			info->arena_nullable = is_arena_nullable;
 			model->arg_flags[arg_no] |= BTF_FMODEL_ARENA_ARG;
 			if (is_arena_nullable)
 				model->arg_flags[arg_no] |= BTF_FMODEL_NULLABLE_ARG;
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 67d0ba8a05cd..b349e0817184 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -540,40 +540,33 @@ static bool bpf_prog_has_arena_ctx_arg(const struct bpf_prog *prog)
 }
 
 /*
- * Collect which ctx slots of a struct_ops trampoline hold arena kernel
- * pointers that save_args() must convert to the arena pointer form. Only
- * the struct_ops indirect trampoline converts: it dispatches to a single
- * prog whose arena is known at generation time. Return false when there
- * is nothing to convert.
+ * The arena base against which save_args() converts the arguments marked
+ * with BTF_FMODEL_ARENA_ARG. Only the struct_ops indirect trampoline
+ * converts: it dispatches to a single prog whose arena is known at
+ * generation time. Return 0 when there is nothing to convert.
  */
-bool bpf_tramp_collect_arena_args(struct bpf_tramp_nodes *tnodes, u32 flags,
-				  struct bpf_tramp_arena_args *aargs)
+u64 bpf_tramp_arena_base(const struct btf_func_model *m,
+			 struct bpf_tramp_nodes *tnodes, u32 flags)
 {
 	const struct bpf_prog *prog;
 	int i;
 
-	memset(aargs, 0, sizeof(*aargs));
-
 	if (!(flags & BPF_TRAMP_F_INDIRECT) ||
 	    tnodes[BPF_TRAMP_FENTRY].nr_nodes != 1)
-		return false;
+		return 0;
 
-	prog = tnodes[BPF_TRAMP_FENTRY].nodes[0]->link->prog;
-	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
-		const struct bpf_ctx_arg_aux *info = &prog->aux->ctx_arg_info[i];
+	for (i = 0; i < m->nr_args; i++)
+		if (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG)
+			break;
+	if (i == m->nr_args)
+		return 0;
 
-		if (base_type(info->reg_type) != PTR_TO_ARENA)
-			continue;
-		aargs->slots |= BIT(info->offset / 8);
-		if (info->arena_nullable)
-			aargs->nullable_slots |= BIT(info->offset / 8);
-	}
-	if (!aargs->slots)
-		return false;
+	/* Verification rejects an arena argument without an arena. */
+	prog = tnodes[BPF_TRAMP_FENTRY].nodes[0]->link->prog;
 	if (WARN_ON_ONCE(!prog->aux->arena))
-		return false;
-	aargs->kern_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
-	return true;
+		return 0;
+
+	return bpf_arena_get_kern_vm_start(prog->aux->arena);
 }
 
 static void bpf_tramp_image_free(struct bpf_tramp_image *im)
-- 
2.53.0