[PATCH bpf-next v1 06/14] bpf, s390: Convert struct_ops arena arguments

[Kumar Kartikeya Dwivedi <memxor@gmail.com>]

A struct_ops callback receives a native kernel arena address, while its BPF
program expects the corresponding zero-extended 32-bit arena offset in the
ctx. The s390 trampoline currently copies native arguments verbatim, so it
cannot support callbacks whose stub marks an argument with __arena.

Obtain the arena base for the single-program indirect trampoline and
convert each tagged argument while copying it into the BPF ctx. Test a
nullable source as a full 64-bit kernel pointer, subtract the low 32 bits
of kern_vm_start, and zero-extend the result. This preserves NULL and
provides the register form required by arena loads.

Keep the native argument index separate from the BPF ctx slot index. The
former selects r2-r6 or one caller stack slot per s390 ABI argument, while
the latter still expands 16-byte arguments to two slots. This also converts
arena pointers passed after the fifth argument without shifting later
values.

bpf_tramp_arena_base() returns a base only for an indirect struct_ops
trampoline. Assert the incompatible flag combinations so converted
arguments can never be passed back to the original kernel function, and
advertise the struct_ops-specific arena argument capability.

Cc: Ilya Leoshkevich <iii@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
---
 arch/s390/net/bpf_jit_comp.c | 69 ++++++++++++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 3 deletions(-)

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 20b6e53999b3..964f1ad82637 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2607,6 +2607,34 @@ static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)
 	EMIT6_IMM(0xc00d0000, dst_reg, val);
 }
 
+/*
+ * Convert an arena kernel address into the arena pointer form on its way
+ * into the BPF ctx, dst = (u32)(src - kern_vm_start). A nullable arg
+ * preserves NULL, tested on the full 64-bit kernel pointer. The 32-bit
+ * subtraction followed by zero-extension keeps the upper half clear.
+ */
+static void emit_arena_arg_conv(struct bpf_jit *jit, int dst, int src,
+				bool nullable, u32 base_lo)
+{
+	if (dst != src) {
+		/* lgr %dst,%src */
+		EMIT4(0xb9040000, dst, src);
+	}
+	if (nullable) {
+		/* ltgr %dst,%dst */
+		EMIT4(0xb9020000, dst, dst);
+		/* brc 8,1f */
+		EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg + 16);
+	}
+	/* llilf %w1,base_lo */
+	EMIT6_IMM(0xc00f0000, REG_W1, base_lo);
+	/* sr %dst,%w1 */
+	EMIT2(0x1b00, dst, REG_W1);
+	/* llgfr %dst,%dst */
+	EMIT4(0xb9160000, dst, dst);
+	/* 1: */
+}
+
 static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm)
 {
 	load_imm64(jit, tmp_reg, imm);
@@ -2740,6 +2768,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	int cookie_cnt, cookie_off, fsession_cnt;
 	struct bpf_jit *jit = &tjit->common;
 	int arg, bpf_arg_off;
+	u64 arena_base;
 	u64 func_meta;
 	int i, j;
 
@@ -2749,6 +2778,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	if (nr_stack_args > MAX_NR_STACK_ARGS)
 		return -ENOTSUPP;
 
+	/*
+	 * F_INDIRECT is only compatible with F_RET_FENTRY_RET. Arena conversion
+	 * relies on the indirect trampoline never calling the original function
+	 * with converted arguments.
+	 */
+	WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) &&
+		     (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET)));
+
+	arena_base = bpf_tramp_arena_base(m, tnodes, flags);
+
 	/* Return to %r14 in the struct_ops case. */
 	if (flags & BPF_TRAMP_F_INDIRECT)
 		flags |= BPF_TRAMP_F_SKIP_FRAME;
@@ -2829,14 +2868,33 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 			      (i - MAX_NR_REG_ARGS) * sizeof(u64);
 		bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64);
 		if (m->arg_size[i] <= 8) {
-			if (i < MAX_NR_REG_ARGS)
+			bool arena_arg = arena_base &&
+					 (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
+			bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
+
+			if (arena_arg) {
+				if (i < MAX_NR_REG_ARGS) {
+					emit_arena_arg_conv(jit, REG_W0, arg, nullable,
+							    (u32)arena_base);
+				} else {
+					/* lg %w0,arg(%r15) */
+					EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W0,
+						      REG_0, REG_15, arg);
+					emit_arena_arg_conv(jit, REG_W0, REG_W0,
+							    nullable, (u32)arena_base);
+				}
+				/* stg %w0,bpf_arg_off(%r15) */
+				EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0,
+						      REG_0, REG_15, bpf_arg_off);
+			} else if (i < MAX_NR_REG_ARGS) {
 				/* stg %arg,bpf_arg_off(%r15) */
 				EMIT6_DISP_LH(0xe3000000, 0x0024, arg,
-					      REG_0, REG_15, bpf_arg_off);
-			else
+						      REG_0, REG_15, bpf_arg_off);
+			} else {
 				/* mvc bpf_arg_off(8,%r15),arg(%r15) */
 				_EMIT6(0xd207f000 | bpf_arg_off,
 				       0xf000 | arg);
+			}
 			j += 1;
 		} else {
 			if (i < MAX_NR_REG_ARGS) {
@@ -3092,6 +3150,11 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 	return true;
 }
 
+bool bpf_jit_supports_arena_struct_ops_args(void)
+{
+	return true;
+}
+
 bool bpf_jit_supports_arena(void)
 {
 	return true;
-- 
2.53.0