[PATCH bpf-next v3 1/6] bpf: Inline bpf_iter_num_new() kfunc

[Puranjay Mohan <puranjay@kernel.org>]

The numeric iterator kfuncs bpf_iter_num_{new,next,destroy}() back the
open-coded iterator macro bpf_for() and are emitted as regular kfunc
calls by the verifier. bpf_iter_num_new() is small and only touches the
on-stack iterator state, so the verifier can open-code it and avoid the
call overhead of setting up an iterator.

Inline it in bpf_fixup_kfunc_call() by replacing the call with an
equivalent instruction sequence. R1 holds the pointer to the on-stack
bpf_iter_num, while R2 and R3 hold the start and end arguments. The
arithmetic and the error paths mirror the kfunc exactly, so a program
that inspects the return value keeps observing the same -EINVAL /
-E2BIG / 0 results.

The kfunc guards against overflow with a (s64)end - (s64)start range
check. The start > end case is rejected first, so start <= end at that
point and the range end - start fits in a u32. The inlined code computes
it with a 32-bit subtraction that zero-extends into a 64-bit register
and compares the result against BPF_MAX_LOOPS. Sign-extending the
operands with movsx instead would emit a cpuv4 instruction after
verification; JITs that do not implement movsx (e.g. x86-32, mips32,
sparc64) decode it as a plain move and would miscompile the check.

The emitted instructions are plain BPF and are handled by the
interpreter, so interpreter fallback stays correct and no jit_required
marking is needed.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 kernel/bpf/verifier.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 52be0a118cce0..ff76a1ed04556 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19770,6 +19770,43 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
 	*cnt = 4;
 }
 
+/*
+ * Inline bpf_iter_num_new(). R1 holds the pointer to the iterator, R2 and R3 hold the (int)
+ * start and end arguments. Keep in sync with the kfunc in kernel/bpf/bpf_iter.c.
+ */
+static int inline_bpf_iter_num_new(struct bpf_insn *insn_buf)
+{
+	int i = 0;
+
+	/* if (start > end) goto einval; */
+	insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8);
+	/*
+	 * start <= end here, so the range end - start fits in a u32; compute it as a 32-bit
+	 * subtraction that zero-extends into r0.
+	 */
+	insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3);
+	insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2);
+	/* if (r0 > BPF_MAX_LOOPS) goto e2big; */
+	insn_buf[i++] = BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, BPF_MAX_LOOPS, 8);
+	/* s->cur = start - 1; */
+	insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1);
+	insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0);
+	/* s->end = end; */
+	insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4);
+	/* return 0; */
+	insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0);
+	insn_buf[i++] = BPF_JMP_A(5);
+	/* einval: s->cur = s->end = 0; return -EINVAL; */
+	insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
+	insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
+	insn_buf[i++] = BPF_JMP_A(2);
+	/* e2big: s->cur = s->end = 0; return -E2BIG; */
+	insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
+	insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG);
+
+	return i;
+}
+
 int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		     struct bpf_insn *insn_buf, int insn_idx, int *cnt)
 {
@@ -19899,6 +19936,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1);
 		insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0);
 		*cnt = 6;
+	} else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) {
+		*cnt = inline_bpf_iter_num_new(insn_buf);
 	}
 
 	if (env->insn_aux_data[insn_idx].arg_prog) {
-- 
2.53.0-Meta