[PATCH bpf-next 2/4] bpf: Inline bpf_iter_num_next() kfunc

[Puranjay Mohan <puranjay@kernel.org>]

bpf_iter_num_next() is called on every iteration of a bpf_for() loop and
is the hot path of the numeric open-coded iterator. It only advances the
on-stack iterator state and returns a pointer to it, so open-coding it in
the verifier removes a function call from each loop iteration.

Inline it in bpf_fixup_kfunc_call() by replacing the call with an
equivalent instruction sequence. R1 holds the pointer to the on-stack
bpf_iter_num; the returned pointer to s->cur is R1 itself since s->cur is
the first member.

s->cur and s->end are int, so the kfunc's (s64)(s->cur + 1) >= s->end
test is equivalent to a signed 32-bit comparison of (s->cur + 1) against
s->end: s->cur + 1 is computed as a 32-bit int in the kfunc as well, and
sign-extending both sides of a comparison of two int values does not
change its result. The inlined code therefore uses a 32-bit compare and
needs no sign extension.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 kernel/bpf/verifier.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f578cebbade4f..e8b66f56598dd 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19749,6 +19749,33 @@ static int inline_bpf_iter_num_new(struct bpf_insn *insn_buf)
 	return 15;
 }
 
+/*
+ * Inline bpf_iter_num_next(). R1 holds the pointer to the iterator. Keep in
+ * sync with the kfunc in kernel/bpf/bpf_iter.c.
+ */
+static int inline_bpf_iter_num_next(struct bpf_insn *insn_buf)
+{
+	/*
+	 * s->cur and s->end are int, so the (s64)(s->cur + 1) >= s->end check
+	 * is equivalent to a signed 32-bit comparison of (s->cur + 1) against
+	 * s->end and needs no sign extension.
+	 */
+	insn_buf[0] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0);
+	insn_buf[1] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1);
+	insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4);
+	/* if ((s32)(s->cur + 1) >= (s32)s->end) goto done; */
+	insn_buf[3] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3);
+	/* s->cur++; return &s->cur; */
+	insn_buf[4] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0);
+	insn_buf[5] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
+	insn_buf[6] = BPF_JMP_A(2);
+	/* done: s->cur = s->end = 0; return NULL; */
+	insn_buf[7] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0);
+	insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, 0);
+
+	return 9;
+}
+
 int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		     struct bpf_insn *insn_buf, int insn_idx, int *cnt)
 {
@@ -19880,6 +19907,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		*cnt = 6;
 	} else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) {
 		*cnt = inline_bpf_iter_num_new(insn_buf);
+	} else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) {
+		*cnt = inline_bpf_iter_num_next(insn_buf);
 	}
 
 	if (env->insn_aux_data[insn_idx].arg_prog) {
-- 
2.53.0-Meta