From: Qiliang Yuan <odys.yuan@gmail.com>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
John Fastabend <john.fastabend@gmail.com>,
Andrii Nakryiko <andrii@kernel.org>,
Eduard Zingerman <eddyz87@gmail.com>,
Kumar Kartikeya Dwivedi <memxor@gmail.com>,
Martin KaFai Lau <martin.lau@linux.dev>,
Song Liu <song@kernel.org>,
Yonghong Song <yonghong.song@linux.dev>,
Jiri Olsa <jolsa@kernel.org>,
Emil Tsalapatis <emil@etsalapatis.com>,
Ihor Solodrai <ihor.solodrai@linux.dev>,
Shuah Khan <shuah@kernel.org>
Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-kselftest@vger.kernel.org,
Qiliang Yuan <odys.yuan@gmail.com>
Subject: [PATCH bpf-next v2 04/22] bpf: Queue the rewrites of bpf_do_misc_fixups()
Date: Fri, 30 Oct 2026 07:51:49 -0400 [thread overview]
Message-ID: <20261030-bpf-verifier-patch-batch-v2-4-55cd43439f7c@gmail.com> (raw)
In-Reply-To: <20261030-bpf-verifier-patch-batch-v2-0-55cd43439f7c@gmail.com>
Each rewrite in the main loop of bpf_do_misc_fixups() patches the
program right away with bpf_patch_insn_data().
Every rewrite moves the tail of the program and walks all of it, which
costs O(prog->len) each time.
Queue the rewrites with bpf_patch_list_add() and commit them once the
loop is done, before the stack of subprogs is initialized for may_goto.
Walk the unpatched program, so that delta stays 0, and derive insn
from the index at the end of each iteration, so that a rewrite that
fixes up a helper call after queueing it can go on to patch_call_imm
with insn pointing to the queued copy.
perf stat -r30 -- veristat -q <obj>, average of two runs, x86_64 VM
with 32 vCPUs, before and after this patch:
before after
pyperf180 0.507 s 0.348 s
pyperf600 1.406 s 0.887 s
Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
kernel/bpf/fixups.c | 239 +++++++++++++++++-----------------------------------
1 file changed, 75 insertions(+), 164 deletions(-)
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index a0dd582312d86..c9963d8d81356 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -2042,14 +2042,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (env->insn_aux_data[i + delta].arena_scalar) {
cnt = arena_scalar_access(insn, insn_buf);
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- prog = new_prog;
- env->prog = prog;
- insn = prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2072,13 +2067,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2163,13 +2154,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = patch - insn_buf;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2193,13 +2180,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
*patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2213,13 +2196,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
return -EFAULT;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2266,13 +2245,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2309,13 +2284,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt,
tail, ARRAY_SIZE(tail));
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
} else if (bpf_is_may_goto_insn(insn)) {
int stack_off = -stack_depth - 8;
@@ -2328,13 +2299,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = may_goto_expand(insn_buf, insn->off, stack_off,
tail, ARRAY_SIZE(tail));
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2365,13 +2332,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 2;
i++;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2386,13 +2349,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (cnt == 0)
goto next_insn;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2477,13 +2436,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
map)->index_mask);
insn_buf[2] = *insn;
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2511,13 +2466,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[2] = *insn;
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto patch_call_imm;
}
@@ -2530,13 +2481,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[1] = *insn;
cnt = 2;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto patch_call_imm;
}
@@ -2573,14 +2520,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (bpf_map_is_percpu_map(map_ptr->map_type))
prog->jit_required = true;
- new_prog = bpf_patch_insn_data(env, i + delta,
- insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2656,14 +2598,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
BPF_REG_0, 0);
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
- cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2686,13 +2623,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
cnt = 1;
#endif
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2705,13 +2638,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
cnt = 3;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
#endif
@@ -2739,13 +2668,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[cnt++] = BPF_JMP_A(1);
insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2771,13 +2696,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 1;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2797,13 +2718,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
cnt = 2;
}
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2813,12 +2730,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
/* Load IP address from ctx - 16 */
insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
- if (!new_prog)
- return -ENOMEM;
-
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, 1);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2868,13 +2782,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT);
cnt = 11;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
@@ -2886,13 +2796,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
cnt = 2;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
goto next_insn;
}
patch_call_imm:
@@ -2923,9 +2829,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
stack_depth_extra = 0;
}
i++;
- insn++;
+ insn = &prog->insnsi[i + delta];
}
+ ret = bpf_patch_list_commit(env);
+ if (ret)
+ return ret;
+ prog = env->prog;
+
env->prog->aux->stack_depth = subprogs[0].stack_depth;
for (i = 0; i < env->subprog_cnt; i++) {
int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1;
--
2.43.0
next prev parent reply other threads:[~2026-10-30 11:52 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-30 11:51 [PATCH bpf-next v2 00/22] bpf: Replace bpf_patch_insn_data() with a list of deferred patches Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 01/22] bpf: Add a list of deferred instruction patches Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 02/22] bpf: Remap the verifier state through the patch list Qiliang Yuan
2026-10-30 12:44 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 03/22] bpf: Apply single-instruction patches of the verifier in place Qiliang Yuan
2026-10-30 11:51 ` Qiliang Yuan [this message]
2026-10-30 11:51 ` [PATCH bpf-next v2 05/22] bpf: Expand a backward may_goto to gotol when patches push it out of range Qiliang Yuan
2026-10-30 12:59 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 06/22] bpf: Drop the constant delta from bpf_do_misc_fixups() Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 07/22] bpf: Let the patch list remove instructions Qiliang Yuan
2026-10-30 12:11 ` sashiko-bot
2026-10-30 11:51 ` [PATCH bpf-next v2 08/22] bpf: Remove dead code through the patch list Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 09/22] bpf: Strip nops " Qiliang Yuan
2026-10-30 12:45 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 10/22] bpf: Batch the patches of bpf_opt_subreg_zext_lo32_rnd_hi32() Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 11/22] bpf: Allow patching an instruction twice through the patch list Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 12/22] bpf: Teach the patch list to insert in front of an instruction Qiliang Yuan
2026-10-30 12:44 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 13/22] bpf: Allow inserting in front of a queued patch Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 14/22] bpf: Move bpf_convert_ctx_accesses() to the patch list Qiliang Yuan
2026-10-30 12:24 ` sashiko-bot
2026-10-30 11:52 ` [PATCH bpf-next v2 15/22] bpf: Insert the may_goto counter setup through " Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 16/22] bpf: Append the hidden subprog " Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 17/22] bpf: Inline bpf_loop() calls " Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 18/22] bpf: Apply the patch list without a verifier env Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 19/22] bpf: Blind constants through the patch list Qiliang Yuan
2026-10-30 12:44 ` bot+bpf-ci
2026-10-30 11:52 ` [PATCH bpf-next v2 20/22] bpf: Remove the old instruction patching code Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 21/22] selftests/bpf: Test jumps around patches of bpf_do_misc_fixups() Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 22/22] selftests/bpf: Cover a backward may_goto pushed out of range by patches Qiliang Yuan
2026-10-30 12:13 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261030-bpf-verifier-patch-batch-v2-4-55cd43439f7c@gmail.com \
--to=odys.yuan@gmail.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=emil@etsalapatis.com \
--cc=ihor.solodrai@linux.dev \
--cc=john.fastabend@gmail.com \
--cc=jolsa@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=martin.lau@linux.dev \
--cc=memxor@gmail.com \
--cc=shuah@kernel.org \
--cc=song@kernel.org \
--cc=yonghong.song@linux.dev \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox