From: Qiliang Yuan <odys.yuan@gmail.com>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
John Fastabend <john.fastabend@gmail.com>,
Andrii Nakryiko <andrii@kernel.org>,
Eduard Zingerman <eddyz87@gmail.com>,
Kumar Kartikeya Dwivedi <memxor@gmail.com>,
Martin KaFai Lau <martin.lau@linux.dev>,
Song Liu <song@kernel.org>,
Yonghong Song <yonghong.song@linux.dev>,
Jiri Olsa <jolsa@kernel.org>,
Emil Tsalapatis <emil@etsalapatis.com>,
Ihor Solodrai <ihor.solodrai@linux.dev>,
Shuah Khan <shuah@kernel.org>
Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-kselftest@vger.kernel.org,
Qiliang Yuan <odys.yuan@gmail.com>
Subject: [PATCH bpf-next v2 14/22] bpf: Move bpf_convert_ctx_accesses() to the patch list
Date: Fri, 30 Oct 2026 07:51:59 -0400 [thread overview]
Message-ID: <20261030-bpf-verifier-patch-batch-v2-14-55cd43439f7c@gmail.com> (raw)
In-Reply-To: <20261030-bpf-verifier-patch-batch-v2-0-55cd43439f7c@gmail.com>
bpf_convert_ctx_accesses() patches the program for every context
access that it rewrites and every barrier that it adds, one
bpf_patch_insn_data() at a time.
Each of these patches costs O(prog->len), so a program with many
context accesses pays it once for every one of them.
Queue the prologue, the store of the context pointer for the epilogue
and the rewrites, and commit them at the end of the pass. Queue a
barrier in front of an instruction before rewriting the instruction,
whose rewrite then replaces the last instruction of the barrier. As
ops->gen_prologue() ends the prologue with the first instruction of the
program, insert the prologue without it with bpf_patch_list_insert().
As the epilogue is now found by its index in the unpatched program,
which can be 0, mark that there is none yet with -1.
Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
kernel/bpf/fixups.c | 132 +++++++++++++++++++---------------------------------
1 file changed, 49 insertions(+), 83 deletions(-)
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 6d99de1be2581..08c89f2baffb6 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1090,16 +1090,15 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprogs = env->subprog_info;
const struct bpf_verifier_ops *ops = env->ops;
- int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
+ int i, cnt, size, ctx_field_size, ret, epilogue_cnt = 0;
const int insn_cnt = env->prog->len;
struct bpf_insn *epilogue_buf = env->epilogue_buf;
struct bpf_insn *insn_buf = env->insn_buf;
struct bpf_insn *insn;
u32 target_size, size_default, off;
- struct bpf_prog *new_prog;
enum bpf_access_type type;
bool is_narrower_load;
- int epilogue_idx = 0;
+ int epilogue_idx = -1;
if (ops->gen_epilogue) {
epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,
@@ -1109,17 +1108,12 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
return -EFAULT;
} else if (epilogue_cnt) {
/* Save the ARG_PTR_TO_CTX for the epilogue to use */
- cnt = 0;
subprogs[0].stack_depth += 8;
- insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
- -subprogs[0].stack_depth);
- insn_buf[cnt++] = env->prog->insnsi[0];
- new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
- BPF_PATCH_MOVE_TARGET);
- if (!new_prog)
- return -ENOMEM;
- env->prog = new_prog;
- delta += cnt - 1;
+ insn_buf[0] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
+ -subprogs[0].stack_depth);
+ ret = bpf_patch_list_insert(env, 0, insn_buf, 1);
+ if (ret)
+ return ret;
ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
if (ret < 0)
@@ -1150,13 +1144,14 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
env->prog->aux->attach_btf_id))
return -EINVAL;
env->prog->aux->has_prologue = true;
- new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
- BPF_PATCH_MOVE_TARGET);
- if (!new_prog)
- return -ENOMEM;
-
- env->prog = new_prog;
- delta += cnt - 1;
+ /* the prologue ends with the first insn of the program */
+ if (verifier_bug_if(memcmp(&insn_buf[cnt - 1], env->prog->insnsi,
+ sizeof(*insn_buf)), env,
+ "prologue doesn't end with the first insn"))
+ return -EFAULT;
+ ret = bpf_patch_list_insert(env, 0, insn_buf, cnt - 1);
+ if (ret)
+ return ret;
ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
if (ret < 0)
@@ -1164,21 +1159,18 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
}
}
- if (delta)
- WARN_ON(adjust_jmp_off(env->prog, 0, delta));
-
if (bpf_prog_is_offloaded(env->prog->aux))
- return 0;
-
- insn = env->prog->insnsi + delta;
+ return bpf_patch_list_commit(env);
- for (i = 0; i < insn_cnt; i++, insn++) {
+ for (i = 0; i < insn_cnt; i++) {
bpf_convert_ctx_access_t convert_ctx_access;
enum bpf_reg_type ptr_type;
u8 mode;
- if (env->insn_aux_data[i + delta].nospec) {
- WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);
+ insn = &env->prog->insnsi[i];
+
+ if (env->insn_aux_data[i].nospec) {
+ WARN_ON_ONCE(env->insn_aux_data[i].alu_state);
struct bpf_insn *patch = insn_buf;
*patch++ = BPF_ST_NOSPEC();
@@ -1186,17 +1178,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
ret = bpf_adj_moved_insn(patch++, 1);
if (ret) {
verbose(env, "insn %d cannot be patched due to 16-bit range\n",
- env->insn_aux_data[i + delta].orig_idx);
+ env->insn_aux_data[i].orig_idx);
return ret;
}
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
/* This can not be easily merged with the
* nospec_result-case, because an insn may require a
* nospec before and after itself. Therefore also do not
@@ -1226,42 +1214,34 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) ||
insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
- env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
+ env->insn_aux_data[i].ptr_type == PTR_TO_ARENA) {
insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);
env->prog->aux->num_exentries++;
continue;
} else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) &&
insn->imm == BPF_LOAD_ACQ &&
- env->insn_aux_data[i + delta].arena_member) {
+ env->insn_aux_data[i].arena_member) {
/* an acquire load of an __arena member is rewritten like a plain load */
- cnt = arena_member_fixup(env, i + delta, insn, insn_buf);
+ cnt = arena_member_fixup(env, i, insn, insn_buf);
if (cnt < 0)
return cnt;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
continue;
} else if (insn->code == (BPF_JMP | BPF_EXIT) &&
epilogue_cnt &&
- i + delta < subprogs[1].start) {
+ i < subprogs[1].start) {
/* Generate epilogue for the main prog */
- if (epilogue_idx) {
+ if (epilogue_idx >= 0) {
/* jump back to the earlier generated epilogue */
- insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1);
+ insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - 1);
cnt = 1;
} else {
memcpy(insn_buf, epilogue_buf,
epilogue_cnt * sizeof(*epilogue_buf));
cnt = epilogue_cnt;
- /* epilogue_idx cannot be 0. It must have at
- * least one ctx ptr saving insn before the
- * epilogue.
- */
- epilogue_idx = i + delta;
+ epilogue_idx = i;
}
goto patch_insn_buf;
} else {
@@ -1269,7 +1249,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
}
if (type == BPF_WRITE &&
- env->insn_aux_data[i + delta].nospec_result) {
+ env->insn_aux_data[i].nospec_result) {
/* nospec_result is only used to mitigate Spectre v4 and
* to limit verification-time for Spectre v1.
*/
@@ -1278,17 +1258,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
*patch++ = *insn;
*patch++ = BPF_ST_NOSPEC();
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
continue;
}
- ptr_type = env->insn_aux_data[i + delta].ptr_type;
+ ptr_type = env->insn_aux_data[i].ptr_type;
switch ((int)ptr_type) {
case PTR_TO_CTX:
if (!ops->convert_ctx_access)
@@ -1341,22 +1317,18 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
return -EFAULT;
}
- cnt = arena_member_fixup(env, i + delta, insn, insn_buf);
+ cnt = arena_member_fixup(env, i, insn, insn_buf);
if (cnt < 0)
return cnt;
if (!cnt)
continue;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
continue;
}
- ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
+ ctx_field_size = env->insn_aux_data[i].ctx_field_size;
size = BPF_LDST_BYTES(insn);
mode = BPF_MODE(insn->code);
@@ -1424,18 +1396,12 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
size * 8, 0);
patch_insn_buf:
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
-
- /* keep walking new program and skip insns we just inserted */
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
}
- return 0;
+ return bpf_patch_list_commit(env);
}
static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
--
2.43.0
next prev parent reply other threads:[~2026-10-30 11:53 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-30 11:51 [PATCH bpf-next v2 00/22] bpf: Replace bpf_patch_insn_data() with a list of deferred patches Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 01/22] bpf: Add a list of deferred instruction patches Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 02/22] bpf: Remap the verifier state through the patch list Qiliang Yuan
2026-10-30 12:44 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 03/22] bpf: Apply single-instruction patches of the verifier in place Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 04/22] bpf: Queue the rewrites of bpf_do_misc_fixups() Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 05/22] bpf: Expand a backward may_goto to gotol when patches push it out of range Qiliang Yuan
2026-10-30 12:59 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 06/22] bpf: Drop the constant delta from bpf_do_misc_fixups() Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 07/22] bpf: Let the patch list remove instructions Qiliang Yuan
2026-10-30 12:11 ` sashiko-bot
2026-10-30 11:51 ` [PATCH bpf-next v2 08/22] bpf: Remove dead code through the patch list Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 09/22] bpf: Strip nops " Qiliang Yuan
2026-10-30 12:45 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 10/22] bpf: Batch the patches of bpf_opt_subreg_zext_lo32_rnd_hi32() Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 11/22] bpf: Allow patching an instruction twice through the patch list Qiliang Yuan
2026-10-30 11:51 ` [PATCH bpf-next v2 12/22] bpf: Teach the patch list to insert in front of an instruction Qiliang Yuan
2026-10-30 12:44 ` bot+bpf-ci
2026-10-30 11:51 ` [PATCH bpf-next v2 13/22] bpf: Allow inserting in front of a queued patch Qiliang Yuan
2026-10-30 11:51 ` Qiliang Yuan [this message]
2026-10-30 12:24 ` [PATCH bpf-next v2 14/22] bpf: Move bpf_convert_ctx_accesses() to the patch list sashiko-bot
2026-10-30 11:52 ` [PATCH bpf-next v2 15/22] bpf: Insert the may_goto counter setup through " Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 16/22] bpf: Append the hidden subprog " Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 17/22] bpf: Inline bpf_loop() calls " Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 18/22] bpf: Apply the patch list without a verifier env Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 19/22] bpf: Blind constants through the patch list Qiliang Yuan
2026-10-30 12:44 ` bot+bpf-ci
2026-10-30 11:52 ` [PATCH bpf-next v2 20/22] bpf: Remove the old instruction patching code Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 21/22] selftests/bpf: Test jumps around patches of bpf_do_misc_fixups() Qiliang Yuan
2026-10-30 11:52 ` [PATCH bpf-next v2 22/22] selftests/bpf: Cover a backward may_goto pushed out of range by patches Qiliang Yuan
2026-10-30 12:13 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261030-bpf-verifier-patch-batch-v2-14-55cd43439f7c@gmail.com \
--to=odys.yuan@gmail.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=emil@etsalapatis.com \
--cc=ihor.solodrai@linux.dev \
--cc=john.fastabend@gmail.com \
--cc=jolsa@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=martin.lau@linux.dev \
--cc=memxor@gmail.com \
--cc=shuah@kernel.org \
--cc=song@kernel.org \
--cc=yonghong.song@linux.dev \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox