[PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs

Eduard Zingerman
2024-08-17T01:51:58Z

Recognize bpf_fastcall patterns around kfunc calls.
For example, suppose bpf_cast_to_kern_ctx() follows bpf_fastcall
contract (which it does), in such a case allow verifier to rewrite BPF
program below:

r2 = 1;
*(u64 *)(r10 - 32) = r2;
call %[bpf_cast_to_kern_ctx];
r2 = *(u64 *)(r10 - 32);
r0 = r2;

By removing the spill/fill pair:

r2 = 1;
call %[bpf_cast_to_kern_ctx];
r0 = r2;

Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
---
kernel/bpf/verifier.c | 37 ++++++++++++++++++++++++++++++++++++-
1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index b18a21bb5e6e..5dafcfff729e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -16112,7 +16112,7 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,
*/
static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
{
- u8 mask;
+ u32 mask;
int i;

mask = 0;
@@ -16140,6 +16140,28 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
}
}

+/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
+static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
+{
+ const struct btf_param *params;
+ u32 vlen, i, mask;
+
+ params = btf_params(meta->func_proto);
+ vlen = btf_type_vlen(meta->func_proto);
+ mask = 0;
+ if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
+ mask |= BIT(BPF_REG_0);
+ for (i = 0; i < vlen; ++i)
+ mask |= BIT(BPF_REG_1 + i);
+ return mask;
+}
+
+/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
+static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
+{
+ return false;
+}
+
/* LLVM define a bpf_fastcall function attribute.
* This attribute means that function scratches only some of
* the caller saved registers defined by ABI.
@@ -16237,6 +16259,19 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
bpf_jit_inlines_helper_call(call->imm));
}

+ if (bpf_pseudo_kfunc_call(call)) {
+ struct bpf_kfunc_call_arg_meta meta;
+ int err;
+
+ err = fetch_kfunc_meta(env, call, &meta, NULL);
+ if (err < 0)
+ /* error would be reported later */
+ return;
+
+ clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);
+ can_be_inlined = is_fastcall_kfunc_call(&meta);
+ }
+
if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)
return;

--
2.45.2