Re: [PATCH bpf-next 04/10] bpf: Support stack arguments for kfunc calls

Amery Hung
2026-04-02T21:03:00Z

On Wed, Apr 1, 2026 at 6:28 PM Yonghong Song <yonghong.song@linux.dev> wrote:
>
> Extend the stack argument mechanism to kfunc calls, allowing kfuncs
> with more than 5 parameters to receive additional arguments via the
> r12-based stack arg area.
>
> For kfuncs, the caller is a BPF program and the callee is a kernel
> function. The BPF program writes outgoing args at r12-relative offsets
> past its own incoming area.
>
> The following is an example to show how stack arguments are saved:
>
> int foo(int a1, int a2, int a3, int a4, int a5, int a6, int a7) {
> ...
> kfunc1(a1, a2, a3, a4, a5, a6, a7, a8);
> ...
> kfunc2(a1, a2, a3, a4, a5, a6, a7, a8, a9);
> ...
> }
>
> The following is an illustration:
>
> Caller (foo)
> ============
> r12-relative stack arg area:
>
> r12-8: [incoming arg 6]
> r12-16: [incoming arg 7]
>
> ---- incoming/outgoing boundary (kfunc1)
> r12-24: [outgoing arg 6 to callee]
> r12-32: [outgoing arg 7 to callee]
> r12-40: [outgoing arg 8 to callee]
> ...
> Back from kfunc1
> ...
>
> ---- incoming/outgoing boundary
> r12-24: [outgoing arg 6 to callee]
> r12-32: [outgoing arg 7 to callee]
> r12-40: [outgoing arg 8 to callee]
> r12-48: [outgoing arg 9 to callee]
>
> Later JIT will marshal outgoing arguments to the native calling convention
> for kfunc1() and kfunc2().
>
> In check_kfunc_args(), for args beyond the 5th, retrieve the spilled
> register state from the caller's stack arg slots. Temporarily copy
> it into regs[BPF_REG_1] to reuse the existing type checking
> infrastructure, then restore after checking. Also in fixup_kfunc_call(),
> repurpose insn->off (no longer needed after kfunc address resolution)
> to store the number of stack args, so the JIT knows how many args to marshal.
>
> Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
> ---
> kernel/bpf/verifier.c | 97 +++++++++++++++++++++++++++++++++++--------
> 1 file changed, 80 insertions(+), 17 deletions(-)
>
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index d424fe611ef8..6579156486b8 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -3502,7 +3502,7 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
> struct bpf_kfunc_meta kfunc;
> struct bpf_kfunc_desc *desc;
> unsigned long addr;
> - int err;
> + int i, err;
>
> prog_aux = env->prog->aux;
> tab = prog_aux->kfunc_tab;
> @@ -3578,6 +3578,14 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
> if (err)
> return err;
>
> + for (i = MAX_BPF_FUNC_REG_ARGS; i < func_model.nr_args; i++) {
> + if (func_model.arg_size[i] > sizeof(u64)) {
> + verbose(env, "kfunc %s arg#%d size %d > %zu not supported for stack args\n",
> + kfunc.name, i, func_model.arg_size[i], sizeof(u64));
> + return -EINVAL;
> + }
> + }
> +
> desc = &tab->descs[tab->nr_descs++];
> desc->func_id = func_id;
> desc->offset = offset;
> @@ -12995,9 +13003,8 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
> struct bpf_kfunc_call_arg_meta *meta,
> const struct btf_type *t, const struct btf_type *ref_t,
> const char *ref_tname, const struct btf_param *args,
> - int argno, int nargs)
> + int argno, int nargs, u32 regno)
> {
> - u32 regno = argno + 1;
> struct bpf_reg_state *regs = cur_regs(env);
> struct bpf_reg_state *reg = &regs[regno];
> bool arg_mem_size = false;

[...]

> @@ -13677,9 +13684,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
>
> args = (const struct btf_param *)(meta->func_proto + 1);
> nargs = btf_type_vlen(meta->func_proto);
> - if (nargs > MAX_BPF_FUNC_REG_ARGS) {
> + if (nargs > MAX_BPF_FUNC_ARGS) {
> verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
> - MAX_BPF_FUNC_REG_ARGS);
> + MAX_BPF_FUNC_ARGS);
> return -EINVAL;
> }
>
> @@ -13687,13 +13694,41 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> * verifier sees.
> */
> for (i = 0; i < nargs; i++) {
> - struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];
> + struct bpf_reg_state *regs = cur_regs(env), *reg;
> + struct bpf_reg_state saved_reg;
> const struct btf_type *t, *ref_t, *resolve_ret;
> enum bpf_arg_type arg_type = ARG_DONTCARE;
> u32 regno = i + 1, ref_id, type_size;
> bool is_ret_buf_sz = false;
> + bool is_stack_arg = false;
> int kf_arg_type;
>
> + if (i < MAX_BPF_FUNC_REG_ARGS) {
> + reg = &regs[i + 1];
> + } else {
> + /*
> + * Retrieve the spilled reg state from the stack arg slot.
> + * Reuse the existing type checking infrastructure which
> + * reads from cur_regs(env)[regno], temporarily copy the
> + * stack arg reg state into regs[BPF_REG_1] and restore
> + * it after checking.
> + */
> + struct bpf_func_state *caller = cur_func(env);
> + int spi = caller->incoming_stack_arg_depth / BPF_REG_SIZE +
> + (i - MAX_BPF_FUNC_REG_ARGS);
> +
> + if (!is_stack_arg_slot_initialized(caller, spi)) {
> + verbose(env, "stack arg#%d not properly initialized\n", i);
> + return -EINVAL;
> + }
> +
> + is_stack_arg = true;
> + regno = BPF_REG_1;

is_kfunc_arg_mem_size() and is_kfunc_arg_const_mem_size() will not
work properly in get_kfunc_ptr_arg_type(). Since they assume regno + 1
to be the size, R2 would be accidentally treated as size.

Maybe consider creating a helper to retrieve bpf_reg_state of a
specific arg and use it in get_kfunc_ptr_arg_type() instead of copying
and restoring?

> + saved_reg = regs[BPF_REG_1];
> + regs[BPF_REG_1] = caller->stack_arg_slots[spi].spilled_ptr;
> + reg = &regs[BPF_REG_1];
> + }
> +
> if (is_kfunc_arg_prog_aux(btf, &args[i])) {
> /* Reject repeated use bpf_prog_aux */
> if (meta->arg_prog) {
> @@ -13702,7 +13737,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> }
> meta->arg_prog = true;
> cur_aux(env)->arg_prog = regno;
> - continue;
> + goto next_arg;
> }
>
> if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i))
> @@ -13725,9 +13760,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> verbose(env, "R%d must be a known constant\n", regno);
> return -EINVAL;
> }
> - ret = mark_chain_precision(env, regno);
> - if (ret < 0)
> - return ret;
> + if (i < MAX_BPF_FUNC_REG_ARGS) {
> + ret = mark_chain_precision(env, regno);
> + if (ret < 0)
> + return ret;
> + }
> meta->arg_constant.found = true;
> meta->arg_constant.value = reg->var_off.value;
> } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) {
> @@ -13749,11 +13786,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> }
>
> meta->r0_size = reg->var_off.value;
> - ret = mark_chain_precision(env, regno);
> - if (ret)
> - return ret;
> + if (i < MAX_BPF_FUNC_REG_ARGS) {
> + ret = mark_chain_precision(env, regno);
> + if (ret)
> + return ret;
> + }
> }
> - continue;
> + goto next_arg;
> }
>
> if (!btf_type_is_ptr(t)) {
> @@ -13782,13 +13821,14 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
> ref_tname = btf_name_by_offset(btf, ref_t->name_off);
>
> - kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);
> + kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs,
> + regno);
> if (kf_arg_type < 0)
> return kf_arg_type;
>
> switch (kf_arg_type) {
> case KF_ARG_PTR_TO_NULL:
> - continue;
> + goto next_arg;
> case KF_ARG_PTR_TO_MAP:
> if (!reg->map_ptr) {
> verbose(env, "pointer in R%d isn't map pointer\n", regno);
> @@ -14201,6 +14241,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> break;
> }
> }
> +next_arg:
> + if (is_stack_arg)
> + regs[BPF_REG_1] = saved_reg;
> }
>
> if (is_kfunc_release(meta) && !meta->release_regno) {
> @@ -14778,7 +14821,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>
> nargs = btf_type_vlen(meta.func_proto);
> args = (const struct btf_param *)(meta.func_proto + 1);
> - for (i = 0; i < nargs; i++) {
> + for (i = 0; i < nargs && i < MAX_BPF_FUNC_REG_ARGS; i++) {
> u32 regno = i + 1;
>
> t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
> @@ -14789,6 +14832,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> mark_btf_func_reg_size(env, regno, t->size);
> }
>
> + /* Track outgoing stack arg depth for kfuncs with >5 args */
> + if (nargs > MAX_BPF_FUNC_REG_ARGS) {
> + struct bpf_func_state *caller = cur_func(env);
> + struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno];
> + u16 kfunc_stack_arg_depth = (nargs - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE;
> +
> + if (kfunc_stack_arg_depth > caller_info->outgoing_stack_arg_depth)
> + caller_info->outgoing_stack_arg_depth = kfunc_stack_arg_depth;
> + }
> +
> if (is_iter_next_kfunc(&meta)) {
> err = process_iter_next_call(env, insn_idx, &meta);
> if (err)
> @@ -23615,6 +23668,16 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> if (!bpf_jit_supports_far_kfunc_call())
> insn->imm = BPF_CALL_IMM(desc->addr);
>
> + /*
> + * After resolving the kfunc address, insn->off is no longer needed
> + * for BTF fd index. Repurpose it to store the number of stack args
> + * so the JIT can marshal them.
> + */
> + if (desc->func_model.nr_args > MAX_BPF_FUNC_REG_ARGS)
> + insn->off = desc->func_model.nr_args - MAX_BPF_FUNC_REG_ARGS;
> + else
> + insn->off = 0;
> +
> if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) {
> struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
> struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
> --
> 2.52.0
>
>