[PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Puranjay Mohan]

save_args() reads stack-passed arguments relative to FP assuming the
trampoline is entered through the fentry call from a traced function, in
which case both the parent frame (FP/x9) and the traced function frame
(FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a
function pointer (blr), so only the FP/LR frame is pushed and the
arguments start at FP + 16, not FP + 32. Every stack-passed argument of
a struct_ops callback with more than eight argument slots is read two
slots off.

This has gone unnoticed because no in-tree struct_ops member passes
arguments on the stack. Pass is_struct_ops into save_args() and pick the
offset accordingly, mirroring the x86 fix.

Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index c6756b964c843..4fe235f7ce894 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
 static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		      const struct btf_func_model *m,
 		      const struct arg_aux *a,
-		      bool for_call_origin)
+		      bool for_call_origin, bool is_struct_ops)
 {
 	int i;
 	int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		bargs_off += 8;
 	}
 
-	soff = 32; /* on stack arguments start from FP + 32 */
+	/*
+	 * On-stack arguments start above the frame(s) pushed by the
+	 * trampoline prologue. A traced function is entered through the
+	 * fentry call, so both the parent (FP/x9) and the traced function
+	 * (FP/LR) frames are saved and the arguments start at FP + 32. A
+	 * struct_ops callback is called indirectly, so only the FP/LR frame
+	 * is saved and the arguments start at FP + 16.
+	 */
+	soff = is_struct_ops ? 16 : 32;
 	doff = (for_call_origin ? oargs_off : bargs_off);
 
 	/* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 	store_func_meta(ctx, func_meta, func_meta_off);
 
 	/* save args for bpf */
-	save_args(ctx, bargs_off, oargs_off, m, a, false);
+	save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
 	/* save callee saved registers */
 	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		/* save args for original func */
-		save_args(ctx, bargs_off, oargs_off, m, a, true);
+		save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
 		/* call original func */
 		emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
 		emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d
-- 
2.53.0-Meta

Re: [PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Puranjay Mohan]

+cc Xu

On Fri, Jul 17, 2026 at 3:17 PM Puranjay Mohan <puranjay@kernel.org> wrote:
>
> save_args() reads stack-passed arguments relative to FP assuming the
> trampoline is entered through the fentry call from a traced function, in
> which case both the parent frame (FP/x9) and the traced function frame
> (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
>
> An indirect trampoline for a struct_ops callback is entered through a
> function pointer (blr), so only the FP/LR frame is pushed and the
> arguments start at FP + 16, not FP + 32. Every stack-passed argument of
> a struct_ops callback with more than eight argument slots is read two
> slots off.
>
> This has gone unnoticed because no in-tree struct_ops member passes
> arguments on the stack. Pass is_struct_ops into save_args() and pick the
> offset accordingly, mirroring the x86 fix.
>
> Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
>  arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
>  1 file changed, 12 insertions(+), 4 deletions(-)
>
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index c6756b964c843..4fe235f7ce894 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
>  static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>                       const struct btf_func_model *m,
>                       const struct arg_aux *a,
> -                     bool for_call_origin)
> +                     bool for_call_origin, bool is_struct_ops)
>  {
>         int i;
>         int reg;
> @@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>                 bargs_off += 8;
>         }
>
> -       soff = 32; /* on stack arguments start from FP + 32 */
> +       /*
> +        * On-stack arguments start above the frame(s) pushed by the
> +        * trampoline prologue. A traced function is entered through the
> +        * fentry call, so both the parent (FP/x9) and the traced function
> +        * (FP/LR) frames are saved and the arguments start at FP + 32. A
> +        * struct_ops callback is called indirectly, so only the FP/LR frame
> +        * is saved and the arguments start at FP + 16.
> +        */
> +       soff = is_struct_ops ? 16 : 32;
>         doff = (for_call_origin ? oargs_off : bargs_off);
>
>         /* save on stack arguments */
> @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>         store_func_meta(ctx, func_meta, func_meta_off);
>
>         /* save args for bpf */
> -       save_args(ctx, bargs_off, oargs_off, m, a, false);
> +       save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
>
>         /* save callee saved registers */
>         emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
> @@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>
>         if (flags & BPF_TRAMP_F_CALL_ORIG) {
>                 /* save args for original func */
> -               save_args(ctx, bargs_off, oargs_off, m, a, true);
> +               save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
>                 /* call original func */
>                 emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
>                 emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
>
> base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d
> --
> 2.53.0-Meta
>

Re: [PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Xu Kuohai]

On 7/17/2026 10:17 PM, Puranjay Mohan wrote:
> save_args() reads stack-passed arguments relative to FP assuming the
> trampoline is entered through the fentry call from a traced function, in
> which case both the parent frame (FP/x9) and the traced function frame
> (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
> 
> An indirect trampoline for a struct_ops callback is entered through a
> function pointer (blr), so only the FP/LR frame is pushed and the
> arguments start at FP + 16, not FP + 32. Every stack-passed argument of
> a struct_ops callback with more than eight argument slots is read two
> slots off.
> 
> This has gone unnoticed because no in-tree struct_ops member passes
> arguments on the stack. Pass is_struct_ops into save_args() and pick the
> offset accordingly, mirroring the x86 fix.
>

Which x86 fix? Add a reference to it?

> Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
>   arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
>   1 file changed, 12 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index c6756b964c843..4fe235f7ce894 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
>   static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>   		      const struct btf_func_model *m,
>   		      const struct arg_aux *a,
> -		      bool for_call_origin)
> +		      bool for_call_origin, bool is_struct_ops)
>   {
>   	int i;
>   	int reg;
> @@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>   		bargs_off += 8;
>   	}
>   
> -	soff = 32; /* on stack arguments start from FP + 32 */
> +	/*
> +	 * On-stack arguments start above the frame(s) pushed by the
> +	 * trampoline prologue. A traced function is entered through the
> +	 * fentry call, so both the parent (FP/x9) and the traced function
> +	 * (FP/LR) frames are saved and the arguments start at FP + 32. A
> +	 * struct_ops callback is called indirectly, so only the FP/LR frame
> +	 * is saved and the arguments start at FP + 16.
> +	 */
> +	soff = is_struct_ops ? 16 : 32;
>   	doff = (for_call_origin ? oargs_off : bargs_off);
>   
>   	/* save on stack arguments */
> @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>   	store_func_meta(ctx, func_meta, func_meta_off);
>   
>   	/* save args for bpf */
> -	save_args(ctx, bargs_off, oargs_off, m, a, false);
> +	save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
>   
>   	/* save callee saved registers */
>   	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
> @@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>   
>   	if (flags & BPF_TRAMP_F_CALL_ORIG) {
>   		/* save args for original func */
> -		save_args(ctx, bargs_off, oargs_off, m, a, true);
> +		save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
>   		/* call original func */
>   		emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
>   		emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
>

LGTM. Since this issue is not covered by the existing selftests, I think
we need to add a new one.

> base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d

Re: [PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Puranjay Mohan]

On Sat, Jul 18, 2026 at 3:50 AM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
>
> On 7/17/2026 10:17 PM, Puranjay Mohan wrote:
> > save_args() reads stack-passed arguments relative to FP assuming the
> > trampoline is entered through the fentry call from a traced function, in
> > which case both the parent frame (FP/x9) and the traced function frame
> > (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
> >
> > An indirect trampoline for a struct_ops callback is entered through a
> > function pointer (blr), so only the FP/LR frame is pushed and the
> > arguments start at FP + 16, not FP + 32. Every stack-passed argument of
> > a struct_ops callback with more than eight argument slots is read two
> > slots off.
> >
> > This has gone unnoticed because no in-tree struct_ops member passes
> > arguments on the stack. Pass is_struct_ops into save_args() and pick the
> > offset accordingly, mirroring the x86 fix.
> >
>
> Which x86 fix? Add a reference to it?
>
> > Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
> > Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> > ---
> >   arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
> >   1 file changed, 12 insertions(+), 4 deletions(-)
> >
> > diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> > index c6756b964c843..4fe235f7ce894 100644
> > --- a/arch/arm64/net/bpf_jit_comp.c
> > +++ b/arch/arm64/net/bpf_jit_comp.c
> > @@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
> >   static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
> >                     const struct btf_func_model *m,
> >                     const struct arg_aux *a,
> > -                   bool for_call_origin)
> > +                   bool for_call_origin, bool is_struct_ops)
> >   {
> >       int i;
> >       int reg;
> > @@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
> >               bargs_off += 8;
> >       }
> >
> > -     soff = 32; /* on stack arguments start from FP + 32 */
> > +     /*
> > +      * On-stack arguments start above the frame(s) pushed by the
> > +      * trampoline prologue. A traced function is entered through the
> > +      * fentry call, so both the parent (FP/x9) and the traced function
> > +      * (FP/LR) frames are saved and the arguments start at FP + 32. A
> > +      * struct_ops callback is called indirectly, so only the FP/LR frame
> > +      * is saved and the arguments start at FP + 16.
> > +      */
> > +     soff = is_struct_ops ? 16 : 32;
> >       doff = (for_call_origin ? oargs_off : bargs_off);
> >
> >       /* save on stack arguments */
> > @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
> >       store_func_meta(ctx, func_meta, func_meta_off);
> >
> >       /* save args for bpf */
> > -     save_args(ctx, bargs_off, oargs_off, m, a, false);
> > +     save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
> >
> >       /* save callee saved registers */
> >       emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
> > @@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
> >
> >       if (flags & BPF_TRAMP_F_CALL_ORIG) {
> >               /* save args for original func */
> > -             save_args(ctx, bargs_off, oargs_off, m, a, true);
> > +             save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
> >               /* call original func */
> >               emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
> >               emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
> >
>
> LGTM. Since this issue is not covered by the existing selftests, I think
> we need to add a new one.

Kumar is working on a similar fix for x86 and will add a selftest with
it. I will wait and repost this patch again when Kumar's patchset
lands.
Thanks for your review.

Re: [PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Kumar Kartikeya Dwivedi]

On Tue Jul 21, 2026 at 12:50 PM CEST, Puranjay Mohan wrote:
> On Sat, Jul 18, 2026 at 3:50 AM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
>>
>> On 7/17/2026 10:17 PM, Puranjay Mohan wrote:
>> > save_args() reads stack-passed arguments relative to FP assuming the
>> > trampoline is entered through the fentry call from a traced function, in
>> > which case both the parent frame (FP/x9) and the traced function frame
>> > (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
>> >
>> > An indirect trampoline for a struct_ops callback is entered through a
>> > function pointer (blr), so only the FP/LR frame is pushed and the
>> > arguments start at FP + 16, not FP + 32. Every stack-passed argument of
>> > a struct_ops callback with more than eight argument slots is read two
>> > slots off.
>> >
>> > This has gone unnoticed because no in-tree struct_ops member passes
>> > arguments on the stack. Pass is_struct_ops into save_args() and pick the
>> > offset accordingly, mirroring the x86 fix.
>> >
>>
>> Which x86 fix? Add a reference to it?
>>
>> > Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
>> > Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
>> > ---
>> >   arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
>> >   1 file changed, 12 insertions(+), 4 deletions(-)
>> >
>> > diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
>> > index c6756b964c843..4fe235f7ce894 100644
>> > --- a/arch/arm64/net/bpf_jit_comp.c
>> > +++ b/arch/arm64/net/bpf_jit_comp.c
>> > @@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
>> >   static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>> >                     const struct btf_func_model *m,
>> >                     const struct arg_aux *a,
>> > -                   bool for_call_origin)
>> > +                   bool for_call_origin, bool is_struct_ops)
>> >   {
>> >       int i;
>> >       int reg;
>> > @@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>> >               bargs_off += 8;
>> >       }
>> >
>> > -     soff = 32; /* on stack arguments start from FP + 32 */
>> > +     /*
>> > +      * On-stack arguments start above the frame(s) pushed by the
>> > +      * trampoline prologue. A traced function is entered through the
>> > +      * fentry call, so both the parent (FP/x9) and the traced function
>> > +      * (FP/LR) frames are saved and the arguments start at FP + 32. A
>> > +      * struct_ops callback is called indirectly, so only the FP/LR frame
>> > +      * is saved and the arguments start at FP + 16.
>> > +      */
>> > +     soff = is_struct_ops ? 16 : 32;
>> >       doff = (for_call_origin ? oargs_off : bargs_off);
>> >
>> >       /* save on stack arguments */
>> > @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>> >       store_func_meta(ctx, func_meta, func_meta_off);
>> >
>> >       /* save args for bpf */
>> > -     save_args(ctx, bargs_off, oargs_off, m, a, false);
>> > +     save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
>> >
>> >       /* save callee saved registers */
>> >       emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
>> > @@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>> >
>> >       if (flags & BPF_TRAMP_F_CALL_ORIG) {
>> >               /* save args for original func */
>> > -             save_args(ctx, bargs_off, oargs_off, m, a, true);
>> > +             save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
>> >               /* call original func */
>> >               emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
>> >               emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
>> >
>>
>> LGTM. Since this issue is not covered by the existing selftests, I think
>> we need to add a new one.
>
> Kumar is working on a similar fix for x86 and will add a selftest with
> it. I will wait and repost this patch again when Kumar's patchset
> lands.
> Thanks for your review.

Yes, please resend when that lands, along with changes to enable tests on arm64
and the remaining JIT-side changes needed to translate arena args.

pw-bot: cr