[PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Puranjay Mohan]

save_args() reads stack-passed arguments relative to FP assuming the
trampoline is entered through the fentry call from a traced function, in
which case both the parent frame (FP/x9) and the traced function frame
(FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a
function pointer (blr), so only the FP/LR frame is pushed and the
arguments start at FP + 16, not FP + 32. Every stack-passed argument of
a struct_ops callback with more than eight argument slots is read two
slots off.

This has gone unnoticed because no in-tree struct_ops member passes
arguments on the stack. Pass is_struct_ops into save_args() and pick the
offset accordingly, mirroring the x86 fix.

Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index c6756b964c843..4fe235f7ce894 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
 static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		      const struct btf_func_model *m,
 		      const struct arg_aux *a,
-		      bool for_call_origin)
+		      bool for_call_origin, bool is_struct_ops)
 {
 	int i;
 	int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		bargs_off += 8;
 	}
 
-	soff = 32; /* on stack arguments start from FP + 32 */
+	/*
+	 * On-stack arguments start above the frame(s) pushed by the
+	 * trampoline prologue. A traced function is entered through the
+	 * fentry call, so both the parent (FP/x9) and the traced function
+	 * (FP/LR) frames are saved and the arguments start at FP + 32. A
+	 * struct_ops callback is called indirectly, so only the FP/LR frame
+	 * is saved and the arguments start at FP + 16.
+	 */
+	soff = is_struct_ops ? 16 : 32;
 	doff = (for_call_origin ? oargs_off : bargs_off);
 
 	/* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 	store_func_meta(ctx, func_meta, func_meta_off);
 
 	/* save args for bpf */
-	save_args(ctx, bargs_off, oargs_off, m, a, false);
+	save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
 	/* save callee saved registers */
 	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		/* save args for original func */
-		save_args(ctx, bargs_off, oargs_off, m, a, true);
+		save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
 		/* call original func */
 		emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
 		emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d
-- 
2.53.0-Meta

Re: [PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Puranjay Mohan]

+cc Xu

On Fri, Jul 17, 2026 at 3:17 PM Puranjay Mohan <puranjay@kernel.org> wrote:
>
> save_args() reads stack-passed arguments relative to FP assuming the
> trampoline is entered through the fentry call from a traced function, in
> which case both the parent frame (FP/x9) and the traced function frame
> (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
>
> An indirect trampoline for a struct_ops callback is entered through a
> function pointer (blr), so only the FP/LR frame is pushed and the
> arguments start at FP + 16, not FP + 32. Every stack-passed argument of
> a struct_ops callback with more than eight argument slots is read two
> slots off.
>
> This has gone unnoticed because no in-tree struct_ops member passes
> arguments on the stack. Pass is_struct_ops into save_args() and pick the
> offset accordingly, mirroring the x86 fix.
>
> Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
>  arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
>  1 file changed, 12 insertions(+), 4 deletions(-)
>
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index c6756b964c843..4fe235f7ce894 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
>  static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>                       const struct btf_func_model *m,
>                       const struct arg_aux *a,
> -                     bool for_call_origin)
> +                     bool for_call_origin, bool is_struct_ops)
>  {
>         int i;
>         int reg;
> @@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>                 bargs_off += 8;
>         }
>
> -       soff = 32; /* on stack arguments start from FP + 32 */
> +       /*
> +        * On-stack arguments start above the frame(s) pushed by the
> +        * trampoline prologue. A traced function is entered through the
> +        * fentry call, so both the parent (FP/x9) and the traced function
> +        * (FP/LR) frames are saved and the arguments start at FP + 32. A
> +        * struct_ops callback is called indirectly, so only the FP/LR frame
> +        * is saved and the arguments start at FP + 16.
> +        */
> +       soff = is_struct_ops ? 16 : 32;
>         doff = (for_call_origin ? oargs_off : bargs_off);
>
>         /* save on stack arguments */
> @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>         store_func_meta(ctx, func_meta, func_meta_off);
>
>         /* save args for bpf */
> -       save_args(ctx, bargs_off, oargs_off, m, a, false);
> +       save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
>
>         /* save callee saved registers */
>         emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
> @@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>
>         if (flags & BPF_TRAMP_F_CALL_ORIG) {
>                 /* save args for original func */
> -               save_args(ctx, bargs_off, oargs_off, m, a, true);
> +               save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
>                 /* call original func */
>                 emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
>                 emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
>
> base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d
> --
> 2.53.0-Meta
>

Re: [PATCH bpf-next] bpf, arm64: Fix stack-passed arguments for indirect trampolines

[Xu Kuohai]

On 7/17/2026 10:17 PM, Puranjay Mohan wrote:
> save_args() reads stack-passed arguments relative to FP assuming the
> trampoline is entered through the fentry call from a traced function, in
> which case both the parent frame (FP/x9) and the traced function frame
> (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
> 
> An indirect trampoline for a struct_ops callback is entered through a
> function pointer (blr), so only the FP/LR frame is pushed and the
> arguments start at FP + 16, not FP + 32. Every stack-passed argument of
> a struct_ops callback with more than eight argument slots is read two
> slots off.
> 
> This has gone unnoticed because no in-tree struct_ops member passes
> arguments on the stack. Pass is_struct_ops into save_args() and pick the
> offset accordingly, mirroring the x86 fix.
>

Which x86 fix? Add a reference to it?

> Fixes: 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
> Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
> ---
>   arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
>   1 file changed, 12 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index c6756b964c843..4fe235f7ce894 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
>   static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>   		      const struct btf_func_model *m,
>   		      const struct arg_aux *a,
> -		      bool for_call_origin)
> +		      bool for_call_origin, bool is_struct_ops)
>   {
>   	int i;
>   	int reg;
> @@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
>   		bargs_off += 8;
>   	}
>   
> -	soff = 32; /* on stack arguments start from FP + 32 */
> +	/*
> +	 * On-stack arguments start above the frame(s) pushed by the
> +	 * trampoline prologue. A traced function is entered through the
> +	 * fentry call, so both the parent (FP/x9) and the traced function
> +	 * (FP/LR) frames are saved and the arguments start at FP + 32. A
> +	 * struct_ops callback is called indirectly, so only the FP/LR frame
> +	 * is saved and the arguments start at FP + 16.
> +	 */
> +	soff = is_struct_ops ? 16 : 32;
>   	doff = (for_call_origin ? oargs_off : bargs_off);
>   
>   	/* save on stack arguments */
> @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>   	store_func_meta(ctx, func_meta, func_meta_off);
>   
>   	/* save args for bpf */
> -	save_args(ctx, bargs_off, oargs_off, m, a, false);
> +	save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
>   
>   	/* save callee saved registers */
>   	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
> @@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
>   
>   	if (flags & BPF_TRAMP_F_CALL_ORIG) {
>   		/* save args for original func */
> -		save_args(ctx, bargs_off, oargs_off, m, a, true);
> +		save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
>   		/* call original func */
>   		emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
>   		emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);
>

LGTM. Since this issue is not covered by the existing selftests, I think
we need to add a new one.

> base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d