RE: [PATCH v6 7/9] bpf/arm64: add BPF_ABS/BPF_IND packet load support

[Marat Khalili <marat.khalili@huawei.com>]

> -----Original Message-----
> From: Stephen Hemminger <stephen@networkplumber.org>
> Sent: Thursday 25 June 2026 18:30
> To: dev@dpdk.org
> Cc: Stephen Hemminger <stephen@networkplumber.org>; Wathsala Vithanage <wathsala.vithanage@arm.com>;
> Konstantin Ananyev <konstantin.ananyev@huawei.com>; Marat Khalili <marat.khalili@huawei.com>
> Subject: [PATCH v6 7/9] bpf/arm64: add BPF_ABS/BPF_IND packet load support
> 
> The arm64 JIT rejected BPF_LD | BPF_ABS and BPF_LD | BPF_IND with
> "invalid opcode", so cBPF programs converted by rte_bpf_convert() could
> not be JITed. Add these opcodes, mirroring the x86 JIT: a fast path for
> data held in the first mbuf segment, and a __rte_pktmbuf_read() slow
> path for everything else.
> 
> The forward branches over the call cannot use fixed distances:
> emit_call() materializes the helper address with a variable number of
> mov/movk instructions, so the block sizes are not known up front. Size
> the three blocks (fast path, slow path, common tail) in a dry run, then
> emit for real with the branches resolved from the measured offsets.
> 
> The effective offset is validated before use: src is a runtime value for
> BPF_IND, so a negative offset is routed to the slow path rather than
> read from the first segment, and the offset is bounded to UINT32_MAX
> before __rte_pktmbuf_read(), whose off argument is uint32_t.
> 
> Programs using these opcodes use the call register layout, since the
> slow path makes a function call.
> 
> For example, BPF_LD | BPF_IND | BPF_W (4-byte indirect load, mbuf in
> R6/x19, effective offset kept in x9) emits:
> 
> 	mov	x9, #imm		// off  = imm
> 	add	x9, x9, src		// off += src		(BPF_IND)
> 	cmp	x9, xzr			// reject negative
> 	b.mi	slow			//   effective offset
> 	mov	x10, #data_len_ofs
> 	ldrh	w10, [x19, x10]		// mbuf->data_len
> 	sub	x10, x10, x9		// data_len - off
> 	mov	x11, #sz
> 	cmp	x10, x11
> 	b.lt	slow			// not in first segment
> 	mov	x10, #data_off_ofs
> 	ldrh	w10, [x19, x10]		// mbuf->data_off
> 	mov	x7, #buf_addr_ofs
> 	ldr	x7, [x19, x7]		// mbuf->buf_addr
> 	add	x7, x7, x10
> 	add	x7, x7, x9		// ptr = buf_addr + data_off + off
> 	b	load
> slow:
> 	mov	x10, #UINT32_MAX
> 	cmp	x9, x10
> 	b.ls	1f			// off fits uint32_t ...
> 	mov	x7, #0			//   else return 0
> 	b	epilogue
> 1:	mov	x1, x9			// __rte_pktmbuf_read(mbuf, off, sz, buf)
> 	mov	x0, x19
> 	mov	w2, #sz
> 	sub	x3, x25, #stack_ofs
> 	mov	x9, #<helper lo>
> 	movk	x9, #<helper hi>
> 	blr	x9
> 	mov	x7, x0			// ptr = return value
> 	cbnz	x7, load		// non-NULL -> common tail
> 	mov	x7, #0			//   else return 0
> 	b	epilogue
> load:
> 	ldr	w7, [x7, xzr]		// *(uint32_t *)ptr	(size varies)
> 	rev32	x7, x7			// ntoh	(size varies; omitted for BPF_B)
> 
> For BPF_ABS the "add x9, x9, src" is omitted; the final load/byte-swap
> vary with the access size.
> 
> Bugzilla ID: 1427
> 
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>


Acked-by: Marat Khalili <marat.khalili@huawei.com>


> ---
>  lib/bpf/bpf_jit_arm64.c | 169 +++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 168 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c
> index 51906c7f0d..6d531dc83d 100644
> --- a/lib/bpf/bpf_jit_arm64.c
> +++ b/lib/bpf/bpf_jit_arm64.c
> @@ -1133,6 +1133,155 @@ emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off)
>  	emit_b_cond(ctx, ebpf_to_a64_cond(op), jump_offset_get(ctx, i, off));
>  }
> 
> +/* LD_ABS/LD_IND code block offsets (in arm64 instructions) */
> +enum {
> +	LDMB_FAST_OFS, /* fast path */
> +	LDMB_SLOW_OFS, /* slow path */
> +	LDMB_FIN_OFS,  /* common tail */
> +	LDMB_OFS_NUM
> +};
> +
> +/*
> + * Helper for emit_ld_mbuf(): fast path.
> + * Compute the packet offset; if it lies inside the first segment leave the
> + * data pointer in R0, otherwise branch to the slow path.
> + */
> +static void
> +emit_ldmb_fast_path(struct a64_jit_ctx *ctx, uint8_t src, uint8_t mode,
> +		    uint32_t sz, int32_t imm, const uint32_t ofs[LDMB_OFS_NUM])
> +{
> +	uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
> +	uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6);
> +	uint8_t tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
> +	uint8_t tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
> +	uint8_t tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
> +
> +	/* off = imm (+ src for BPF_IND) */
> +	emit_mov_imm(ctx, 1, tmp1, imm);
> +	if (mode == BPF_IND)
> +		emit_add(ctx, 1, tmp1, src);
> +
> +	/*
> +	 * A negative effective offset (src can be < 0 for BPF_IND) would pass
> +	 * the signed check below and read before the segment, so route it to
> +	 * the slow path, which rejects it via the uint32_t bound on off.
> +	 */
> +	emit_cmp(ctx, 1, tmp1, A64_ZR);
> +	emit_b_cond(ctx, A64_MI, (int32_t)(ofs[LDMB_SLOW_OFS] - ctx->idx));
> +
> +	/* if ((int64_t)(mbuf->data_len - off) < sz) goto slow_path */
> +	emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_len));
> +	emit_ldr(ctx, BPF_H, tmp2, r6, tmp2);
> +	emit_sub(ctx, 1, tmp2, tmp1);
> +	emit_mov_imm(ctx, 1, tmp3, sz);
> +	emit_cmp(ctx, 1, tmp2, tmp3);
> +	emit_b_cond(ctx, A64_LT, (int32_t)(ofs[LDMB_SLOW_OFS] - ctx->idx));
> +
> +	/* R0 = mbuf->buf_addr + mbuf->data_off + off */
> +	emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_off));
> +	emit_ldr(ctx, BPF_H, tmp2, r6, tmp2);
> +	emit_mov_imm(ctx, 1, r0, offsetof(struct rte_mbuf, buf_addr));
> +	emit_ldr(ctx, EBPF_DW, r0, r6, r0);
> +	emit_add(ctx, 1, r0, tmp2);
> +	emit_add(ctx, 1, r0, tmp1);
> +
> +	emit_b(ctx, (int32_t)(ofs[LDMB_FIN_OFS] - ctx->idx));
> +}
> +
> +/*
> + * Helper for emit_ld_mbuf(): slow path.
> + * R0 = __rte_pktmbuf_read(mbuf, off, sz, buf); return 0 if NULL.
> + * The scratch buffer is the space reserved by __rte_bpf_validate() at the
> + * bottom of the eBPF stack frame, i.e. (frame_pointer - stack_ofs).
> + */
> +static void
> +emit_ldmb_slow_path(struct a64_jit_ctx *ctx, uint32_t sz, uint32_t stack_ofs)
> +{
> +	uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
> +	uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6);
> +	uint8_t fp = ebpf_to_a64_reg(ctx, EBPF_FP);
> +	uint8_t tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
> +	uint8_t tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
> +
> +	/*
> +	 * __rte_pktmbuf_read() takes a uint32_t off, so a 64-bit off that does
> +	 * not fit would be silently truncated.  Return 0 if it is out of range;
> +	 * this also catches the negative off routed here by the fast path.
> +	 */
> +	emit_mov_imm(ctx, 1, tmp2, UINT32_MAX);
> +	emit_cmp(ctx, 1, tmp1, tmp2);
> +	emit_b_cond(ctx, A64_LS, 3);		/* off <= UINT32_MAX: do the call */
> +	emit_mov_imm(ctx, 1, r0, 0);
> +	emit_b(ctx, (ctx->program_start + ctx->program_sz) - ctx->idx);
> +
> +	/* arguments of __rte_pktmbuf_read(mbuf, off, len, buf) */
> +	emit_mov_64(ctx, A64_R(1), tmp1);		/* off (held in tmp1) */
> +	emit_mov_64(ctx, A64_R(0), r6);			/* mbuf */
> +	emit_mov_imm(ctx, 0, A64_R(2), sz);		/* len */
> +	emit_sub_imm_64(ctx, A64_R(3), fp, stack_ofs);	/* buf */
> +
> +	emit_call(ctx, tmp1, (void *)(uintptr_t)__rte_pktmbuf_read);
> +	emit_return_zero_if_src_zero(ctx, 1, r0);
> +}
> +
> +/*
> + * Helper for emit_ld_mbuf(): common tail.
> + * Load the value pointed to by R0 and convert from network byte order.
> + */
> +static void
> +emit_ldmb_fin(struct a64_jit_ctx *ctx, uint8_t opsz, uint32_t sz)
> +{
> +	uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
> +
> +	emit_ldr(ctx, opsz, r0, r0, A64_ZR);
> +	if (opsz != BPF_B)
> +		emit_be(ctx, r0, sz * 8);
> +}
> +
> +/*
> + * Emit code for BPF_LD | BPF_ABS and BPF_LD | BPF_IND packet loads:
> + *
> + *	off = imm (+ src for BPF_IND)
> + *	if (off >= 0 && mbuf->data_len - off >= sz)	    -- fast path
> + *		ptr = mbuf->buf_addr + mbuf->data_off + off;
> + *	else						    -- slow path
> + *		if ((uint64_t)off > UINT32_MAX)
> + *			return 0;
> + *		ptr = __rte_pktmbuf_read(mbuf, off, sz, buf);
> + *		if (ptr == NULL)
> + *			return 0;
> + *	R0 = ntoh(*(size *)ptr);			    -- common tail
> + *
> + * The three blocks are sized in a dry run so the forward branches can be
> + * resolved, then emitted for real (arm64 instructions are fixed width, so
> + * the dry run reproduces the real instruction count exactly).
> + */
> +static void
> +emit_ld_mbuf(struct a64_jit_ctx *ctx, uint8_t op, uint8_t src, int32_t imm,
> +	     uint32_t stack_ofs)
> +{
> +	uint8_t mode = BPF_MODE(op);
> +	uint8_t opsz = BPF_SIZE(op);
> +	uint32_t sz = bpf_size(opsz);
> +	uint32_t ofs[LDMB_OFS_NUM];
> +
> +	/* seed offsets so the dry-run branches stay in range */
> +	ofs[LDMB_FAST_OFS] = ofs[LDMB_SLOW_OFS] = ofs[LDMB_FIN_OFS] = ctx->idx;
> +
> +	/* dry run to record block offsets */
> +	emit_ldmb_fast_path(ctx, src, mode, sz, imm, ofs);
> +	ofs[LDMB_SLOW_OFS] = ctx->idx;
> +	emit_ldmb_slow_path(ctx, sz, stack_ofs);
> +	ofs[LDMB_FIN_OFS] = ctx->idx;
> +	emit_ldmb_fin(ctx, opsz, sz);
> +
> +	/* rewind and emit for real with resolved offsets */
> +	ctx->idx = ofs[LDMB_FAST_OFS];
> +	emit_ldmb_fast_path(ctx, src, mode, sz, imm, ofs);
> +	emit_ldmb_slow_path(ctx, sz, stack_ofs);
> +	emit_ldmb_fin(ctx, opsz, sz);
> +}
> +
>  static void
>  check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
>  {
> @@ -1145,8 +1294,17 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
>  		op = ins->code;
> 
>  		switch (op) {
> -		/* Call imm */
> +		/*
> +		 * BPF_ABS/BPF_IND can fall through to __rte_pktmbuf_read(),
> +		 * so they need the call-clobbered register layout as well.
> +		 */
>  		case (BPF_JMP | EBPF_CALL):
> +		case (BPF_LD | BPF_ABS | BPF_B):
> +		case (BPF_LD | BPF_ABS | BPF_H):
> +		case (BPF_LD | BPF_ABS | BPF_W):
> +		case (BPF_LD | BPF_IND | BPF_B):
> +		case (BPF_LD | BPF_IND | BPF_H):
> +		case (BPF_LD | BPF_IND | BPF_W):
>  			ctx->foundcall = 1;
>  			return;
>  		}
> @@ -1348,6 +1506,15 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
>  			emit_mov_imm(ctx, 1, dst, u64);
>  			i++;
>  			break;
> +		/* R0 = ntoh(*(size *)(mbuf data + (src) + imm)) */
> +		case (BPF_LD | BPF_ABS | BPF_B):
> +		case (BPF_LD | BPF_ABS | BPF_H):
> +		case (BPF_LD | BPF_ABS | BPF_W):
> +		case (BPF_LD | BPF_IND | BPF_B):
> +		case (BPF_LD | BPF_IND | BPF_H):
> +		case (BPF_LD | BPF_IND | BPF_W):
> +			emit_ld_mbuf(ctx, op, src, imm, bpf->stack_sz);
> +			break;
>  		/* *(size *)(dst + off) = src */
>  		case (BPF_STX | BPF_MEM | BPF_B):
>  		case (BPF_STX | BPF_MEM | BPF_H):
> --
> 2.53.0