[PATCH bpf-next v2 3/8] bpf, arm64: JIT UHMUL and SHMUL

[PATCH bpf-next v2 3/8] bpf, arm64: JIT UHMUL and SHMUL

[tangyazhou]

From: Yazhou Tang <tangyazhou518@outlook.com>

arm64 provides UMULH and SMULH instructions that return the high 64 bits
of an unsigned or signed 64-bit multiplication, respectively.

Add encodings for these instructions and use them to lower the UHMUL and
SHMUL BPF_MUL variants for both register and immediate sources. Keep the
existing low-half and ALU32 multiplication lowering unchanged.

Suggested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
---
 arch/arm64/include/asm/insn.h |  4 ++++
 arch/arm64/lib/insn.c         |  6 ++++++
 arch/arm64/net/bpf_jit.h      |  6 ++++++
 arch/arm64/net/bpf_jit_comp.c | 15 +++++++++++++--
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index 1ce75a442638..f38738fa54a2 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -247,6 +247,8 @@ enum aarch64_insn_data2_type {
 enum aarch64_insn_data3_type {
 	AARCH64_INSN_DATA3_MADD,
 	AARCH64_INSN_DATA3_MSUB,
+	AARCH64_INSN_DATA3_SMULH,
+	AARCH64_INSN_DATA3_UMULH,
 };
 
 enum aarch64_insn_logic_type {
@@ -396,6 +398,8 @@ __AARCH64_INSN_FUNCS(sub_ext,	0x7FE00000, 0x4B200000)
 __AARCH64_INSN_FUNCS(subs_ext,	0x7FE00000, 0x6B200000)
 __AARCH64_INSN_FUNCS(madd,	0x7FE08000, 0x1B000000)
 __AARCH64_INSN_FUNCS(msub,	0x7FE08000, 0x1B008000)
+__AARCH64_INSN_FUNCS(smulh,	0x7FE0FC00, 0x1B407C00)
+__AARCH64_INSN_FUNCS(umulh,	0x7FE0FC00, 0x1BC07C00)
 __AARCH64_INSN_FUNCS(udiv,	0x7FE0FC00, 0x1AC00800)
 __AARCH64_INSN_FUNCS(sdiv,	0x7FE0FC00, 0x1AC00C00)
 __AARCH64_INSN_FUNCS(lslv,	0x7FE0FC00, 0x1AC02000)
diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c
index e70ac0238515..dc483d1b8bd5 100644
--- a/arch/arm64/lib/insn.c
+++ b/arch/arm64/lib/insn.c
@@ -1155,6 +1155,12 @@ u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
 	case AARCH64_INSN_DATA3_MSUB:
 		insn = aarch64_insn_get_msub_value();
 		break;
+	case AARCH64_INSN_DATA3_SMULH:
+		insn = aarch64_insn_get_smulh_value();
+		break;
+	case AARCH64_INSN_DATA3_UMULH:
+		insn = aarch64_insn_get_umulh_value();
+		break;
 	default:
 		pr_err("%s: unknown data3 encoding %d\n", __func__, type);
 		return AARCH64_BREAK_FAULT;
diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
index b2fe6e6dcf44..70fc329c661d 100644
--- a/arch/arm64/net/bpf_jit.h
+++ b/arch/arm64/net/bpf_jit.h
@@ -281,6 +281,12 @@
 	A64_VARIANT(sf), AARCH64_INSN_DATA3_MSUB)
 /* Rd = Rn * Rm */
 #define A64_MUL(sf, Rd, Rn, Rm) A64_MADD(sf, Rd, A64_ZR, Rn, Rm)
+/* Rd = high 64 bits of signed Rn * Rm */
+#define A64_SMULH(Rd, Rn, Rm) aarch64_insn_gen_data3(Rd, A64_ZR, Rn, Rm, \
+	AARCH64_INSN_VARIANT_64BIT, AARCH64_INSN_DATA3_SMULH)
+/* Rd = high 64 bits of unsigned Rn * Rm */
+#define A64_UMULH(Rd, Rn, Rm) aarch64_insn_gen_data3(Rd, A64_ZR, Rn, Rm, \
+	AARCH64_INSN_VARIANT_64BIT, AARCH64_INSN_DATA3_UMULH)
 
 /* Logical (shifted register) */
 #define A64_LOGIC_SREG(sf, Rd, Rn, Rm, type) \
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 326e9e1dea0a..a9989cd5098c 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -144,6 +144,17 @@ static inline void emit_a64_mov_i(const int is64, const int reg,
 	}
 }
 
+static inline void emit_a64_mul(const bool is64, const u8 dst, const u8 src,
+				const s16 off, struct jit_ctx *ctx)
+{
+	if (is64 && off == BPF_UHMUL)
+		emit(A64_UMULH(dst, dst, src), ctx);
+	else if (is64 && off == BPF_SHMUL)
+		emit(A64_SMULH(dst, dst, src), ctx);
+	else
+		emit(A64_MUL(is64, dst, dst, src), ctx);
+}
+
 static int i64_i16_blocks(const u64 val, bool inverse)
 {
 	return (((val >>  0) & 0xffff) != (inverse ? 0xffff : 0x0000)) +
@@ -1531,7 +1542,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		break;
 	case BPF_ALU | BPF_MUL | BPF_X:
 	case BPF_ALU64 | BPF_MUL | BPF_X:
-		emit(A64_MUL(is64, dst, dst, src), ctx);
+		emit_a64_mul(is64, dst, src, off, ctx);
 		break;
 	case BPF_ALU | BPF_DIV | BPF_X:
 	case BPF_ALU64 | BPF_DIV | BPF_X:
@@ -1660,7 +1671,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 	case BPF_ALU | BPF_MUL | BPF_K:
 	case BPF_ALU64 | BPF_MUL | BPF_K:
 		emit_a64_mov_i(is64, tmp, imm, ctx);
-		emit(A64_MUL(is64, dst, dst, tmp), ctx);
+		emit_a64_mul(is64, dst, tmp, off, ctx);
 		break;
 	case BPF_ALU | BPF_DIV | BPF_K:
 	case BPF_ALU64 | BPF_DIV | BPF_K:

-- 
2.56.0

[PATCH bpf-next v2 4/8] bpf, riscv: JIT UHMUL and SHMUL on RV64

[tangyazhou]

From: Yazhou Tang <tangyazhou518@outlook.com>

On RV64, the RISC-V M extension provides MULHU and MULH instructions that
return the high 64 bits of an unsigned or signed 64-bit multiplication,
respectively.

Add the missing MULH encoder and use MULHU and MULH to lower the UHMUL and
SHMUL BPF_MUL variants for both register and immediate sources. Keep the
existing low-half and ALU32 multiplication lowering unchanged.

Suggested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
---
 arch/riscv/net/bpf_jit.h        |  5 +++++
 arch/riscv/net/bpf_jit_comp64.c | 18 +++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h
index 5c4f53060c91..21a190a6a6d9 100644
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -436,6 +436,11 @@ static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
 	return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
 }
 
+static inline u32 rv_mulh(u8 rd, u8 rs1, u8 rs2)
+{
+	return rv_r_insn(1, rs2, rs1, 1, rd, 0x33);
+}
+
 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
 {
 	return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 788fcc21fd5b..dfaf85926f20 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -197,6 +197,19 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
 		emit_addi(rd, rd, lower, ctx);
 }
 
+static void emit_mul(u8 rd, u8 rs, bool is64, s16 off,
+		     struct rv_jit_context *ctx)
+{
+	if (!is64)
+		emit(rv_mulw(rd, rd, rs), ctx);
+	else if (off == BPF_UHMUL)
+		emit(rv_mulhu(rd, rd, rs), ctx);
+	else if (off == BPF_SHMUL)
+		emit(rv_mulh(rd, rd, rs), ctx);
+	else
+		emit(rv_mul(rd, rd, rs), ctx);
+}
+
 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
 {
 	int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
@@ -1526,7 +1539,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 		break;
 	case BPF_ALU | BPF_MUL | BPF_X:
 	case BPF_ALU64 | BPF_MUL | BPF_X:
-		emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
+		emit_mul(rd, rs, is64, off, ctx);
 		if (!is64 && !aux->verifier_zext)
 			emit_zextw(rd, rd, ctx);
 		break;
@@ -1662,8 +1675,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 	case BPF_ALU | BPF_MUL | BPF_K:
 	case BPF_ALU64 | BPF_MUL | BPF_K:
 		emit_imm(RV_REG_T1, imm, ctx);
-		emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
-		     rv_mulw(rd, rd, RV_REG_T1), ctx);
+		emit_mul(rd, RV_REG_T1, is64, off, ctx);
 		if (!is64 && !aux->verifier_zext)
 			emit_zextw(rd, rd, ctx);
 		break;

-- 
2.56.0

Re: [PATCH bpf-next v2 4/8] bpf, riscv: JIT UHMUL and SHMUL on RV64

[Björn Töpel]

tangyazhou@zju.edu.cn writes:

> From: Yazhou Tang <tangyazhou518@outlook.com>
>
> On RV64, the RISC-V M extension provides MULHU and MULH instructions that
> return the high 64 bits of an unsigned or signed 64-bit multiplication,
> respectively.
>
> Add the missing MULH encoder and use MULHU and MULH to lower the UHMUL and
> SHMUL BPF_MUL variants for both register and immediate sources. Keep the
> existing low-half and ALU32 multiplication lowering unchanged.
>
> Suggested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com>
> Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
> Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
> Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
> Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
> Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>

Acked-by: Björn Töpel <bjorn@kernel.org>

[PATCH bpf-next v2 5/8] bpf: Reject UHMUL/SHMUL in unsupported backends

[tangyazhou]

From: Yazhou Tang <tangyazhou518@outlook.com>

CPU JIT backends without explicit UHMUL/SHMUL support currently lower
BPF_MUL based on the opcode without examining insn->off. Once the verifier
accepts UHMUL and SHMUL, a backend without explicit support would therefore
lower either variant as the existing low-half multiplication and silently
produce the wrong result.

Reject UHMUL and SHMUL in each affected CPU JIT without native support,
using its existing unsupported-instruction path. This makes JIT compilation
fail cleanly instead of emitting an incorrect low-half multiply, and allows
the normal interpreter fallback when available.

The NFP hardware offload compiler has the same opcode-only dispatch:
BPF_ALU64 MUL instructions are routed to mul_reg64() or mul_imm64()
without considering insn->off. Reject UHMUL and SHMUL in the NFP offload
verifier as well, before they can reach the ordinary multiplication lowering.

Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
---
 arch/arc/net/bpf_jit_core.c                       | 3 +++
 arch/arm/net/bpf_jit_32.c                         | 3 +++
 arch/loongarch/net/bpf_jit.c                      | 3 +++
 arch/mips/net/bpf_jit_comp32.c                    | 3 +++
 arch/mips/net/bpf_jit_comp64.c                    | 3 +++
 arch/parisc/net/bpf_jit_comp32.c                  | 3 +++
 arch/parisc/net/bpf_jit_comp64.c                  | 3 +++
 arch/powerpc/net/bpf_jit_comp32.c                 | 3 +++
 arch/powerpc/net/bpf_jit_comp64.c                 | 3 +++
 arch/riscv/net/bpf_jit_comp32.c                   | 3 +++
 arch/s390/net/bpf_jit_comp.c                      | 3 +++
 arch/sparc/net/bpf_jit_comp_64.c                  | 3 +++
 arch/x86/net/bpf_jit_comp32.c                     | 3 +++
 drivers/net/ethernet/netronome/nfp/bpf/verifier.c | 6 ++++++
 14 files changed, 45 insertions(+)

diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c
index 639a2736f029..78b1819cee5a 100644
--- a/arch/arc/net/bpf_jit_core.c
+++ b/arch/arc/net/bpf_jit_core.c
@@ -734,6 +734,9 @@ static int handle_insn(struct jit_context *ctx, u32 idx)
 	u8  len = 0;
 	int ret = 0;
 
+	if (bpf_insn_is_hmul(insn))
+		return -EOPNOTSUPP;
+
 	switch (code) {
 	/* dst += src (32-bit) */
 	case BPF_ALU | BPF_ADD | BPF_X:
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 9ede81afbc50..aa466fc79d96 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -1611,6 +1611,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 } while (0)
 #define check_imm24(imm) check_imm(24, imm)
 
+	if (bpf_insn_is_hmul(insn))
+		goto notyet;
+
 	switch (code) {
 	/* ALU operations */
 
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 9393e4b0db8e..71bd518ceaf3 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -709,6 +709,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
 	const s32 imm = insn->imm;
 	const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
 
+	if (bpf_insn_is_hmul(insn))
+		return -EOPNOTSUPP;
+
 	switch (code) {
 	/* dst = src */
 	case BPF_ALU | BPF_MOV | BPF_X:
diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c
index 84d9c7036e4f..4a8bedddf61b 100644
--- a/arch/mips/net/bpf_jit_comp32.c
+++ b/arch/mips/net/bpf_jit_comp32.c
@@ -1544,6 +1544,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	s32 val, rel;
 	u8 alu, jmp;
 
+	if (bpf_insn_is_hmul(insn))
+		goto notyet;
+
 	switch (code) {
 	/* ALU operations */
 	/* dst = imm */
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index 994589c3de46..75202cac31f4 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c
@@ -719,6 +719,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	s32 val, rel;
 	u8 alu, jmp;
 
+	if (bpf_insn_is_hmul(insn))
+		goto notyet;
+
 	switch (code) {
 	/* ALU operations */
 	/* dst = imm */
diff --git a/arch/parisc/net/bpf_jit_comp32.c b/arch/parisc/net/bpf_jit_comp32.c
index eea0a34a3bfe..294eee24f2d9 100644
--- a/arch/parisc/net/bpf_jit_comp32.c
+++ b/arch/parisc/net/bpf_jit_comp32.c
@@ -1142,6 +1142,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 		BPF_CLASS(code), code, (code & BPF_ALU64) ? 1:0, BPF_SIZE(code),
 		BPF_OP(code), insn->src_reg, insn->dst_reg);
 
+	if (bpf_insn_is_hmul(insn))
+		return -EOPNOTSUPP;
+
 	switch (code) {
 	/* dst = src */
 	case BPF_ALU64 | BPF_MOV | BPF_X:
diff --git a/arch/parisc/net/bpf_jit_comp64.c b/arch/parisc/net/bpf_jit_comp64.c
index c326fa737ec8..319bd28f1e5b 100644
--- a/arch/parisc/net/bpf_jit_comp64.c
+++ b/arch/parisc/net/bpf_jit_comp64.c
@@ -610,6 +610,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 
 	init_regs(&rd, &rs, insn, ctx);
 
+	if (bpf_insn_is_hmul(insn))
+		return -EOPNOTSUPP;
+
 	switch (code) {
 	/* dst = src */
 	case BPF_ALU | BPF_MOV | BPF_X:
diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
index bfdc50740da8..6bfc97a029d7 100644
--- a/arch/powerpc/net/bpf_jit_comp32.c
+++ b/arch/powerpc/net/bpf_jit_comp32.c
@@ -349,6 +349,9 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
 		u32 true_cond;
 		u32 tmp_idx;
 
+		if (bpf_insn_is_hmul(&insn[i]))
+			return -EOPNOTSUPP;
+
 		if (i && (BPF_CLASS(code) == BPF_ALU64 || BPF_CLASS(code) == BPF_ALU) &&
 		    (BPF_CLASS(prevcode) == BPF_ALU64 || BPF_CLASS(prevcode) == BPF_ALU) &&
 		    BPF_OP(prevcode) == BPF_MOV && BPF_SRC(prevcode) == BPF_X &&
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index fc9db691e820..192c96e0ed34 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -992,6 +992,9 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
 		u32 tmp_idx;
 		u32 jmp_off;
 
+		if (bpf_insn_is_hmul(&insn[i]))
+			return -EOPNOTSUPP;
+
 		/*
 		 * addrs[] maps a BPF bytecode address into a real offset from
 		 * the start of the body code.
diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c
index a9e0bd5cc81d..64bc6779d25c 100644
--- a/arch/riscv/net/bpf_jit_comp32.c
+++ b/arch/riscv/net/bpf_jit_comp32.c
@@ -1014,6 +1014,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 	const s8 *tmp1 = bpf2rv32[TMP_REG_1];
 	const s8 *tmp2 = bpf2rv32[TMP_REG_2];
 
+	if (bpf_insn_is_hmul(insn))
+		goto notsupported;
+
 	switch (code) {
 	case BPF_ALU64 | BPF_MOV | BPF_X:
 		if (insn->off != 0) {
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 8eaaac7028a1..35af9a32294a 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -993,6 +993,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 
 	bpf_jit_probe_init(&probe);
 
+	if (bpf_insn_is_hmul(insn))
+		return -EOPNOTSUPP;
+
 	switch (insn->code) {
 	/*
 	 * BPF_MOV
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index 2fa0e9375127..1154ddc515b0 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -901,6 +901,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	const s16 off = insn->off;
 	const s32 imm = insn->imm;
 
+	if (bpf_insn_is_hmul(insn))
+		return -EOPNOTSUPP;
+
 	if (insn->src_reg == BPF_REG_FP)
 		ctx->saw_frame_pointer = true;
 
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 852baf2e4db4..11ed65130d1e 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -1683,6 +1683,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 		int ilen;
 		u8 *func;
 
+		if (bpf_insn_is_hmul(insn))
+			goto notyet;
+
 		switch (code) {
 		/* ALU operations */
 		/* dst = src */
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 9ddbfe9d6019..4481c8d84d28 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -3,6 +3,7 @@
 
 #include <linux/bpf.h>
 #include <linux/bpf_verifier.h>
+#include <linux/filter.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/pkt_cls.h>
@@ -561,6 +562,11 @@ nfp_bpf_check_alu(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	const struct bpf_reg_state *dreg =
 		cur_regs(env) + meta->insn.dst_reg;
 
+	if (bpf_insn_is_hmul(&meta->insn)) {
+		pr_vlog(env, "UHMUL/SHMUL are not supported\n");
+		return -EOPNOTSUPP;
+	}
+
 	meta->umin_src = min(meta->umin_src, reg_umin(sreg));
 	meta->umax_src = max(meta->umax_src, reg_umax(sreg));
 	meta->umin_dst = min(meta->umin_dst, reg_umin(dreg));

-- 
2.56.0

[PATCH bpf-next v2 6/8] bpf: Refactor ALU instruction variant validation

[tangyazhou]

From: Yazhou Tang <tangyazhou518@outlook.com>

check_alu_fields() currently validates insn->off together with the
source-specific reserved fields. This duplicates the DIV and MOD
variant rules between register and immediate forms and makes the valid
instruction variants difficult to identify.

Move the opcode-specific insn->off validation into is_valid_alu_variant().
The helper permits off == 1 for DIV and MOD and requires off == 0 for all
other ALU operations. Validate the variant once, then separately check the
reserved immediate field for register sources and src_reg for immediate sources.

This is a pure refactoring and does not change which instruction encodings
the verifier accepts.

Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
---
 kernel/bpf/verifier.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5601d2056218..4f9d7b3de442 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21194,6 +21194,18 @@ static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
 	return -EPROTO;
 }
 
+static bool is_valid_alu_variant(const struct bpf_insn *insn)
+{
+	switch (BPF_OP(insn->code)) {
+	case BPF_DIV:
+	case BPF_MOD:
+		/* off == 1 selects SDIV/SMOD. */
+		return insn->off == 0 || insn->off == 1;
+	default:
+		return insn->off == 0;
+	}
+}
+
 static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 {
 	u8 class = BPF_CLASS(insn->code);
@@ -21249,15 +21261,9 @@ static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 	case BPF_MUL:
 	case BPF_DIV:
 	case BPF_MOD:
-		if (BPF_SRC(insn->code) == BPF_X) {
-			if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) ||
-			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
-				verbose(env, "BPF_ALU uses reserved fields\n");
-				return -EINVAL;
-			}
-		} else if (insn->src_reg != BPF_REG_0 ||
-			   (insn->off != 0 && insn->off != 1) ||
-			   (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
+		if (!is_valid_alu_variant(insn) ||
+		    (BPF_SRC(insn->code) == BPF_X && insn->imm != 0) ||
+		    (BPF_SRC(insn->code) == BPF_K && insn->src_reg != BPF_REG_0)) {
 			verbose(env, "BPF_ALU uses reserved fields\n");
 			return -EINVAL;
 		}

-- 
2.56.0

[PATCH bpf-next v2 7/8] bpf: Add verifier support for UHMUL and SHMUL

[tangyazhou]

From: Tianci Cao <ziye@zju.edu.cn>

Extend ALU variant validation to accept off == 1 and off == 2 as UHMUL and
SHMUL, respectively, for BPF_ALU64 MUL instructions. Both offsets remain
reserved for BPF_ALU MUL, since the high-half variants are only defined
for 64-bit operations. All other MUL offsets remain invalid.

For scalar tracking, compute the exact high-half result when both operands
are constants. Mark the destination unknown when either operand is variable
until range analysis for these variants is added. Keep the existing scalar
analysis for ordinary low-half MUL unchanged.

Reject UHMUL and SHMUL when either operand is PTR_TO_ARENA. Arena
arithmetic can be rewritten from BPF_ALU64 to BPF_ALU to enforce its 32-bit
pointer representation, but such a rewrite cannot preserve the semantics of
variants that are only defined for BPF_ALU64.

Link: https://lore.kernel.org/bpf/CAADnVQ+2O2zgdy11+vxE6dOp+3Br4zZBysHF-Hne7cP+eeucnQ@mail.gmail.com/
Suggested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Yazhou Tang <tangyazhou518@outlook.com>
Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
---
 kernel/bpf/verifier.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4f9d7b3de442..fbb920239783 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -16664,6 +16664,24 @@ static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
 					cnum64_from_srange(smin, smax));
 }
 
+static void scalar_min_max_uhmul(struct bpf_reg_state *dst_reg,
+				 struct bpf_reg_state *src_reg)
+{
+	u64 dst = reg_const_value(dst_reg, false);
+	u64 src = reg_const_value(src_reg, false);
+
+	___mark_reg_known(dst_reg, bpf_uhmul64(dst, src));
+}
+
+static void scalar_min_max_shmul(struct bpf_reg_state *dst_reg,
+				 struct bpf_reg_state *src_reg)
+{
+	s64 dst = reg_const_value(dst_reg, false);
+	s64 src = reg_const_value(src_reg, false);
+
+	___mark_reg_known(dst_reg, bpf_shmul64(dst, src));
+}
+
 static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg,
 				  struct bpf_reg_state *src_reg)
 {
@@ -17364,6 +17382,25 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
 		reg_step_reset(dst_reg);
 		break;
 	case BPF_MUL:
+		/*
+		 * Range analysis for UHMUL/SHMUL is not implemented yet.
+		 * Compute an exact result only when both operands are constants.
+		 * Ordinary low-half MUL continues to use the analysis below.
+		 */
+		if (bpf_insn_is_hmul(insn) &&
+		    (!is_reg_const(dst_reg, false) ||
+		     !is_reg_const(&src_reg, false))) {
+			__mark_reg_unknown(env, dst_reg);
+			break;
+		}
+		if (!alu32 && insn->off == BPF_UHMUL) {
+			scalar_min_max_uhmul(dst_reg, &src_reg);
+			break;
+		}
+		if (!alu32 && insn->off == BPF_SHMUL) {
+			scalar_min_max_shmul(dst_reg, &src_reg);
+			break;
+		}
 		scalar_step_mul(dst_reg, &src_reg);
 		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
 		scalar32_min_max_mul(dst_reg, &src_reg);
@@ -17499,6 +17536,11 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 	/* Case where at least one operand is an arena. */
 	if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) {
 
+		if (bpf_insn_is_hmul(insn)) {
+			verbose(env, "UHMUL/SHMUL with arena pointers are not supported\n");
+			return -EOPNOTSUPP;
+		}
+
 		if (dst_reg->type != PTR_TO_ARENA)
 			*dst_reg = *src_reg;
 
@@ -21197,6 +21239,8 @@ static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
 static bool is_valid_alu_variant(const struct bpf_insn *insn)
 {
 	switch (BPF_OP(insn->code)) {
+	case BPF_MUL:
+		return insn->off == 0 || bpf_insn_is_hmul(insn);
 	case BPF_DIV:
 	case BPF_MOD:
 		/* off == 1 selects SDIV/SMOD. */

-- 
2.56.0

[PATCH bpf-next v2 8/8] selftests/bpf: Add bytecode tests for UHMUL and SHMUL

[tangyazhou]

From: Yazhou Tang <tangyazhou518@outlook.com>

Add test_progs coverage for UHMUL and SHMUL with the following changes:

1. Add arithmetic execution tests in verifier_hmul.c and register the new
   test suite with the verifier test runner:
   - Gate the execution cases on x86, arm64, and RV64, matching the JIT support
     added in this series, and provide a dummy program for other targets.
   - Inject only UHMUL and SHMUL as raw .8byte values built with
     BPF_RAW_INSN(), while keeping the surrounding instructions in the
     existing assembly syntax. This avoids requiring assembler support
     for the new mnemonics.
   - Exercise register and immediate sources, signed and unsigned boundary
     values, negative immediate sign extension, and R0, R1, and R3
     destinations. The R3 case covers the x86 JIT path where BPF R3 aliases
     the implicit RDX result register. Run these cases in both privileged
     and unprivileged modes.
   - BPF_PROG_TEST_RUN exposes only the low 32 bits of R0 as its return
     value, so a direct __retval() check cannot validate the upper 32 bits.
     Build a 64-bit mismatch mask as actual ^ expected, then fold it to
     32 bits as (u32)mismatch | (u32)(mismatch >> 32) and return the folded
     value through R0. The return value is zero if and only if the complete
     64-bit result matches the expected value.

2. Add arena pointer rejection tests in verifier_hmul_arena.c. Use an
   arena pointer as the destination of UHMUL and as the source of SHMUL,
   and verify that the verifier rejects both programs.

   Keep these cases in a separate BPF object with a dedicated userspace
   runner. Probe BPF_MAP_TYPE_ARENA support before loading the object,
   and skip this suite when arena maps are unavailable, while keeping
   the arithmetic tests in verifier_hmul.c runnable. Retain
   __BPF_FEATURE_ADDR_SPACE_CAST gating and a dummy skip program for
   toolchains without addr_space_cast support.

3. Extend the illegal instruction encoding tests in verifier_value_illegal_alu.c.
   Check reserved MUL offsets and UHMUL and SHMUL encodings in the BPF_ALU
   class, reusing the existing DEFINE_BAD_OFFSET_TEST() macro.

Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
---
 tools/testing/selftests/bpf/prog_tests/verifier.c  |   2 +
 .../selftests/bpf/prog_tests/verifier_hmul_arena.c |  19 ++
 tools/testing/selftests/bpf/progs/verifier_hmul.c  | 235 +++++++++++++++++++++
 .../selftests/bpf/progs/verifier_hmul_arena.c      |  70 ++++++
 .../bpf/progs/verifier_value_illegal_alu.c         |   7 +-
 5 files changed, 332 insertions(+), 1 deletion(-)

diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 04740ba37e91..795c3e56c23e 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -59,6 +59,7 @@
 #include "verifier_helper_packet_access.skel.h"
 #include "verifier_helper_restricted.skel.h"
 #include "verifier_helper_value_access.skel.h"
+#include "verifier_hmul.skel.h"
 #include "verifier_idoms.skel.h"
 #include "verifier_int_ptr.skel.h"
 #include "verifier_iterating_callbacks.skel.h"
@@ -251,6 +252,7 @@ void test_verifier_helper_access_var_len(void) { RUN(verifier_helper_access_var_
 void test_verifier_helper_packet_access(void) { RUN(verifier_helper_packet_access); }
 void test_verifier_helper_restricted(void)    { RUN(verifier_helper_restricted); }
 void test_verifier_helper_value_access(void)  { RUN(verifier_helper_value_access); }
+void test_verifier_hmul(void)                 { RUN(verifier_hmul); }
 void test_verifier_idoms(void)                { RUN(verifier_idoms); }
 void test_verifier_int_ptr(void)              { RUN(verifier_int_ptr); }
 void test_verifier_iterating_callbacks(void)  { RUN(verifier_iterating_callbacks); }
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier_hmul_arena.c b/tools/testing/selftests/bpf/prog_tests/verifier_hmul_arena.c
new file mode 100644
index 000000000000..f9cd7b7b9781
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/verifier_hmul_arena.c
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include "verifier_hmul_arena.skel.h"
+
+void test_verifier_hmul_arena(void)
+{
+	int ret;
+
+	/* Arena support can depend on CPU features, such as Zacas on RV64. */
+	ret = libbpf_probe_bpf_map_type(BPF_MAP_TYPE_ARENA, NULL);
+	if (!ASSERT_GE(ret, 0, "probe_arena"))
+		return;
+	if (!ret) {
+		test__skip();
+		return;
+	}
+	RUN_TESTS(verifier_hmul_arena);
+}
diff --git a/tools/testing/selftests/bpf/progs/verifier_hmul.c b/tools/testing/selftests/bpf/progs/verifier_hmul.c
new file mode 100644
index 000000000000..acd92355c636
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_hmul.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bpf.h>
+#include <limits.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+#include "../../../include/linux/filter.h"
+
+#define BPF_UHMUL64_REG(DST, SRC) \
+	BPF_RAW_INSN(BPF_ALU64 | BPF_MUL | BPF_X, DST, SRC, BPF_UHMUL, 0)
+
+#define BPF_UHMUL64_IMM(DST, IMM) \
+	BPF_RAW_INSN(BPF_ALU64 | BPF_MUL | BPF_K, DST, 0, BPF_UHMUL, IMM)
+
+#define BPF_SHMUL64_REG(DST, SRC) \
+	BPF_RAW_INSN(BPF_ALU64 | BPF_MUL | BPF_X, DST, SRC, BPF_SHMUL, 0)
+
+#define BPF_SHMUL64_IMM(DST, IMM) \
+	BPF_RAW_INSN(BPF_ALU64 | BPF_MUL | BPF_K, DST, 0, BPF_SHMUL, IMM)
+
+/*
+ * Inject UHMUL/SHMUL as raw instructions so these tests do not require
+ * assembler support for the new mnemonics. All surrounding instructions use
+ * existing BPF assembly syntax supported by the baseline selftests toolchain.
+ */
+
+#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \
+	(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)
+
+/*
+ * BPF_PROG_TEST_RUN reports a 32-bit retval. Fold both halves of
+ * (actual ^ expected) into w0 so retval == 0 checks the full 64-bit result.
+ */
+
+SEC("socket")
+__description("UHMUL64, U64_MAX * U64_MAX, register source")
+__success __success_unpriv __retval(0)
+__naked void uhmul64_max_reg(void)
+{
+	asm volatile (
+	"r0 = -1;"
+	"r1 = -1;"
+	".8byte %[hmul];"
+	"r2 = -2;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_insn(hmul, BPF_UHMUL64_REG(BPF_REG_0, BPF_REG_1))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("UHMUL64, (1ULL << 63) * 2, register source")
+__success __success_unpriv __retval(0)
+__naked void uhmul64_pow2_reg(void)
+{
+	asm volatile (
+	"r0 = 2;"
+	"r1 = %[llong_min] ll;"
+	".8byte %[hmul];"
+	"r0 = r1;"
+	"r2 = 1;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_const(llong_min, LLONG_MIN),
+	  __imm_insn(hmul, BPF_UHMUL64_REG(BPF_REG_1, BPF_REG_0))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("UHMUL64, (1ULL << 63) * 2, R3 destination")
+__success __success_unpriv __retval(0)
+__naked void uhmul64_r3_dst(void)
+{
+	asm volatile (
+	"r3 = %[llong_min] ll;"
+	"r0 = 2;"
+	".8byte %[hmul];"
+	"r2 = 1;"
+	"r3 ^= r2;"
+	"r2 = r3;"
+	"r2 >>= 32;"
+	"w3 |= w2;"
+	"r0 = r3;"
+	"exit;"
+	:
+	: __imm_const(llong_min, LLONG_MIN),
+	  __imm_insn(hmul, BPF_UHMUL64_REG(BPF_REG_3, BPF_REG_0))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("UHMUL64, (1ULL << 63) * 2, immediate source")
+__success __success_unpriv __retval(0)
+__naked void uhmul64_pow2_imm(void)
+{
+	asm volatile (
+	"r0 = %[llong_min] ll;"
+	".8byte %[hmul];"
+	"r2 = 1;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_const(llong_min, LLONG_MIN),
+	  __imm_insn(hmul, BPF_UHMUL64_IMM(BPF_REG_0, 2))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("UHMUL64, (1ULL << 63) * -2, immediate source")
+__success __success_unpriv __retval(0)
+__naked void uhmul64_neg_imm(void)
+{
+	asm volatile (
+	"r0 = %[llong_min] ll;"
+	".8byte %[hmul];"
+	"r2 = %[llong_max] ll;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_const(llong_min, LLONG_MIN),
+	  __imm_const(llong_max, LLONG_MAX),
+	  __imm_insn(hmul, BPF_UHMUL64_IMM(BPF_REG_0, -2))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("SHMUL64, INT64_MAX * INT64_MAX, register source")
+__success __success_unpriv __retval(0)
+__naked void shmul64_max_reg(void)
+{
+	asm volatile (
+	"r0 = %[llong_max] ll;"
+	"r1 = %[llong_max] ll;"
+	".8byte %[hmul];"
+	"r2 = %[expected] ll;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_const(expected, 0x3fffffffffffffffULL),
+	  __imm_const(llong_max, LLONG_MAX),
+	  __imm_insn(hmul, BPF_SHMUL64_REG(BPF_REG_0, BPF_REG_1))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("SHMUL64, INT64_MIN * -2, register source")
+__success __success_unpriv __retval(0)
+__naked void shmul64_min_neg2_reg(void)
+{
+	asm volatile (
+	"r0 = %[llong_min] ll;"
+	"r1 = -2;"
+	".8byte %[hmul];"
+	"r2 = 1;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_const(llong_min, LLONG_MIN),
+	  __imm_insn(hmul, BPF_SHMUL64_REG(BPF_REG_0, BPF_REG_1))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("SHMUL64, 1 * -2, immediate source")
+__success __success_unpriv __retval(0)
+__naked void shmul64_neg_imm(void)
+{
+	asm volatile (
+	"r0 = 1;"
+	".8byte %[hmul];"
+	"r2 = -1;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_insn(hmul, BPF_SHMUL64_IMM(BPF_REG_0, -2))
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("SHMUL64, INT64_MIN * 2, immediate source")
+__success __success_unpriv __retval(0)
+__naked void shmul64_min_imm(void)
+{
+	asm volatile (
+	"r0 = %[llong_min] ll;"
+	".8byte %[hmul];"
+	"r2 = -1;"
+	"r0 ^= r2;"
+	"r2 = r0;"
+	"r2 >>= 32;"
+	"w0 |= w2;"
+	"exit;"
+	:
+	: __imm_const(llong_min, LLONG_MIN),
+	  __imm_insn(hmul, BPF_SHMUL64_IMM(BPF_REG_0, 2))
+	: __clobber_all);
+}
+
+#else
+
+SEC("socket")
+__description("UHMUL/SHMUL are not supported by this JIT, use a dummy test")
+__skip("UHMUL/SHMUL are not supported by this jit")
+__success
+int dummy_test(void)
+{
+	return 0;
+}
+
+#endif
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_hmul_arena.c b/tools/testing/selftests/bpf/progs/verifier_hmul_arena.c
new file mode 100644
index 000000000000..fc0ca88114d0
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_hmul_arena.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+#include "../../../include/linux/filter.h"
+
+#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
+
+#define BPF_UHMUL64_IMM(DST, IMM) \
+	BPF_RAW_INSN(BPF_ALU64 | BPF_MUL | BPF_K, DST, 0, BPF_UHMUL, IMM)
+
+#define BPF_SHMUL64_REG(DST, SRC) \
+	BPF_RAW_INSN(BPF_ALU64 | BPF_MUL | BPF_X, DST, SRC, BPF_SHMUL, 0)
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARENA);
+	__uint(map_flags, BPF_F_MMAPABLE);
+	__uint(max_entries, 1);
+} arena SEC(".maps");
+
+SEC("syscall")
+__description("UHMUL64 with arena pointer dst")
+__failure __msg("UHMUL/SHMUL with arena pointers are not supported")
+__naked void uhmul64_arena_ptr(void)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r0 = 1;"
+	"r0 = addr_space_cast(r0, 0x0, 0x1);"
+	".8byte %[hmul];"
+	"exit;"
+	:
+	: __imm_addr(arena),
+	  __imm_insn(hmul, BPF_UHMUL64_IMM(BPF_REG_0, 2))
+	: __clobber_all);
+}
+
+SEC("syscall")
+__description("SHMUL64 with arena pointer src")
+__failure __msg("UHMUL/SHMUL with arena pointers are not supported")
+__naked void shmul64_arena_ptr(void)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r1 = 1;"
+	"r1 = addr_space_cast(r1, 0x0, 0x1);"
+	"r0 = 2;"
+	".8byte %[hmul];"
+	"exit;"
+	:
+	: __imm_addr(arena),
+	  __imm_insn(hmul, BPF_SHMUL64_REG(BPF_REG_0, BPF_REG_1))
+	: __clobber_all);
+}
+
+#else
+
+SEC("socket")
+__description("HMUL arena tests require compiler support for addr_space_cast")
+__skip("compiler doesn't support addr_space_cast")
+__success
+int dummy_test(void)
+{
+	return 0;
+}
+
+#endif
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c
index 31663338866d..209d5be55155 100644
--- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c
+++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c
@@ -367,7 +367,8 @@ __naked void flow_keys_illegal_variable_offset_alu(void)
 
 /*
  * Offset fields of 0 and 1 are legal for BPF_{DIV,MOD} instructions.
- * Offset fields of 0 are legal for the rest of ALU instructions.
+ * Offset fields of 0, 1 and 2 are legal for BPF_ALU64 | BPF_MUL.
+ * Only offset 0 is legal for BPF_ALU | BPF_MUL.
  * Test that error is reported for illegal offsets, assuming that tests
  * for legal offsets exist.
  */
@@ -377,5 +378,9 @@ DEFINE_BAD_OFFSET_TEST(bad_offset_addx, BPF_ALU64 | BPF_ADD | BPF_X, -1, 0)
 DEFINE_BAD_OFFSET_TEST(bad_offset_divx2, BPF_ALU64 | BPF_DIV | BPF_X, 2, 0)
 DEFINE_BAD_OFFSET_TEST(bad_offset_modk2, BPF_ALU64 | BPF_MOD | BPF_K, 2, 1)
 DEFINE_BAD_OFFSET_TEST(bad_offset_addx2, BPF_ALU64 | BPF_ADD | BPF_X, 1, 0)
+DEFINE_BAD_OFFSET_TEST(bad_offset_mulx, BPF_ALU64 | BPF_MUL | BPF_X, -1, 0)
+DEFINE_BAD_OFFSET_TEST(bad_offset_mulk, BPF_ALU64 | BPF_MUL | BPF_K, 3, 0)
+DEFINE_BAD_OFFSET_TEST(bad_uhmul32_x, BPF_ALU | BPF_MUL | BPF_X, BPF_UHMUL, 0)
+DEFINE_BAD_OFFSET_TEST(bad_shmul32_k, BPF_ALU | BPF_MUL | BPF_K, BPF_SHMUL, 0)
 
 char _license[] SEC("license") = "GPL";

-- 
2.56.0

Re: [PATCH bpf-next v2 1/8] bpf: Add UHMUL and SHMUL instructions

[Alexei Starovoitov]

On Tue, Oct 20, 2026 at 10:40 PM tangyazhou@zju.edu.cn <tangyazhou@zju.edu.cn> wrote:

> +static inline bool bpf_insn_is_hmul(const struct bpf_insn *insn)
> +{
> +	return BPF_CLASS(insn->code) == BPF_ALU64 &&
> +	       BPF_OP(insn->code) == BPF_MUL &&
> +	       bpf_is_hmul_variant(insn->off);
> +}

Patch 5 adds this check to 13 JITs and nfp to opt them out.
Lets make JITs opt in instead, like bpf_jit_supports_callx().
The verifier checks it once. x86, arm64 and riscv patches add
bpf_jit_supports_hmul() that returns true. bpf_prog_is_offloaded()
covers nfp.
When JIT is on and it doesn't support them the prog is rejected with
a message in the log instead of silent fallback to the interpreter.
Then patch 5 goes away and only arm64 and riscv acks are needed.

Like:

diff --git a/include/linux/filter.h b/include/linux/filter.h
index e0218e6..6ee8d1a 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -152,6 +152,13 @@ struct ctl_table_header;
 #define BPF_ALU32_IMM(OP, DST, IMM)				\
 	BPF_ALU32_IMM_OFF(OP, DST, IMM, 0)

+static inline bool bpf_insn_is_hmul(const struct bpf_insn *insn)
+{
+	return BPF_CLASS(insn->code) == BPF_ALU64 &&
+	       BPF_OP(insn->code) == BPF_MUL &&
+	       (insn->off == BPF_UHMUL || insn->off == BPF_SHMUL);
+}
+
 /* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */

 #define BPF_ENDIAN(TYPE, DST, LEN)				\
@@ -1281,6 +1288,7 @@ bool bpf_jit_supports_kfunc_call(void);
 bool bpf_jit_supports_callx(void);
 bool bpf_jit_supports_kfunc_ret_reg_pair(void);
 bool bpf_jit_supports_unwind(void);
+bool bpf_jit_supports_hmul(void);
 bool bpf_jit_supports_stack_args(void);
 bool bpf_jit_supports_arena_args(void);
 bool bpf_jit_supports_far_kfunc_call(void);
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index f29b0aa..90c30e5 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -34,6 +34,12 @@
 #define BPF_FROM_LE	BPF_TO_LE
 #define BPF_FROM_BE	BPF_TO_BE

+/* BPF_MUL off values for high-half multiplication; BPF_ALU64 only. */
+enum bpf_mul_variant {
+	BPF_UHMUL = 1,
+	BPF_SHMUL = 2,
+};
+
 /* jmp encodings */
 #define BPF_JNE		0x50	/* jump != */
 #define BPF_JLT		0xa0	/* LT is unsigned, '<' */
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 3030beb..e17704a 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3405,6 +3405,12 @@ bool __weak bpf_jit_supports_unwind(void)
 	return false;
 }

+/* uhmul and shmul */
+bool __weak bpf_jit_supports_hmul(void)
+{
+	return false;
+}
+
 /*
  * How this arch places a by-value kfunc argument, or NULL for one that has
  * not opted in and so only takes an argument of a single eightbyte, which
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5601d20..9a3c97d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21194,6 +21194,17 @@ static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
 	return -EPROTO;
 }

+/* The interpreter supports uhmul and shmul. JITs opt in. */
+static int check_hmul_jit(struct bpf_verifier_env *env)
+{
+	if (bpf_prog_is_offloaded(env->prog->aux) ||
+	    (env->prog->jit_requested && !bpf_jit_supports_hmul())) {
+		verbose(env, "JIT doesn't support uhmul and shmul\n");
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
 static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 {
 	u8 class = BPF_CLASS(insn->code);
@@ -21249,6 +21260,13 @@ static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 	case BPF_MUL:
 	case BPF_DIV:
 	case BPF_MOD:
+		if (bpf_insn_is_hmul(insn)) {
+			if (BPF_SRC(insn->code) == BPF_X ? insn->imm : insn->src_reg) {
+				verbose(env, "BPF_ALU uses reserved fields\n");
+				return -EINVAL;
+			}
+			return check_hmul_jit(env);
+		}
 		if (BPF_SRC(insn->code) == BPF_X) {
 			if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) ||
 			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {

pw-bot: cr

Re: [PATCH bpf-next v2 1/8] bpf: Add UHMUL and SHMUL instructions

[Yazhou Tang]

On 10/22/26 01:11, Alexei Starovoitov wrote:
> On Tue, Oct 20, 2026 at 10:40 PM tangyazhou@zju.edu.cn <tangyazhou@zju.edu.cn> wrote:
> 
>> +static inline bool bpf_insn_is_hmul(const struct bpf_insn *insn)
>> +{
>> +	return BPF_CLASS(insn->code) == BPF_ALU64 &&
>> +	       BPF_OP(insn->code) == BPF_MUL &&
>> +	       bpf_is_hmul_variant(insn->off);
>> +}
> 
> Patch 5 adds this check to 13 JITs and nfp to opt them out.
> Lets make JITs opt in instead, like bpf_jit_supports_callx().
> The verifier checks it once. x86, arm64 and riscv patches add
> bpf_jit_supports_hmul() that returns true. bpf_prog_is_offloaded()
> covers nfp.
> When JIT is on and it doesn't support them the prog is rejected with
> a message in the log instead of silent fallback to the interpreter.
> Then patch 5 goes away and only arm64 and riscv acks are needed.

Thanks, Alexei. This is a cleaner approach. I'll add
bpf_jit_supports_hmul() as you suggested and remove
the opt-out checks from unsupported JITs in v3.

Apologies for the noise to the other JIT maintainers.

> 
> Like:
> 
> diff --git a/include/linux/filter.h b/include/linux/filter.h
> index e0218e6..6ee8d1a 100644
> --- a/include/linux/filter.h
> +++ b/include/linux/filter.h
> @@ -152,6 +152,13 @@ struct ctl_table_header;
>   #define BPF_ALU32_IMM(OP, DST, IMM)				\
>   	BPF_ALU32_IMM_OFF(OP, DST, IMM, 0)
> 
> +static inline bool bpf_insn_is_hmul(const struct bpf_insn *insn)
> +{
> +	return BPF_CLASS(insn->code) == BPF_ALU64 &&
> +	       BPF_OP(insn->code) == BPF_MUL &&
> +	       (insn->off == BPF_UHMUL || insn->off == BPF_SHMUL);
> +}
> +
>   /* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */
> 
>   #define BPF_ENDIAN(TYPE, DST, LEN)				\
> @@ -1281,6 +1288,7 @@ bool bpf_jit_supports_kfunc_call(void);
>   bool bpf_jit_supports_callx(void);
>   bool bpf_jit_supports_kfunc_ret_reg_pair(void);
>   bool bpf_jit_supports_unwind(void);
> +bool bpf_jit_supports_hmul(void);
>   bool bpf_jit_supports_stack_args(void);
>   bool bpf_jit_supports_arena_args(void);
>   bool bpf_jit_supports_far_kfunc_call(void);
> diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
> index f29b0aa..90c30e5 100644
> --- a/include/uapi/linux/bpf.h
> +++ b/include/uapi/linux/bpf.h
> @@ -34,6 +34,12 @@
>   #define BPF_FROM_LE	BPF_TO_LE
>   #define BPF_FROM_BE	BPF_TO_BE
> 
> +/* BPF_MUL off values for high-half multiplication; BPF_ALU64 only. */
> +enum bpf_mul_variant {
> +	BPF_UHMUL = 1,
> +	BPF_SHMUL = 2,
> +};
> +
>   /* jmp encodings */
>   #define BPF_JNE		0x50	/* jump != */
>   #define BPF_JLT		0xa0	/* LT is unsigned, '<' */
> diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
> index 3030beb..e17704a 100644
> --- a/kernel/bpf/core.c
> +++ b/kernel/bpf/core.c
> @@ -3405,6 +3405,12 @@ bool __weak bpf_jit_supports_unwind(void)
>   	return false;
>   }
> 
> +/* uhmul and shmul */
> +bool __weak bpf_jit_supports_hmul(void)
> +{
> +	return false;
> +}
> +
>   /*
>    * How this arch places a by-value kfunc argument, or NULL for one that has
>    * not opted in and so only takes an argument of a single eightbyte, which
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 5601d20..9a3c97d 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -21194,6 +21194,17 @@ static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
>   	return -EPROTO;
>   }
> 
> +/* The interpreter supports uhmul and shmul. JITs opt in. */
> +static int check_hmul_jit(struct bpf_verifier_env *env)
> +{
> +	if (bpf_prog_is_offloaded(env->prog->aux) ||
> +	    (env->prog->jit_requested && !bpf_jit_supports_hmul())) {
> +		verbose(env, "JIT doesn't support uhmul and shmul\n");
> +		return -EOPNOTSUPP;
> +	}
> +	return 0;
> +}
> +
>   static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
>   {
>   	u8 class = BPF_CLASS(insn->code);
> @@ -21249,6 +21260,13 @@ static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
>   	case BPF_MUL:
>   	case BPF_DIV:
>   	case BPF_MOD:
> +		if (bpf_insn_is_hmul(insn)) {
> +			if (BPF_SRC(insn->code) == BPF_X ? insn->imm : insn->src_reg) {
> +				verbose(env, "BPF_ALU uses reserved fields\n");
> +				return -EINVAL;
> +			}
> +			return check_hmul_jit(env);
> +		}
>   		if (BPF_SRC(insn->code) == BPF_X) {
>   			if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) ||
>   			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
> 
> pw-bot: cr