[PATCH bpf-next v5 0/5] emit ENDBR/BTI instructions for indirect jump targets

[PATCH bpf-next v5 0/5] emit ENDBR/BTI instructions for indirect jump targets

[Xu Kuohai]

On x86 CPUs with CET/IBT and arm64 CPUs with BTI, missing landing pad instructions
at indirect jump targets triggers kernel panic. So emit ENDBR instructions for
indirect jump targets on x86 and BTI on arm64. Indirect jump targets are identified
based on the insn_aux_data created by the verifier.

v5:
- Switch to pass env to JIT directly to get rid of coping private insn_aux_data for
  each prog

v4: https://lore.kernel.org/all/20260114093914.2403982-1-xukuohai@huaweicloud.com/
- Switch to the approach proposed by Eduard, using insn_aux_data to indentify indirect
  jump targets, and emit ENDBR on x86

v3: https://lore.kernel.org/bpf/20251227081033.240336-1-xukuohai@huaweicloud.com/
- Get rid of unnecessary enum definition (Yonghong Song, Anton Protopopov)

v2: https://lore.kernel.org/bpf/20251223085447.139301-1-xukuohai@huaweicloud.com/
- Exclude instruction arrays not used for indirect jumps (Anton Protopopov)

v1: https://lore.kernel.org/bpf/20251127140318.3944249-1-xukuohai@huaweicloud.com/

Xu Kuohai (5):
  bpf: Move JIT for single-subprog programs to verifier
  bpf: Pass bpf_verifier_env to jit
  bpf: Add helper to detect indirect jump targets
  bpf, x86: Emit ENDBR for indirect jump targets
  bpf, arm64: Emit BTI for indirect jump target

 arch/arc/net/bpf_jit_core.c      | 19 +++----
 arch/arm/net/bpf_jit_32.c        |  4 +-
 arch/arm64/net/bpf_jit_comp.c    | 21 ++++----
 arch/loongarch/net/bpf_jit.c     |  4 +-
 arch/mips/net/bpf_jit_comp.c     |  4 +-
 arch/parisc/net/bpf_jit_core.c   |  4 +-
 arch/powerpc/net/bpf_jit_comp.c  |  4 +-
 arch/riscv/net/bpf_jit_core.c    |  4 +-
 arch/s390/net/bpf_jit_comp.c     |  4 +-
 arch/sparc/net/bpf_jit_comp_64.c |  4 +-
 arch/x86/net/bpf_jit_comp.c      | 25 +++++----
 arch/x86/net/bpf_jit_comp32.c    |  4 +-
 include/linux/bpf.h              |  2 +
 include/linux/bpf_verifier.h     | 10 ++--
 include/linux/filter.h           |  6 ++-
 kernel/bpf/core.c                | 93 ++++++++++++++++++++++++--------
 kernel/bpf/syscall.c             |  2 +-
 kernel/bpf/verifier.c            | 22 ++++++--
 18 files changed, 157 insertions(+), 79 deletions(-)

-- 
2.47.3

[PATCH bpf-next v5 1/5] bpf: Move JIT for single-subprog programs to verifier

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

JIT for single-subprog programs is done after the verification stage. This
prevents the JIT stage from accessing the verifier's internal datas, like
env->insn_aux_data. So move it to the verifier. After the movement, all bpf
progs loaded with bpf_prog_load() are JITed in the verifier. The JIT in
bpf_prog_select_runtime() is preserved for bpf_migrate_filter() and test
cases.

Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 include/linux/filter.h |  2 ++
 kernel/bpf/core.c      | 51 +++++++++++++++++++++++++++---------------
 kernel/bpf/syscall.c   |  2 +-
 kernel/bpf/verifier.c  |  7 +++++-
 4 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 44d7ae95ddbc..632c03e126d9 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1108,6 +1108,8 @@ static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb,
 	return sk_filter_trim_cap(sk, skb, 1, reason);
 }
 
+struct bpf_prog *bpf_prog_select_jit(struct bpf_prog *fp, int *err);
+struct bpf_prog *__bpf_prog_select_runtime(struct bpf_prog *fp, bool jit_attempted, int *err);
 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
 void bpf_prog_free(struct bpf_prog *fp);
 
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 229c74f3d6ae..00be578a438d 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2505,18 +2505,18 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
 	return select_interpreter;
 }
 
-/**
- *	bpf_prog_select_runtime - select exec runtime for BPF program
- *	@fp: bpf_prog populated with BPF program
- *	@err: pointer to error variable
- *
- * Try to JIT eBPF program, if JIT is not available, use interpreter.
- * The BPF program will be executed via bpf_prog_run() function.
- *
- * Return: the &fp argument along with &err set to 0 for success or
- * a negative errno code on failure
- */
-struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
+struct bpf_prog *bpf_prog_select_jit(struct bpf_prog *fp, int *err)
+{
+	*err = bpf_prog_alloc_jited_linfo(fp);
+	if (*err)
+		return fp;
+
+	fp = bpf_int_jit_compile(fp);
+	bpf_prog_jit_attempt_done(fp);
+	return fp;
+}
+
+struct bpf_prog *__bpf_prog_select_runtime(struct bpf_prog *fp, bool jit_attempted, int *err)
 {
 	/* In case of BPF to BPF calls, verifier did all the prep
 	 * work with regards to JITing, etc.
@@ -2540,12 +2540,11 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
 	 * be JITed, but falls back to the interpreter.
 	 */
 	if (!bpf_prog_is_offloaded(fp->aux)) {
-		*err = bpf_prog_alloc_jited_linfo(fp);
-		if (*err)
-			return fp;
-
-		fp = bpf_int_jit_compile(fp);
-		bpf_prog_jit_attempt_done(fp);
+		if (!jit_attempted) {
+			fp = bpf_prog_select_jit(fp, err);
+			if (*err)
+				return fp;
+		}
 		if (!fp->jited && jit_needed) {
 			*err = -ENOTSUPP;
 			return fp;
@@ -2570,6 +2569,22 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
 
 	return fp;
 }
+
+/**
+ *	bpf_prog_select_runtime - select exec runtime for BPF program
+ *	@fp: bpf_prog populated with BPF program
+ *	@err: pointer to error variable
+ *
+ * Try to JIT eBPF program, if JIT is not available, use interpreter.
+ * The BPF program will be executed via bpf_prog_run() function.
+ *
+ * Return: the &fp argument along with &err set to 0 for success or
+ * a negative errno code on failure
+ */
+struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
+{
+	return __bpf_prog_select_runtime(fp, false, err);
+}
 EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
 
 static unsigned int __bpf_prog_ret1(const void *ctx,
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 274039e36465..d6982107ba80 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3090,7 +3090,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
 	if (err < 0)
 		goto free_used_maps;
 
-	prog = bpf_prog_select_runtime(prog, &err);
+	prog = __bpf_prog_select_runtime(prog, true, &err);
 	if (err < 0)
 		goto free_used_maps;
 
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index fc4ccd1de569..ab2bc0850770 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -26086,6 +26086,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 		convert_pseudo_ld_imm64(env);
 	}
 
+	/* constants blinding in the JIT may increase prog->len */
+	len = env->prog->len;
+	if (env->subprog_cnt == 1)
+		env->prog = bpf_prog_select_jit(env->prog, &ret);
+
 	adjust_btf_func(env);
 
 err_release_maps:
@@ -26111,7 +26116,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 err_unlock:
 	if (!is_priv)
 		mutex_unlock(&bpf_verifier_lock);
-	clear_insn_aux_data(env, 0, env->prog->len);
+	clear_insn_aux_data(env, 0, len);
 	vfree(env->insn_aux_data);
 err_free_env:
 	bpf_stack_liveness_free(env);
-- 
2.47.3

[PATCH bpf-next v5 2/5] bpf: Pass bpf_verifier_env to jit

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

Pass bpf_verifier_env to bpf_int_jit_compile() and bpf_jit_blind_constants().
The follow-up patch will use env->insn_aux_data in the JIT stage to detect
indirect jump targets.

Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 arch/arc/net/bpf_jit_core.c      | 19 ++++++++++---------
 arch/arm/net/bpf_jit_32.c        |  4 ++--
 arch/arm64/net/bpf_jit_comp.c    |  4 ++--
 arch/loongarch/net/bpf_jit.c     |  4 ++--
 arch/mips/net/bpf_jit_comp.c     |  4 ++--
 arch/parisc/net/bpf_jit_core.c   |  4 ++--
 arch/powerpc/net/bpf_jit_comp.c  |  4 ++--
 arch/riscv/net/bpf_jit_core.c    |  4 ++--
 arch/s390/net/bpf_jit_comp.c     |  4 ++--
 arch/sparc/net/bpf_jit_comp_64.c |  4 ++--
 arch/x86/net/bpf_jit_comp.c      |  4 ++--
 arch/x86/net/bpf_jit_comp32.c    |  4 ++--
 include/linux/filter.h           |  6 +++---
 kernel/bpf/core.c                | 10 +++++-----
 kernel/bpf/verifier.c            |  6 +++---
 15 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c
index 1421eeced0f5..076aaf52cb80 100644
--- a/arch/arc/net/bpf_jit_core.c
+++ b/arch/arc/net/bpf_jit_core.c
@@ -157,14 +157,15 @@ static void jit_dump(const struct jit_context *ctx)
 }
 
 /* Initialise the context so there's no garbage. */
-static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog)
+static int jit_ctx_init(struct jit_context *ctx, struct bpf_verifier_env *env,
+			struct bpf_prog *prog)
 {
 	memset(ctx, 0, sizeof(*ctx));
 
 	ctx->orig_prog = prog;
 
 	/* If constant blinding was requested but failed, scram. */
-	ctx->prog = bpf_jit_blind_constants(prog);
+	ctx->prog = bpf_jit_blind_constants(env, prog);
 	if (IS_ERR(ctx->prog))
 		return PTR_ERR(ctx->prog);
 	ctx->blinded = (ctx->prog != ctx->orig_prog);
@@ -1335,7 +1336,7 @@ static int jit_patch_relocations(struct jit_context *ctx)
  * to get the necessary data for the real compilation phase,
  * jit_compile().
  */
-static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
+static struct bpf_prog *do_normal_pass(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct jit_context ctx;
 
@@ -1343,7 +1344,7 @@ static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return prog;
 
-	if (jit_ctx_init(&ctx, prog)) {
+	if (jit_ctx_init(&ctx, env, prog)) {
 		jit_ctx_cleanup(&ctx);
 		return prog;
 	}
@@ -1374,7 +1375,7 @@ static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
  * again to get the newly translated addresses in order to resolve
  * the "call"s.
  */
-static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
+static struct bpf_prog *do_extra_pass(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct jit_context ctx;
 
@@ -1382,7 +1383,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
 	if (check_jit_context(prog))
 		return prog;
 
-	if (jit_ctx_init(&ctx, prog)) {
+	if (jit_ctx_init(&ctx, env, prog)) {
 		jit_ctx_cleanup(&ctx);
 		return prog;
 	}
@@ -1411,15 +1412,15 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
  * (re)locations involved that their addresses are not known
  * during the first run.
  */
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	vm_dump(prog);
 
 	/* Was this program already translated? */
 	if (!prog->jited)
-		return do_normal_pass(prog);
+		return do_normal_pass(env, prog);
 	else
-		return do_extra_pass(prog);
+		return do_extra_pass(env, prog);
 
 	return prog;
 }
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index deeb8f292454..9c07cbf1dbfc 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -2142,7 +2142,7 @@ bool bpf_jit_needs_zext(void)
 	return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_prog *tmp, *orig_prog = prog;
 	struct bpf_binary_header *header;
@@ -2162,7 +2162,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	 * then we must fall back to the interpreter. Otherwise, we save
 	 * the new JITed code.
 	 */
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 
 	if (IS_ERR(tmp))
 		return orig_prog;
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index adf84962d579..823246c7ff5d 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2006,7 +2006,7 @@ struct arm64_jit_data {
 	struct jit_ctx ctx;
 };
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	int image_size, prog_size, extable_size, extable_align, extable_offset;
 	struct bpf_prog *tmp, *orig_prog = prog;
@@ -2027,7 +2027,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	/* If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter.
 	 */
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 3bd89f55960d..b578b176ef01 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1909,7 +1909,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	bool tmp_blinded = false, extra_pass = false;
 	u8 *image_ptr, *ro_image_ptr;
@@ -1927,7 +1927,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	/*
 	 * If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter. Otherwise, we save
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index e355dfca4400..faf0ba098a86 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -909,7 +909,7 @@ bool bpf_jit_needs_zext(void)
 	return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_prog *tmp, *orig_prog = prog;
 	struct bpf_binary_header *header = NULL;
@@ -931,7 +931,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	 * then we must fall back to the interpreter. Otherwise, we save
 	 * the new JITed code.
 	 */
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	if (IS_ERR(tmp))
 		return orig_prog;
 	if (tmp != prog) {
diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c
index a5eb6b51e27a..e85b6e336b19 100644
--- a/arch/parisc/net/bpf_jit_core.c
+++ b/arch/parisc/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
 	return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	unsigned int prog_size = 0, extable_size = 0;
 	bool tmp_blinded = false, extra_pass = false;
@@ -53,7 +53,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	if (IS_ERR(tmp))
 		return orig_prog;
 	if (tmp != prog) {
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 52162e4a7f84..fb77e8beb161 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -129,7 +129,7 @@ bool bpf_jit_needs_zext(void)
 	return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
 {
 	u32 proglen;
 	u32 alloclen;
@@ -154,7 +154,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	if (!fp->jit_requested)
 		return org_fp;
 
-	tmp_fp = bpf_jit_blind_constants(org_fp);
+	tmp_fp = bpf_jit_blind_constants(env, org_fp);
 	if (IS_ERR(tmp_fp))
 		return org_fp;
 
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index b3581e926436..ce157319459f 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
 	return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	unsigned int prog_size = 0, extable_size = 0;
 	bool tmp_blinded = false, extra_pass = false;
@@ -53,7 +53,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	if (IS_ERR(tmp))
 		return orig_prog;
 	if (tmp != prog) {
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 211226748662..84aabfc8a9d6 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2303,7 +2303,7 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
 /*
  * Compile eBPF program "fp"
  */
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
 {
 	struct bpf_prog *tmp, *orig_fp = fp;
 	struct bpf_binary_header *header;
@@ -2316,7 +2316,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	if (!fp->jit_requested)
 		return orig_fp;
 
-	tmp = bpf_jit_blind_constants(fp);
+	tmp = bpf_jit_blind_constants(env, fp);
 	/*
 	 * If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter.
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index b23d1c645ae5..55da61ca2967 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -1477,7 +1477,7 @@ struct sparc64_jit_data {
 	struct jit_ctx ctx;
 };
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_prog *tmp, *orig_prog = prog;
 	struct sparc64_jit_data *jit_data;
@@ -1492,7 +1492,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	/* If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter.
 	 */
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 8f10080e6fe3..43beacaed56d 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3722,7 +3722,7 @@ struct x64_jit_data {
 #define MAX_PASSES 20
 #define PADDING_PASSES (MAX_PASSES - 5)
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_binary_header *rw_header = NULL;
 	struct bpf_binary_header *header = NULL;
@@ -3744,7 +3744,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	/*
 	 * If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter.
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index dda423025c3d..957f7aa951ba 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -2518,7 +2518,7 @@ bool bpf_jit_needs_zext(void)
 	return true;
 }
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_binary_header *header = NULL;
 	struct bpf_prog *tmp, *orig_prog = prog;
@@ -2533,7 +2533,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	if (!prog->jit_requested)
 		return orig_prog;
 
-	tmp = bpf_jit_blind_constants(prog);
+	tmp = bpf_jit_blind_constants(env, prog);
 	/*
 	 * If blinding was requested and we failed during blinding,
 	 * we must fall back to the interpreter.
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 632c03e126d9..8b5e9ac9eee4 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1108,7 +1108,7 @@ static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb,
 	return sk_filter_trim_cap(sk, skb, 1, reason);
 }
 
-struct bpf_prog *bpf_prog_select_jit(struct bpf_prog *fp, int *err);
+struct bpf_prog *bpf_prog_select_jit(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err);
 struct bpf_prog *__bpf_prog_select_runtime(struct bpf_prog *fp, bool jit_attempted, int *err);
 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
 void bpf_prog_free(struct bpf_prog *fp);
@@ -1155,7 +1155,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
 	((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
 	 (void *)__bpf_call_base)
 
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
 void bpf_jit_compile(struct bpf_prog *prog);
 bool bpf_jit_needs_zext(void);
 bool bpf_jit_inlines_helper_call(s32 imm);
@@ -1312,7 +1312,7 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 
 const char *bpf_jit_get_prog_name(struct bpf_prog *prog);
 
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
+struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog);
 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
 
 static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 00be578a438d..7702c232c62e 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1486,7 +1486,7 @@ static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
 #endif
 }
 
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
+struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_insn insn_buff[16], aux[2];
 	struct bpf_prog *clone, *tmp;
@@ -2505,13 +2505,13 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
 	return select_interpreter;
 }
 
-struct bpf_prog *bpf_prog_select_jit(struct bpf_prog *fp, int *err)
+struct bpf_prog *bpf_prog_select_jit(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err)
 {
 	*err = bpf_prog_alloc_jited_linfo(fp);
 	if (*err)
 		return fp;
 
-	fp = bpf_int_jit_compile(fp);
+	fp = bpf_int_jit_compile(env, fp);
 	bpf_prog_jit_attempt_done(fp);
 	return fp;
 }
@@ -2541,7 +2541,7 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_prog *fp, bool jit_attempt
 	 */
 	if (!bpf_prog_is_offloaded(fp->aux)) {
 		if (!jit_attempted) {
-			fp = bpf_prog_select_jit(fp, err);
+			fp = bpf_prog_select_jit(NULL, fp, err);
 			if (*err)
 				return fp;
 		}
@@ -3072,7 +3072,7 @@ const struct bpf_func_proto bpf_tail_call_proto = {
  * It is encouraged to implement bpf_int_jit_compile() instead, so that
  * eBPF and implicitly also cBPF can get JITed!
  */
-struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	return prog;
 }
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ab2bc0850770..1d2d42078ddf 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22844,7 +22844,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 		 * all instruction adjustments should be accumulated
 		 */
 		old_len = func[i]->len;
-		func[i] = bpf_int_jit_compile(func[i]);
+		func[i] = bpf_int_jit_compile(env, func[i]);
 		subprog_start_adjustment += func[i]->len - old_len;
 
 		if (!func[i]->jited) {
@@ -22890,7 +22890,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 	}
 	for (i = 0; i < env->subprog_cnt; i++) {
 		old_bpf_func = func[i]->bpf_func;
-		tmp = bpf_int_jit_compile(func[i]);
+		tmp = bpf_int_jit_compile(env, func[i]);
 		if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
 			err = -ENOTSUPP;
@@ -26089,7 +26089,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 	/* constants blinding in the JIT may increase prog->len */
 	len = env->prog->len;
 	if (env->subprog_cnt == 1)
-		env->prog = bpf_prog_select_jit(env->prog, &ret);
+		env->prog = bpf_prog_select_jit(env, env->prog, &ret);
 
 	adjust_btf_func(env);
 
-- 
2.47.3

[PATCH bpf-next v5 3/5] bpf: Add helper to detect indirect jump targets

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

Introduce helper bpf_insn_is_indirect_target to determine whether a BPF
instruction is an indirect jump target. This helper will be used by
follow-up patches to decide where to emit indirect landing pad instructions.

Add a new flag to struct bpf_insn_aux_data to mark instructions that are
indirect jump targets. The BPF verifier sets this flag, and the helper
checks it to determine whether an instruction is an indirect jump target.

Also add a new field to struct bpf_insn_aux_data to track the instruction
final index in the bpf prog, as the instructions may be rewritten by
constant blinding in the JIT stage. This field is used as a binary search
key to find the corresponding insn_aux_data for a given instruction.

Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 include/linux/bpf.h          |  2 ++
 include/linux/bpf_verifier.h | 10 ++++++----
 kernel/bpf/core.c            | 38 +++++++++++++++++++++++++++++++++---
 kernel/bpf/verifier.c        | 13 +++++++++++-
 4 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 05b34a6355b0..90760e250865 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1541,6 +1541,8 @@ bool bpf_has_frame_pointer(unsigned long ip);
 int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
+bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
+				 int insn_idx);
 #else
 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
 					   struct bpf_trampoline *tr,
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index c1e30096ea7b..f8f70e5414f0 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -577,16 +577,18 @@ struct bpf_insn_aux_data {
 
 	/* below fields are initialized once */
 	unsigned int orig_idx; /* original instruction index */
-	bool jmp_point;
-	bool prune_point;
+	unsigned int final_idx; /* final instruction index */
+	u32 jmp_point:1;
+	u32 prune_point:1;
 	/* ensure we check state equivalence and save state checkpoint and
 	 * this instruction, regardless of any heuristics
 	 */
-	bool force_checkpoint;
+	u32 force_checkpoint:1;
 	/* true if instruction is a call to a helper function that
 	 * accepts callback function as a parameter.
 	 */
-	bool calls_callback;
+	u32 calls_callback:1;
+	u32 indirect_target:1; /* if it is an indirect jump target */
 	/*
 	 * CFG strongly connected component this instruction belongs to,
 	 * zero if it is a singleton SCC.
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 7702c232c62e..9a760cf43d68 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1486,13 +1486,41 @@ static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
 #endif
 }
 
+static int bpf_insn_aux_cmp_by_insn_idx(const void *a, const void *b)
+{
+	int insn_idx = *(int *)a;
+	int final_idx = ((const struct bpf_insn_aux_data *)b)->final_idx;
+
+	return insn_idx - final_idx;
+}
+
+bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
+				 int insn_idx)
+{
+	struct bpf_insn_aux_data *insn_aux;
+	int func_idx, subprog_start, subprog_end;
+
+	if (!env)
+		return false;
+
+	func_idx = prog->aux->func_idx;
+	subprog_start = env->subprog_info[func_idx].start;
+	subprog_end = env->subprog_info[func_idx + 1].start;
+
+	insn_aux = bsearch(&insn_idx, &env->insn_aux_data[subprog_start],
+			   subprog_end - subprog_start,
+			   sizeof(struct bpf_insn_aux_data), bpf_insn_aux_cmp_by_insn_idx);
+
+	return insn_aux && insn_aux->indirect_target;
+}
+
 struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
 {
 	struct bpf_insn insn_buff[16], aux[2];
 	struct bpf_prog *clone, *tmp;
-	int insn_delta, insn_cnt;
+	int insn_delta, insn_cnt, subprog_start;
 	struct bpf_insn *insn;
-	int i, rewritten;
+	int i, j, rewritten;
 
 	if (!prog->blinding_requested || prog->blinded)
 		return prog;
@@ -1503,8 +1531,10 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
 
 	insn_cnt = clone->len;
 	insn = clone->insnsi;
+	subprog_start = env->subprog_info[prog->aux->func_idx].start;
 
-	for (i = 0; i < insn_cnt; i++, insn++) {
+	for (i = 0, j = 0; i < insn_cnt; i++, j++, insn++) {
+		env->insn_aux_data[subprog_start + j].final_idx = i;
 		if (bpf_pseudo_func(insn)) {
 			/* ld_imm64 with an address of bpf subprog is not
 			 * a user controlled constant. Don't randomize it,
@@ -1512,6 +1542,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
 			 */
 			insn++;
 			i++;
+			j++;
+			env->insn_aux_data[subprog_start + j].final_idx = i;
 			continue;
 		}
 
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1d2d42078ddf..5f08d521e58a 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3971,6 +3971,11 @@ static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
 	return env->insn_aux_data[insn_idx].jmp_point;
 }
 
+static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
+{
+	env->insn_aux_data[idx].indirect_target = true;
+}
+
 #define LR_FRAMENO_BITS	3
 #define LR_SPI_BITS	6
 #define LR_ENTRY_BITS	(LR_SPI_BITS + LR_FRAMENO_BITS + 1)
@@ -20943,12 +20948,14 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in
 	}
 
 	for (i = 0; i < n - 1; i++) {
+		mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
 		other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
 					  env->insn_idx, env->cur_state->speculative);
 		if (IS_ERR(other_branch))
 			return PTR_ERR(other_branch);
 	}
 	env->insn_idx = env->gotox_tmp_buf->items[n-1];
+	mark_indirect_target(env, env->insn_idx);
 	return 0;
 }
 
@@ -22817,6 +22824,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 		num_exentries = 0;
 		insn = func[i]->insnsi;
 		for (j = 0; j < func[i]->len; j++, insn++) {
+			env->insn_aux_data[subprog_start + j].final_idx = j;
 			if (BPF_CLASS(insn->code) == BPF_LDX &&
 			    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
 			     BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
@@ -26088,8 +26096,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 
 	/* constants blinding in the JIT may increase prog->len */
 	len = env->prog->len;
-	if (env->subprog_cnt == 1)
+	if (env->subprog_cnt == 1) {
+		for (i = 0; i < len; i++)
+			env->insn_aux_data[i].final_idx = i;
 		env->prog = bpf_prog_select_jit(env, env->prog, &ret);
+	}
 
 	adjust_btf_func(env);
 
-- 
2.47.3

[PATCH bpf-next v5 4/5] bpf, x86: Emit ENDBR for indirect jump targets

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

On CPUs that support CET/IBT, the indirect jump selftest triggers
a kernel panic because the indirect jump targets lack ENDBR
instructions.

To fix it, emit an ENDBR instruction to each indirect jump target. Since
the ENDBR instruction shifts the position of original jited instructions,
fix the instruction address calculation wherever the addresses are used.

For reference, below is a sample panic log.

 Missing ENDBR: bpf_prog_2e5f1c71c13ac3e0_big_jump_table+0x97/0xe1
 ------------[ cut here ]------------
 kernel BUG at arch/x86/kernel/cet.c:133!
 Oops: invalid opcode: 0000 [#1] SMP NOPTI

 ...

  ? 0xffffffffc00fb258
  ? bpf_prog_2e5f1c71c13ac3e0_big_jump_table+0x97/0xe1
  bpf_prog_test_run_syscall+0x110/0x2f0
  ? fdget+0xba/0xe0
  __sys_bpf+0xe4b/0x2590
  ? __kmalloc_node_track_caller_noprof+0x1c7/0x680
  ? bpf_prog_test_run_syscall+0x215/0x2f0
  __x64_sys_bpf+0x21/0x30
  do_syscall_64+0x85/0x620
  ? bpf_prog_test_run_syscall+0x1e2/0x2f0

Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps")
Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 arch/x86/net/bpf_jit_comp.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 43beacaed56d..7a2fa828558a 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1658,8 +1658,8 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
 	return 0;
 }
 
-static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
-		  int oldproglen, struct jit_context *ctx, bool jmp_padding)
+static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image,
+		  u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding)
 {
 	bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
 	struct bpf_insn *insn = bpf_prog->insnsi;
@@ -1743,6 +1743,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
 				dst_reg = X86_REG_R9;
 		}
 
+		if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1))
+			EMIT_ENDBR();
+
 		switch (insn->code) {
 			/* ALU */
 		case BPF_ALU | BPF_ADD | BPF_X:
@@ -2449,7 +2452,7 @@ st:			if (is_imm8(insn->off))
 
 			/* call */
 		case BPF_JMP | BPF_CALL: {
-			u8 *ip = image + addrs[i - 1];
+			u8 *ip = image + addrs[i - 1] + (prog - temp);
 
 			func = (u8 *) __bpf_call_base + imm32;
 			if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
@@ -2474,7 +2477,8 @@ st:			if (is_imm8(insn->off))
 			if (imm32)
 				emit_bpf_tail_call_direct(bpf_prog,
 							  &bpf_prog->aux->poke_tab[imm32 - 1],
-							  &prog, image + addrs[i - 1],
+							  &prog,
+							  image + addrs[i - 1] + (prog - temp),
 							  callee_regs_used,
 							  stack_depth,
 							  ctx);
@@ -2483,7 +2487,7 @@ st:			if (is_imm8(insn->off))
 							    &prog,
 							    callee_regs_used,
 							    stack_depth,
-							    image + addrs[i - 1],
+							    image + addrs[i - 1] + (prog - temp),
 							    ctx);
 			break;
 
@@ -2648,7 +2652,8 @@ st:			if (is_imm8(insn->off))
 			break;
 
 		case BPF_JMP | BPF_JA | BPF_X:
-			emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]);
+			emit_indirect_jump(&prog, insn->dst_reg,
+					   image + addrs[i - 1] + (prog - temp));
 			break;
 		case BPF_JMP | BPF_JA:
 		case BPF_JMP32 | BPF_JA:
@@ -2738,7 +2743,7 @@ st:			if (is_imm8(insn->off))
 			ctx->cleanup_addr = proglen;
 			if (bpf_prog_was_classic(bpf_prog) &&
 			    !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) {
-				u8 *ip = image + addrs[i - 1];
+				u8 *ip = image + addrs[i - 1] + (prog - temp);
 
 				if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))
 					return -EINVAL;
@@ -3820,7 +3825,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	for (pass = 0; pass < MAX_PASSES || image; pass++) {
 		if (!padding && pass >= PADDING_PASSES)
 			padding = true;
-		proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);
+		proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen, &ctx, padding);
 		if (proglen <= 0) {
 out_image:
 			image = NULL;
-- 
2.47.3

[PATCH bpf-next v5 5/5] bpf, arm64: Emit BTI for indirect jump target

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

On CPUs that support BTI, the indirect jump selftest triggers a kernel
panic because there is no BTI instructions at the indirect jump targets.

Fix it by emitting a BTI instruction for each indirect jump target.

For reference, below is a sample panic log.

Internal error: Oops - BTI: 0000000036000003 [#1]  SMP
...
Call trace:
 bpf_prog_2e5f1c71c13ac3e0_big_jump_table+0x54/0xf8 (P)
 bpf_prog_run_pin_on_cpu+0x140/0x468
 bpf_prog_test_run_syscall+0x280/0x3b8
 bpf_prog_test_run+0x22c/0x2c0

Fixes: f4a66cf1cb14 ("bpf: arm64: Add support for indirect jumps")
Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 arch/arm64/net/bpf_jit_comp.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 823246c7ff5d..127e099d3d3a 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1198,8 +1198,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
  * >0 - successfully JITed a 16-byte eBPF instruction.
  * <0 - failed to JIT.
  */
-static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
-		      bool extra_pass)
+static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn *insn,
+		      struct jit_ctx *ctx, bool extra_pass)
 {
 	const u8 code = insn->code;
 	u8 dst = bpf2a64[insn->dst_reg];
@@ -1224,6 +1224,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 	int ret;
 	bool sign_extend;
 
+	if (bpf_insn_is_indirect_target(env, ctx->prog, i))
+		emit_bti(A64_BTI_J, ctx);
+
 	switch (code) {
 	/* dst = src */
 	case BPF_ALU | BPF_MOV | BPF_X:
@@ -1899,7 +1902,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 	return 0;
 }
 
-static int build_body(struct jit_ctx *ctx, bool extra_pass)
+static int build_body(struct bpf_verifier_env *env, struct jit_ctx *ctx, bool extra_pass)
 {
 	const struct bpf_prog *prog = ctx->prog;
 	int i;
@@ -1918,7 +1921,7 @@ static int build_body(struct jit_ctx *ctx, bool extra_pass)
 		int ret;
 
 		ctx->offset[i] = ctx->idx;
-		ret = build_insn(insn, ctx, extra_pass);
+		ret = build_insn(env, insn, ctx, extra_pass);
 		if (ret > 0) {
 			i++;
 			ctx->offset[i] = ctx->idx;
@@ -2100,7 +2103,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 		goto out_off;
 	}
 
-	if (build_body(&ctx, extra_pass)) {
+	if (build_body(env, &ctx, extra_pass)) {
 		prog = orig_prog;
 		goto out_off;
 	}
@@ -2152,7 +2155,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	/* Dont write body instructions to memory for now */
 	ctx.write = false;
 
-	if (build_body(&ctx, extra_pass)) {
+	if (build_body(env, &ctx, extra_pass)) {
 		prog = orig_prog;
 		goto out_free_hdr;
 	}
@@ -2163,7 +2166,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	ctx.write = true;
 
 	/* Pass 3: Adjust jump offset and write final image */
-	if (build_body(&ctx, extra_pass) ||
+	if (build_body(env, &ctx, extra_pass) ||
 		WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) {
 		prog = orig_prog;
 		goto out_free_hdr;
-- 
2.47.3

Re: [PATCH bpf-next v5 1/5] bpf: Move JIT for single-subprog programs to verifier

[bot+bpf-ci]

[-- Attachment #1: Type: text/plain, Size: 2674 bytes --]

> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index fc4ccd1de569..ab2bc0850770 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -26086,6 +26086,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>  		convert_pseudo_ld_imm64(env);
>  	}
>
> +	/* constants blinding in the JIT may increase prog->len */
> +	len = env->prog->len;
> +	if (env->subprog_cnt == 1)
> +		env->prog = bpf_prog_select_jit(env->prog, &ret);
> +
>  	adjust_btf_func(env);
>
>  err_release_maps:

[ ... ]

> @@ -26111,7 +26116,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>  err_unlock:
>  	if (!is_priv)
>  		mutex_unlock(&bpf_verifier_lock);
> -	clear_insn_aux_data(env, 0, env->prog->len);
> +	clear_insn_aux_data(env, 0, len);
>  	vfree(env->insn_aux_data);

Can the len variable be stale when reaching clear_insn_aux_data()
through an error path?

At the top of bpf_check(), len is first assigned as:

    len = (*prog)->len;

This original value is used to allocate insn_aux_data. Later,
instruction rewriting passes like convert_ctx_accesses() and
do_misc_fixups() can change env->prog->len and resize
insn_aux_data to match via bpf_patch_insn_data().

The second assignment added by this patch:

    len = env->prog->len;

is only reached on the success path, after all instruction
rewriting and after the maps/btfs allocation. If an error occurs
between the rewriting passes and this reassignment, for example
bpf_vlog_finalize() returning an error, copy_to_bpfptr_offset()
failing, or kmalloc for used_maps/used_btfs failing, the code
jumps to err_release_maps and eventually reaches
clear_insn_aux_data(env, 0, len) with the original program length
rather than the current post-rewriting length.

If rewriting increased env->prog->len (the common case for
programs with context accesses), the iteration stops short of
the actual insn_aux_data size, potentially leaking jt fields in
the unvisited entries.


---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/22571297111

AI-authorship-score: low
AI-authorship-explanation: Standard kernel refactoring by a known BPF contributor with non-native English patterns in the commit message.
issues-found: 1
issue-severity-score: low
issue-severity-explanation: Stale len variable in bpf_check error paths can cause insn_aux_data jt field memory leaks when instruction rewriting changed prog->len before an error between rewriting and the len reassignment.

Re: [PATCH bpf-next v5 1/5] bpf: Move JIT for single-subprog programs to verifier

[Xu Kuohai]

On 3/2/2026 6:46 PM, bot+bpf-ci@kernel.org wrote:
>> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
>> index fc4ccd1de569..ab2bc0850770 100644
>> --- a/kernel/bpf/verifier.c
>> +++ b/kernel/bpf/verifier.c
>> @@ -26086,6 +26086,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>>   		convert_pseudo_ld_imm64(env);
>>   	}
>>
>> +	/* constants blinding in the JIT may increase prog->len */
>> +	len = env->prog->len;
>> +	if (env->subprog_cnt == 1)
>> +		env->prog = bpf_prog_select_jit(env->prog, &ret);
>> +
>>   	adjust_btf_func(env);
>>
>>   err_release_maps:
> 
> [ ... ]
> 
>> @@ -26111,7 +26116,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>>   err_unlock:
>>   	if (!is_priv)
>>   		mutex_unlock(&bpf_verifier_lock);
>> -	clear_insn_aux_data(env, 0, env->prog->len);
>> +	clear_insn_aux_data(env, 0, len);
>>   	vfree(env->insn_aux_data);
> 
> Can the len variable be stale when reaching clear_insn_aux_data()
> through an error path?
> 
> At the top of bpf_check(), len is first assigned as:
> 
>      len = (*prog)->len;
> 
> This original value is used to allocate insn_aux_data. Later,
> instruction rewriting passes like convert_ctx_accesses() and
> do_misc_fixups() can change env->prog->len and resize
> insn_aux_data to match via bpf_patch_insn_data().
> 
> The second assignment added by this patch:
> 
>      len = env->prog->len;
> 
> is only reached on the success path, after all instruction
> rewriting and after the maps/btfs allocation. If an error occurs
> between the rewriting passes and this reassignment, for example
> bpf_vlog_finalize() returning an error, copy_to_bpfptr_offset()
> failing, or kmalloc for used_maps/used_btfs failing, the code
> jumps to err_release_maps and eventually reaches
> clear_insn_aux_data(env, 0, len) with the original program length
> rather than the current post-rewriting length.
> 
> If rewriting increased env->prog->len (the common case for
> programs with context accesses), the iteration stops short of
> the actual insn_aux_data size, potentially leaking jt fields in
> the unvisited entries.
> 

Right, len is stale in the error path. I'll fix it in the next version.

> 
> ---
> AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
> See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
> 
> CI run summary: https://github.com/kernel-patches/bpf/actions/runs/22571297111
> 
> AI-authorship-score: low
> AI-authorship-explanation: Standard kernel refactoring by a known BPF contributor with non-native English patterns in the commit message.
> issues-found: 1
> issue-severity-score: low
> issue-severity-explanation: Stale len variable in bpf_check error paths can cause insn_aux_data jt field memory leaks when instruction rewriting changed prog->len before an error between rewriting and the len reassignment.

Re: [PATCH bpf-next v5 0/5] emit ENDBR/BTI instructions for indirect jump targets

[Alexis Lothoré]

Hi Xu,

On Mon Mar 2, 2026 at 11:27 AM CET, Xu Kuohai wrote:
> On x86 CPUs with CET/IBT and arm64 CPUs with BTI, missing landing pad instructions
> at indirect jump targets triggers kernel panic. So emit ENDBR instructions for
> indirect jump targets on x86 and BTI on arm64. Indirect jump targets are identified
> based on the insn_aux_data created by the verifier.
>
> v5:
> - Switch to pass env to JIT directly to get rid of coping private insn_aux_data for
>   each prog

Nice, thanks for this. As discussed in [1], I'm planning to reuse this
in my KASAN work to pass info to JIT comp about ldx/stx instructions
(especially, whether those insn are accessing prog stack or elsewhere).
So far I've been using a small POC aiming to do the same kind of thing,
but your series made me aware of a few points I missed, like the offset
induced by constant blinding. I'll keep my work rebased on top of your
series, and track it until it is merged.

Thanks

Alexis

[1] https://lore.kernel.org/bpf/CAADnVQLX7RSnOqQuU32Cgq-e0MVqyeNrtCQSBbk0W2xGkE-ZNw@mail.gmail.com/

> v4: https://lore.kernel.org/all/20260114093914.2403982-1-xukuohai@huaweicloud.com/
> - Switch to the approach proposed by Eduard, using insn_aux_data to indentify indirect
>   jump targets, and emit ENDBR on x86
>
> v3: https://lore.kernel.org/bpf/20251227081033.240336-1-xukuohai@huaweicloud.com/
> - Get rid of unnecessary enum definition (Yonghong Song, Anton Protopopov)
>
> v2: https://lore.kernel.org/bpf/20251223085447.139301-1-xukuohai@huaweicloud.com/
> - Exclude instruction arrays not used for indirect jumps (Anton Protopopov)
>
> v1: https://lore.kernel.org/bpf/20251127140318.3944249-1-xukuohai@huaweicloud.com/
>
> Xu Kuohai (5):
>   bpf: Move JIT for single-subprog programs to verifier
>   bpf: Pass bpf_verifier_env to jit
>   bpf: Add helper to detect indirect jump targets
>   bpf, x86: Emit ENDBR for indirect jump targets
>   bpf, arm64: Emit BTI for indirect jump target
>
>  arch/arc/net/bpf_jit_core.c      | 19 +++----
>  arch/arm/net/bpf_jit_32.c        |  4 +-
>  arch/arm64/net/bpf_jit_comp.c    | 21 ++++----
>  arch/loongarch/net/bpf_jit.c     |  4 +-
>  arch/mips/net/bpf_jit_comp.c     |  4 +-
>  arch/parisc/net/bpf_jit_core.c   |  4 +-
>  arch/powerpc/net/bpf_jit_comp.c  |  4 +-
>  arch/riscv/net/bpf_jit_core.c    |  4 +-
>  arch/s390/net/bpf_jit_comp.c     |  4 +-
>  arch/sparc/net/bpf_jit_comp_64.c |  4 +-
>  arch/x86/net/bpf_jit_comp.c      | 25 +++++----
>  arch/x86/net/bpf_jit_comp32.c    |  4 +-
>  include/linux/bpf.h              |  2 +
>  include/linux/bpf_verifier.h     | 10 ++--
>  include/linux/filter.h           |  6 ++-
>  kernel/bpf/core.c                | 93 ++++++++++++++++++++++++--------
>  kernel/bpf/syscall.c             |  2 +-
>  kernel/bpf/verifier.c            | 22 ++++++--
>  18 files changed, 157 insertions(+), 79 deletions(-)




-- 
Alexis Lothoré, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Re: [PATCH bpf-next v5 3/5] bpf: Add helper to detect indirect jump targets

[Alexei Starovoitov]

On Mon, Mar 2, 2026 at 2:02 AM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
>
> From: Xu Kuohai <xukuohai@huawei.com>
>
> Introduce helper bpf_insn_is_indirect_target to determine whether a BPF
> instruction is an indirect jump target. This helper will be used by
> follow-up patches to decide where to emit indirect landing pad instructions.
>
> Add a new flag to struct bpf_insn_aux_data to mark instructions that are
> indirect jump targets. The BPF verifier sets this flag, and the helper
> checks it to determine whether an instruction is an indirect jump target.
>
> Also add a new field to struct bpf_insn_aux_data to track the instruction
> final index in the bpf prog, as the instructions may be rewritten by
> constant blinding in the JIT stage. This field is used as a binary search
> key to find the corresponding insn_aux_data for a given instruction.
>
> Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
> ---
>  include/linux/bpf.h          |  2 ++
>  include/linux/bpf_verifier.h | 10 ++++++----
>  kernel/bpf/core.c            | 38 +++++++++++++++++++++++++++++++++---
>  kernel/bpf/verifier.c        | 13 +++++++++++-
>  4 files changed, 55 insertions(+), 8 deletions(-)
>
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index 05b34a6355b0..90760e250865 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -1541,6 +1541,8 @@ bool bpf_has_frame_pointer(unsigned long ip);
>  int bpf_jit_charge_modmem(u32 size);
>  void bpf_jit_uncharge_modmem(u32 size);
>  bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
> +bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
> +                                int insn_idx);
>  #else
>  static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
>                                            struct bpf_trampoline *tr,
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> index c1e30096ea7b..f8f70e5414f0 100644
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -577,16 +577,18 @@ struct bpf_insn_aux_data {
>
>         /* below fields are initialized once */
>         unsigned int orig_idx; /* original instruction index */
> -       bool jmp_point;
> -       bool prune_point;
> +       unsigned int final_idx; /* final instruction index */
> +       u32 jmp_point:1;
> +       u32 prune_point:1;
>         /* ensure we check state equivalence and save state checkpoint and
>          * this instruction, regardless of any heuristics
>          */
> -       bool force_checkpoint;
> +       u32 force_checkpoint:1;
>         /* true if instruction is a call to a helper function that
>          * accepts callback function as a parameter.
>          */
> -       bool calls_callback;
> +       u32 calls_callback:1;
> +       u32 indirect_target:1; /* if it is an indirect jump target */
>         /*
>          * CFG strongly connected component this instruction belongs to,
>          * zero if it is a singleton SCC.
> diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
> index 7702c232c62e..9a760cf43d68 100644
> --- a/kernel/bpf/core.c
> +++ b/kernel/bpf/core.c
> @@ -1486,13 +1486,41 @@ static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
>  #endif
>  }
>
> +static int bpf_insn_aux_cmp_by_insn_idx(const void *a, const void *b)
> +{
> +       int insn_idx = *(int *)a;
> +       int final_idx = ((const struct bpf_insn_aux_data *)b)->final_idx;
> +
> +       return insn_idx - final_idx;
> +}
> +
> +bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
> +                                int insn_idx)
> +{
> +       struct bpf_insn_aux_data *insn_aux;
> +       int func_idx, subprog_start, subprog_end;
> +
> +       if (!env)
> +               return false;
> +
> +       func_idx = prog->aux->func_idx;
> +       subprog_start = env->subprog_info[func_idx].start;
> +       subprog_end = env->subprog_info[func_idx + 1].start;
> +
> +       insn_aux = bsearch(&insn_idx, &env->insn_aux_data[subprog_start],
> +                          subprog_end - subprog_start,
> +                          sizeof(struct bpf_insn_aux_data), bpf_insn_aux_cmp_by_insn_idx);
> +
> +       return insn_aux && insn_aux->indirect_target;
> +}
> +
>  struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
>  {
>         struct bpf_insn insn_buff[16], aux[2];
>         struct bpf_prog *clone, *tmp;
> -       int insn_delta, insn_cnt;
> +       int insn_delta, insn_cnt, subprog_start;
>         struct bpf_insn *insn;
> -       int i, rewritten;
> +       int i, j, rewritten;
>
>         if (!prog->blinding_requested || prog->blinded)
>                 return prog;
> @@ -1503,8 +1531,10 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
>
>         insn_cnt = clone->len;
>         insn = clone->insnsi;
> +       subprog_start = env->subprog_info[prog->aux->func_idx].start;
>
> -       for (i = 0; i < insn_cnt; i++, insn++) {
> +       for (i = 0, j = 0; i < insn_cnt; i++, j++, insn++) {
> +               env->insn_aux_data[subprog_start + j].final_idx = i;
>                 if (bpf_pseudo_func(insn)) {
>                         /* ld_imm64 with an address of bpf subprog is not
>                          * a user controlled constant. Don't randomize it,
> @@ -1512,6 +1542,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
>                          */
>                         insn++;
>                         i++;
> +                       j++;
> +                       env->insn_aux_data[subprog_start + j].final_idx = i;

You're adding final_idx because bpf_jit_blind_constants()
doesn't call adjust_insn_aux_data() ?

imo that's an ugly workaround. Just call adjust_insn_aux_data().

And in the future please mention such design decisions in the commit log,
so that reviewers don't need to reverse engineer your thought process.

pw-bot: cr

Re: [PATCH bpf-next v5 1/5] bpf: Move JIT for single-subprog programs to verifier

[Eduard Zingerman]

On Mon, 2026-03-02 at 18:27 +0800, Xu Kuohai wrote:

[...]

> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index fc4ccd1de569..ab2bc0850770 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -26086,6 +26086,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>  		convert_pseudo_ld_imm64(env);
>  	}
>  
> +	/* constants blinding in the JIT may increase prog->len */
> +	len = env->prog->len;
> +	if (env->subprog_cnt == 1)
> +		env->prog = bpf_prog_select_jit(env->prog, &ret);
> +

I probably miss something important, but would it be possible to put a
call to __bpf_prog_select_runtime() here and remove it from syscall.c:bpf_prog_load()?
Thus avoiding the need for `jit_attempted` parameter.

[...]

Re: [PATCH bpf-next v5 4/5] bpf, x86: Emit ENDBR for indirect jump targets

[Eduard Zingerman]

On Mon, 2026-03-02 at 18:27 +0800, Xu Kuohai wrote:

[...]

> @@ -2449,7 +2452,7 @@ st:			if (is_imm8(insn->off))
>  
>  			/* call */
>  		case BPF_JMP | BPF_CALL: {
> -			u8 *ip = image + addrs[i - 1];
> +			u8 *ip = image + addrs[i - 1] + (prog - temp);
                                                        ^^^^^^^^^^^^^
			Could you please comment a bit why this addend is needed?
>  
>  			func = (u8 *) __bpf_call_base + imm32;
>  			if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {

[...]

Re: [PATCH bpf-next v5 0/5] emit ENDBR/BTI instructions for indirect jump targets

[Xu Kuohai]

On 3/4/2026 12:29 AM, Alexis Lothoré wrote:
> Hi Xu,
> 
> On Mon Mar 2, 2026 at 11:27 AM CET, Xu Kuohai wrote:
>> On x86 CPUs with CET/IBT and arm64 CPUs with BTI, missing landing pad instructions
>> at indirect jump targets triggers kernel panic. So emit ENDBR instructions for
>> indirect jump targets on x86 and BTI on arm64. Indirect jump targets are identified
>> based on the insn_aux_data created by the verifier.
>>
>> v5:
>> - Switch to pass env to JIT directly to get rid of coping private insn_aux_data for
>>    each prog
> 
> Nice, thanks for this. As discussed in [1], I'm planning to reuse this
> in my KASAN work to pass info to JIT comp about ldx/stx instructions
> (especially, whether those insn are accessing prog stack or elsewhere).
> So far I've been using a small POC aiming to do the same kind of thing,
> but your series made me aware of a few points I missed, like the offset
> induced by constant blinding. I'll keep my work rebased on top of your
> series, and track it until it is merged.
>

Glad to hear that. Thanks for letting me know.

> Thanks
> 
> Alexis
> 
> [1] https://lore.kernel.org/bpf/CAADnVQLX7RSnOqQuU32Cgq-e0MVqyeNrtCQSBbk0W2xGkE-ZNw@mail.gmail.com/
> 
>> v4: https://lore.kernel.org/all/20260114093914.2403982-1-xukuohai@huaweicloud.com/
>> - Switch to the approach proposed by Eduard, using insn_aux_data to indentify indirect
>>    jump targets, and emit ENDBR on x86
>>
>> v3: https://lore.kernel.org/bpf/20251227081033.240336-1-xukuohai@huaweicloud.com/
>> - Get rid of unnecessary enum definition (Yonghong Song, Anton Protopopov)
>>
>> v2: https://lore.kernel.org/bpf/20251223085447.139301-1-xukuohai@huaweicloud.com/
>> - Exclude instruction arrays not used for indirect jumps (Anton Protopopov)
>>
>> v1: https://lore.kernel.org/bpf/20251127140318.3944249-1-xukuohai@huaweicloud.com/
>>
>> Xu Kuohai (5):
>>    bpf: Move JIT for single-subprog programs to verifier
>>    bpf: Pass bpf_verifier_env to jit
>>    bpf: Add helper to detect indirect jump targets
>>    bpf, x86: Emit ENDBR for indirect jump targets
>>    bpf, arm64: Emit BTI for indirect jump target
>>
>>   arch/arc/net/bpf_jit_core.c      | 19 +++----
>>   arch/arm/net/bpf_jit_32.c        |  4 +-
>>   arch/arm64/net/bpf_jit_comp.c    | 21 ++++----
>>   arch/loongarch/net/bpf_jit.c     |  4 +-
>>   arch/mips/net/bpf_jit_comp.c     |  4 +-
>>   arch/parisc/net/bpf_jit_core.c   |  4 +-
>>   arch/powerpc/net/bpf_jit_comp.c  |  4 +-
>>   arch/riscv/net/bpf_jit_core.c    |  4 +-
>>   arch/s390/net/bpf_jit_comp.c     |  4 +-
>>   arch/sparc/net/bpf_jit_comp_64.c |  4 +-
>>   arch/x86/net/bpf_jit_comp.c      | 25 +++++----
>>   arch/x86/net/bpf_jit_comp32.c    |  4 +-
>>   include/linux/bpf.h              |  2 +
>>   include/linux/bpf_verifier.h     | 10 ++--
>>   include/linux/filter.h           |  6 ++-
>>   kernel/bpf/core.c                | 93 ++++++++++++++++++++++++--------
>>   kernel/bpf/syscall.c             |  2 +-
>>   kernel/bpf/verifier.c            | 22 ++++++--
>>   18 files changed, 157 insertions(+), 79 deletions(-)
> 
> 
> 
> 

Re: [PATCH bpf-next v5 1/5] bpf: Move JIT for single-subprog programs to verifier

[Xu Kuohai]

On 3/4/2026 2:05 PM, Eduard Zingerman wrote:
> On Mon, 2026-03-02 at 18:27 +0800, Xu Kuohai wrote:
> 
> [...]
> 
>> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
>> index fc4ccd1de569..ab2bc0850770 100644
>> --- a/kernel/bpf/verifier.c
>> +++ b/kernel/bpf/verifier.c
>> @@ -26086,6 +26086,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>>   		convert_pseudo_ld_imm64(env);
>>   	}
>>   
>> +	/* constants blinding in the JIT may increase prog->len */
>> +	len = env->prog->len;
>> +	if (env->subprog_cnt == 1)
>> +		env->prog = bpf_prog_select_jit(env->prog, &ret);
>> +
> 
> I probably miss something important, but would it be possible to put a
> call to __bpf_prog_select_runtime() here and remove it from syscall.c:bpf_prog_load()?
> Thus avoiding the need for `jit_attempted` parameter.
>

Makes sense. In fact, at first I planned to move the whole thing to verifier,
but since this series is just fixing a JIT issue, I ultimately decided to only
move the JIT code to avoid unnecessary changes.

> [...]

Re: [PATCH bpf-next v5 3/5] bpf: Add helper to detect indirect jump targets

[Xu Kuohai]

On 3/4/2026 1:19 AM, Alexei Starovoitov wrote:

[...]

>> -       for (i = 0; i < insn_cnt; i++, insn++) {
>> +       for (i = 0, j = 0; i < insn_cnt; i++, j++, insn++) {
>> +               env->insn_aux_data[subprog_start + j].final_idx = i;
>>                  if (bpf_pseudo_func(insn)) {
>>                          /* ld_imm64 with an address of bpf subprog is not
>>                           * a user controlled constant. Don't randomize it,
>> @@ -1512,6 +1542,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
>>                           */
>>                          insn++;
>>                          i++;
>> +                       j++;
>> +                       env->insn_aux_data[subprog_start + j].final_idx = i;
> 
> You're adding final_idx because bpf_jit_blind_constants()
> doesn't call adjust_insn_aux_data() ?
>

Yes, I added final_idx because insn_aux is not updated here.

> imo that's an ugly workaround. Just call adjust_insn_aux_data().
>

If we adjust the env->insn_aux_data here, should we also adjust the global
env->prog->insnsi array? I think env->insn_aux_data should remain consistent
with the global env->prog->insnsi array. Since constant blinding only rewrites
the subprog's private instruction array, updating the env->insn_aux_data
causes a mismatch with the global state.

> And in the future please mention such design decisions in the commit log,
> so that reviewers don't need to reverse engineer your thought process.
>

Sorry for the lack of clarity. I’ll make an effort to clarify things more
clearly in the future.

> pw-bot: cr
> 
> 

Re: [PATCH bpf-next v5 4/5] bpf, x86: Emit ENDBR for indirect jump targets

[Xu Kuohai]

On 3/4/2026 2:23 PM, Eduard Zingerman wrote:
> On Mon, 2026-03-02 at 18:27 +0800, Xu Kuohai wrote:
> 
> [...]
> 
>> @@ -2449,7 +2452,7 @@ st:			if (is_imm8(insn->off))
>>   
>>   			/* call */
>>   		case BPF_JMP | BPF_CALL: {
>> -			u8 *ip = image + addrs[i - 1];
>> +			u8 *ip = image + addrs[i - 1] + (prog - temp);
>                                                          ^^^^^^^^^^^^^
> 			Could you please comment a bit why this addend is needed?
>>

The temp buffer holds the JITed x86 instruction for each BPF instruction, and
prog variable points to the position in temp buffer where the next JITed x86
instruction will be written.

Since there may already be an ENDBR instruction in the temp buffer, which is
written when the current BPF instruction is an indirect jump target, we need to
add addend when computing the actual address of the JITed x86 instruction.

>>   			func = (u8 *) __bpf_call_base + imm32;
>>   			if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
> 
> [...]
> 
> 

Re: [PATCH bpf-next v5 3/5] bpf: Add helper to detect indirect jump targets

[Alexei Starovoitov]

On Wed, Mar 4, 2026 at 4:46 AM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
>
> On 3/4/2026 1:19 AM, Alexei Starovoitov wrote:
>
> [...]
>
> >> -       for (i = 0; i < insn_cnt; i++, insn++) {
> >> +       for (i = 0, j = 0; i < insn_cnt; i++, j++, insn++) {
> >> +               env->insn_aux_data[subprog_start + j].final_idx = i;
> >>                  if (bpf_pseudo_func(insn)) {
> >>                          /* ld_imm64 with an address of bpf subprog is not
> >>                           * a user controlled constant. Don't randomize it,
> >> @@ -1512,6 +1542,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
> >>                           */
> >>                          insn++;
> >>                          i++;
> >> +                       j++;
> >> +                       env->insn_aux_data[subprog_start + j].final_idx = i;
> >
> > You're adding final_idx because bpf_jit_blind_constants()
> > doesn't call adjust_insn_aux_data() ?
> >
>
> Yes, I added final_idx because insn_aux is not updated here.
>
> > imo that's an ugly workaround. Just call adjust_insn_aux_data().
> >
>
> If we adjust the env->insn_aux_data here, should we also adjust the global
> env->prog->insnsi array? I think env->insn_aux_data should remain consistent
> with the global env->prog->insnsi array. Since constant blinding only rewrites
> the subprog's private instruction array, updating the env->insn_aux_data
> causes a mismatch with the global state.

yes, and subprog starts, and pokes that bpf_patch_insn_data() do.

blinding was implemented long before that, so it was never updated.

Re: [PATCH bpf-next v5 3/5] bpf: Add helper to detect indirect jump targets

[Xu Kuohai]

On 3/4/2026 11:37 PM, Alexei Starovoitov wrote:
> On Wed, Mar 4, 2026 at 4:46 AM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
>>
>> On 3/4/2026 1:19 AM, Alexei Starovoitov wrote:
>>
>> [...]
>>
>>>> -       for (i = 0; i < insn_cnt; i++, insn++) {
>>>> +       for (i = 0, j = 0; i < insn_cnt; i++, j++, insn++) {
>>>> +               env->insn_aux_data[subprog_start + j].final_idx = i;
>>>>                   if (bpf_pseudo_func(insn)) {
>>>>                           /* ld_imm64 with an address of bpf subprog is not
>>>>                            * a user controlled constant. Don't randomize it,
>>>> @@ -1512,6 +1542,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
>>>>                            */
>>>>                           insn++;
>>>>                           i++;
>>>> +                       j++;
>>>> +                       env->insn_aux_data[subprog_start + j].final_idx = i;
>>>
>>> You're adding final_idx because bpf_jit_blind_constants()
>>> doesn't call adjust_insn_aux_data() ?
>>>
>>
>> Yes, I added final_idx because insn_aux is not updated here.
>>
>>> imo that's an ugly workaround. Just call adjust_insn_aux_data().
>>>
>>
>> If we adjust the env->insn_aux_data here, should we also adjust the global
>> env->prog->insnsi array? I think env->insn_aux_data should remain consistent
>> with the global env->prog->insnsi array. Since constant blinding only rewrites
>> the subprog's private instruction array, updating the env->insn_aux_data
>> causes a mismatch with the global state.
> 
> yes, and subprog starts, and pokes that bpf_patch_insn_data() do.
> 
> blinding was implemented long before that, so it was never updated.

I see. Since env->prog->insnsi is rewritten by blind_constants now, would it
make sense to move constant blinding to the beginning of jit_subprogs, just
before the global instruction array is split into subprog copies?

This would eliminate the need to invoke constant blinding per subprog from
the arch-specific JIT, simplifying the overall flow.

Re: [PATCH bpf-next v5 3/5] bpf: Add helper to detect indirect jump targets

[Alexei Starovoitov]

On Wed, Mar 4, 2026 at 7:47 PM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
>
> On 3/4/2026 11:37 PM, Alexei Starovoitov wrote:
> > On Wed, Mar 4, 2026 at 4:46 AM Xu Kuohai <xukuohai@huaweicloud.com> wrote:
> >>
> >> On 3/4/2026 1:19 AM, Alexei Starovoitov wrote:
> >>
> >> [...]
> >>
> >>>> -       for (i = 0; i < insn_cnt; i++, insn++) {
> >>>> +       for (i = 0, j = 0; i < insn_cnt; i++, j++, insn++) {
> >>>> +               env->insn_aux_data[subprog_start + j].final_idx = i;
> >>>>                   if (bpf_pseudo_func(insn)) {
> >>>>                           /* ld_imm64 with an address of bpf subprog is not
> >>>>                            * a user controlled constant. Don't randomize it,
> >>>> @@ -1512,6 +1542,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
> >>>>                            */
> >>>>                           insn++;
> >>>>                           i++;
> >>>> +                       j++;
> >>>> +                       env->insn_aux_data[subprog_start + j].final_idx = i;
> >>>
> >>> You're adding final_idx because bpf_jit_blind_constants()
> >>> doesn't call adjust_insn_aux_data() ?
> >>>
> >>
> >> Yes, I added final_idx because insn_aux is not updated here.
> >>
> >>> imo that's an ugly workaround. Just call adjust_insn_aux_data().
> >>>
> >>
> >> If we adjust the env->insn_aux_data here, should we also adjust the global
> >> env->prog->insnsi array? I think env->insn_aux_data should remain consistent
> >> with the global env->prog->insnsi array. Since constant blinding only rewrites
> >> the subprog's private instruction array, updating the env->insn_aux_data
> >> causes a mismatch with the global state.
> >
> > yes, and subprog starts, and pokes that bpf_patch_insn_data() do.
> >
> > blinding was implemented long before that, so it was never updated.
>
> I see. Since env->prog->insnsi is rewritten by blind_constants now, would it
> make sense to move constant blinding to the beginning of jit_subprogs, just
> before the global instruction array is split into subprog copies?
>
> This would eliminate the need to invoke constant blinding per subprog from
> the arch-specific JIT, simplifying the overall flow.

Makes sense to me.
Since we're touching all JITS, lets remove bpf_jit_blind_constants()
call from the too and do it from generic code.

Re: [PATCH bpf-next v5 4/5] bpf, x86: Emit ENDBR for indirect jump targets

[kernel test robot]

Hi Xu,

kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Xu-Kuohai/bpf-Move-JIT-for-single-subprog-programs-to-verifier/20260302-181031
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20260302102726.1126019-5-xukuohai%40huaweicloud.com
patch subject: [PATCH bpf-next v5 4/5] bpf, x86: Emit ENDBR for indirect jump targets
config: x86_64-buildonly-randconfig-001-20260305 (https://download.01.org/0day-ci/archive/20260305/202603051414.AAMjmOHv-lkp@intel.com/config)
compiler: gcc-14 (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260305/202603051414.AAMjmOHv-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202603051414.AAMjmOHv-lkp@intel.com/

All warnings (new ones prefixed by >>):

   arch/x86/net/bpf_jit_comp.c: In function 'do_jit':
>> arch/x86/net/bpf_jit_comp.c:1747:37: warning: suggest braces around empty body in an 'if' statement [-Wempty-body]
    1747 |                         EMIT_ENDBR();
         |                                     ^


vim +/if +1747 arch/x86/net/bpf_jit_comp.c

  1660	
  1661	static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image,
  1662			  u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding)
  1663	{
  1664		bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
  1665		struct bpf_insn *insn = bpf_prog->insnsi;
  1666		bool callee_regs_used[4] = {};
  1667		int insn_cnt = bpf_prog->len;
  1668		bool seen_exit = false;
  1669		u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
  1670		void __percpu *priv_frame_ptr = NULL;
  1671		u64 arena_vm_start, user_vm_start;
  1672		void __percpu *priv_stack_ptr;
  1673		int i, excnt = 0;
  1674		int ilen, proglen = 0;
  1675		u8 *prog = temp;
  1676		u32 stack_depth;
  1677		int err;
  1678	
  1679		stack_depth = bpf_prog->aux->stack_depth;
  1680		priv_stack_ptr = bpf_prog->aux->priv_stack_ptr;
  1681		if (priv_stack_ptr) {
  1682			priv_frame_ptr = priv_stack_ptr + PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
  1683			stack_depth = 0;
  1684		}
  1685	
  1686		arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
  1687		user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
  1688	
  1689		detect_reg_usage(insn, insn_cnt, callee_regs_used);
  1690	
  1691		emit_prologue(&prog, image, stack_depth,
  1692			      bpf_prog_was_classic(bpf_prog), tail_call_reachable,
  1693			      bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
  1694	
  1695		bpf_prog->aux->ksym.fp_start = prog - temp;
  1696	
  1697		/* Exception callback will clobber callee regs for its own use, and
  1698		 * restore the original callee regs from main prog's stack frame.
  1699		 */
  1700		if (bpf_prog->aux->exception_boundary) {
  1701			/* We also need to save r12, which is not mapped to any BPF
  1702			 * register, as we throw after entry into the kernel, which may
  1703			 * overwrite r12.
  1704			 */
  1705			push_r12(&prog);
  1706			push_callee_regs(&prog, all_callee_regs_used);
  1707		} else {
  1708			if (arena_vm_start)
  1709				push_r12(&prog);
  1710			push_callee_regs(&prog, callee_regs_used);
  1711		}
  1712		if (arena_vm_start)
  1713			emit_mov_imm64(&prog, X86_REG_R12,
  1714				       arena_vm_start >> 32, (u32) arena_vm_start);
  1715	
  1716		if (priv_frame_ptr)
  1717			emit_priv_frame_ptr(&prog, priv_frame_ptr);
  1718	
  1719		ilen = prog - temp;
  1720		if (rw_image)
  1721			memcpy(rw_image + proglen, temp, ilen);
  1722		proglen += ilen;
  1723		addrs[0] = proglen;
  1724		prog = temp;
  1725	
  1726		for (i = 1; i <= insn_cnt; i++, insn++) {
  1727			const s32 imm32 = insn->imm;
  1728			u32 dst_reg = insn->dst_reg;
  1729			u32 src_reg = insn->src_reg;
  1730			u8 b2 = 0, b3 = 0;
  1731			u8 *start_of_ldx;
  1732			s64 jmp_offset;
  1733			s16 insn_off;
  1734			u8 jmp_cond;
  1735			u8 *func;
  1736			int nops;
  1737	
  1738			if (priv_frame_ptr) {
  1739				if (src_reg == BPF_REG_FP)
  1740					src_reg = X86_REG_R9;
  1741	
  1742				if (dst_reg == BPF_REG_FP)
  1743					dst_reg = X86_REG_R9;
  1744			}
  1745	
  1746			if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1))
> 1747				EMIT_ENDBR();
  1748	
  1749			switch (insn->code) {
  1750				/* ALU */
  1751			case BPF_ALU | BPF_ADD | BPF_X:
  1752			case BPF_ALU | BPF_SUB | BPF_X:
  1753			case BPF_ALU | BPF_AND | BPF_X:
  1754			case BPF_ALU | BPF_OR | BPF_X:
  1755			case BPF_ALU | BPF_XOR | BPF_X:
  1756			case BPF_ALU64 | BPF_ADD | BPF_X:
  1757			case BPF_ALU64 | BPF_SUB | BPF_X:
  1758			case BPF_ALU64 | BPF_AND | BPF_X:
  1759			case BPF_ALU64 | BPF_OR | BPF_X:
  1760			case BPF_ALU64 | BPF_XOR | BPF_X:
  1761				maybe_emit_mod(&prog, dst_reg, src_reg,
  1762					       BPF_CLASS(insn->code) == BPF_ALU64);
  1763				b2 = simple_alu_opcodes[BPF_OP(insn->code)];
  1764				EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
  1765				break;
  1766	
  1767			case BPF_ALU64 | BPF_MOV | BPF_X:
  1768				if (insn_is_cast_user(insn)) {
  1769					if (dst_reg != src_reg)
  1770						/* 32-bit mov */
  1771						emit_mov_reg(&prog, false, dst_reg, src_reg);
  1772					/* shl dst_reg, 32 */
  1773					maybe_emit_1mod(&prog, dst_reg, true);
  1774					EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32);
  1775	
  1776					/* or dst_reg, user_vm_start */
  1777					maybe_emit_1mod(&prog, dst_reg, true);
  1778					if (is_axreg(dst_reg))
  1779						EMIT1_off32(0x0D,  user_vm_start >> 32);
  1780					else
  1781						EMIT2_off32(0x81, add_1reg(0xC8, dst_reg),  user_vm_start >> 32);
  1782	
  1783					/* rol dst_reg, 32 */
  1784					maybe_emit_1mod(&prog, dst_reg, true);
  1785					EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32);
  1786	
  1787					/* xor r11, r11 */
  1788					EMIT3(0x4D, 0x31, 0xDB);
  1789	
  1790					/* test dst_reg32, dst_reg32; check if lower 32-bit are zero */
  1791					maybe_emit_mod(&prog, dst_reg, dst_reg, false);
  1792					EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg));
  1793	
  1794					/* cmove r11, dst_reg; if so, set dst_reg to zero */
  1795					/* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */
  1796					maybe_emit_mod(&prog, AUX_REG, dst_reg, true);
  1797					EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg));
  1798					break;
  1799				} else if (insn_is_mov_percpu_addr(insn)) {
  1800					/* mov <dst>, <src> (if necessary) */
  1801					EMIT_mov(dst_reg, src_reg);
  1802	#ifdef CONFIG_SMP
  1803					/* add <dst>, gs:[<off>] */
  1804					EMIT2(0x65, add_1mod(0x48, dst_reg));
  1805					EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25);
  1806					EMIT((u32)(unsigned long)&this_cpu_off, 4);
  1807	#endif
  1808					break;
  1809				}
  1810				fallthrough;
  1811			case BPF_ALU | BPF_MOV | BPF_X:
  1812				if (insn->off == 0)
  1813					emit_mov_reg(&prog,
  1814						     BPF_CLASS(insn->code) == BPF_ALU64,
  1815						     dst_reg, src_reg);
  1816				else
  1817					emit_movsx_reg(&prog, insn->off,
  1818						       BPF_CLASS(insn->code) == BPF_ALU64,
  1819						       dst_reg, src_reg);
  1820				break;
  1821	
  1822				/* neg dst */
  1823			case BPF_ALU | BPF_NEG:
  1824			case BPF_ALU64 | BPF_NEG:
  1825				maybe_emit_1mod(&prog, dst_reg,
  1826						BPF_CLASS(insn->code) == BPF_ALU64);
  1827				EMIT2(0xF7, add_1reg(0xD8, dst_reg));
  1828				break;
  1829	
  1830			case BPF_ALU | BPF_ADD | BPF_K:
  1831			case BPF_ALU | BPF_SUB | BPF_K:
  1832			case BPF_ALU | BPF_AND | BPF_K:
  1833			case BPF_ALU | BPF_OR | BPF_K:
  1834			case BPF_ALU | BPF_XOR | BPF_K:
  1835			case BPF_ALU64 | BPF_ADD | BPF_K:
  1836			case BPF_ALU64 | BPF_SUB | BPF_K:
  1837			case BPF_ALU64 | BPF_AND | BPF_K:
  1838			case BPF_ALU64 | BPF_OR | BPF_K:
  1839			case BPF_ALU64 | BPF_XOR | BPF_K:
  1840				maybe_emit_1mod(&prog, dst_reg,
  1841						BPF_CLASS(insn->code) == BPF_ALU64);
  1842	
  1843				/*
  1844				 * b3 holds 'normal' opcode, b2 short form only valid
  1845				 * in case dst is eax/rax.
  1846				 */
  1847				switch (BPF_OP(insn->code)) {
  1848				case BPF_ADD:
  1849					b3 = 0xC0;
  1850					b2 = 0x05;
  1851					break;
  1852				case BPF_SUB:
  1853					b3 = 0xE8;
  1854					b2 = 0x2D;
  1855					break;
  1856				case BPF_AND:
  1857					b3 = 0xE0;
  1858					b2 = 0x25;
  1859					break;
  1860				case BPF_OR:
  1861					b3 = 0xC8;
  1862					b2 = 0x0D;
  1863					break;
  1864				case BPF_XOR:
  1865					b3 = 0xF0;
  1866					b2 = 0x35;
  1867					break;
  1868				}
  1869	
  1870				if (is_imm8(imm32))
  1871					EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
  1872				else if (is_axreg(dst_reg))
  1873					EMIT1_off32(b2, imm32);
  1874				else
  1875					EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
  1876				break;
  1877	
  1878			case BPF_ALU64 | BPF_MOV | BPF_K:
  1879			case BPF_ALU | BPF_MOV | BPF_K:
  1880				emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64,
  1881					       dst_reg, imm32);
  1882				break;
  1883	
  1884			case BPF_LD | BPF_IMM | BPF_DW:
  1885				emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);
  1886				insn++;
  1887				i++;
  1888				break;
  1889	
  1890				/* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
  1891			case BPF_ALU | BPF_MOD | BPF_X:
  1892			case BPF_ALU | BPF_DIV | BPF_X:
  1893			case BPF_ALU | BPF_MOD | BPF_K:
  1894			case BPF_ALU | BPF_DIV | BPF_K:
  1895			case BPF_ALU64 | BPF_MOD | BPF_X:
  1896			case BPF_ALU64 | BPF_DIV | BPF_X:
  1897			case BPF_ALU64 | BPF_MOD | BPF_K:
  1898			case BPF_ALU64 | BPF_DIV | BPF_K: {
  1899				bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
  1900	
  1901				if (dst_reg != BPF_REG_0)
  1902					EMIT1(0x50); /* push rax */
  1903				if (dst_reg != BPF_REG_3)
  1904					EMIT1(0x52); /* push rdx */
  1905	
  1906				if (BPF_SRC(insn->code) == BPF_X) {
  1907					if (src_reg == BPF_REG_0 ||
  1908					    src_reg == BPF_REG_3) {
  1909						/* mov r11, src_reg */
  1910						EMIT_mov(AUX_REG, src_reg);
  1911						src_reg = AUX_REG;
  1912					}
  1913				} else {
  1914					/* mov r11, imm32 */
  1915					EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
  1916					src_reg = AUX_REG;
  1917				}
  1918	
  1919				if (dst_reg != BPF_REG_0)
  1920					/* mov rax, dst_reg */
  1921					emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg);
  1922	
  1923				if (insn->off == 0) {
  1924					/*
  1925					 * xor edx, edx
  1926					 * equivalent to 'xor rdx, rdx', but one byte less
  1927					 */
  1928					EMIT2(0x31, 0xd2);
  1929	
  1930					/* div src_reg */
  1931					maybe_emit_1mod(&prog, src_reg, is64);
  1932					EMIT2(0xF7, add_1reg(0xF0, src_reg));
  1933				} else {
  1934					if (BPF_CLASS(insn->code) == BPF_ALU)
  1935						EMIT1(0x99); /* cdq */
  1936					else
  1937						EMIT2(0x48, 0x99); /* cqo */
  1938	
  1939					/* idiv src_reg */
  1940					maybe_emit_1mod(&prog, src_reg, is64);
  1941					EMIT2(0xF7, add_1reg(0xF8, src_reg));
  1942				}
  1943	
  1944				if (BPF_OP(insn->code) == BPF_MOD &&
  1945				    dst_reg != BPF_REG_3)
  1946					/* mov dst_reg, rdx */
  1947					emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3);
  1948				else if (BPF_OP(insn->code) == BPF_DIV &&
  1949					 dst_reg != BPF_REG_0)
  1950					/* mov dst_reg, rax */
  1951					emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0);
  1952	
  1953				if (dst_reg != BPF_REG_3)
  1954					EMIT1(0x5A); /* pop rdx */
  1955				if (dst_reg != BPF_REG_0)
  1956					EMIT1(0x58); /* pop rax */
  1957				break;
  1958			}
  1959	
  1960			case BPF_ALU | BPF_MUL | BPF_K:
  1961			case BPF_ALU64 | BPF_MUL | BPF_K:
  1962				maybe_emit_mod(&prog, dst_reg, dst_reg,
  1963					       BPF_CLASS(insn->code) == BPF_ALU64);
  1964	
  1965				if (is_imm8(imm32))
  1966					/* imul dst_reg, dst_reg, imm8 */
  1967					EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg),
  1968					      imm32);
  1969				else
  1970					/* imul dst_reg, dst_reg, imm32 */
  1971					EMIT2_off32(0x69,
  1972						    add_2reg(0xC0, dst_reg, dst_reg),
  1973						    imm32);
  1974				break;
  1975	
  1976			case BPF_ALU | BPF_MUL | BPF_X:
  1977			case BPF_ALU64 | BPF_MUL | BPF_X:
  1978				maybe_emit_mod(&prog, src_reg, dst_reg,
  1979					       BPF_CLASS(insn->code) == BPF_ALU64);
  1980	
  1981				/* imul dst_reg, src_reg */
  1982				EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));
  1983				break;
  1984	
  1985				/* Shifts */
  1986			case BPF_ALU | BPF_LSH | BPF_K:
  1987			case BPF_ALU | BPF_RSH | BPF_K:
  1988			case BPF_ALU | BPF_ARSH | BPF_K:
  1989			case BPF_ALU64 | BPF_LSH | BPF_K:
  1990			case BPF_ALU64 | BPF_RSH | BPF_K:
  1991			case BPF_ALU64 | BPF_ARSH | BPF_K:
  1992				maybe_emit_1mod(&prog, dst_reg,
  1993						BPF_CLASS(insn->code) == BPF_ALU64);
  1994	
  1995				b3 = simple_alu_opcodes[BPF_OP(insn->code)];
  1996				if (imm32 == 1)
  1997					EMIT2(0xD1, add_1reg(b3, dst_reg));
  1998				else
  1999					EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
  2000				break;
  2001	
  2002			case BPF_ALU | BPF_LSH | BPF_X:
  2003			case BPF_ALU | BPF_RSH | BPF_X:
  2004			case BPF_ALU | BPF_ARSH | BPF_X:
  2005			case BPF_ALU64 | BPF_LSH | BPF_X:
  2006			case BPF_ALU64 | BPF_RSH | BPF_X:
  2007			case BPF_ALU64 | BPF_ARSH | BPF_X:
  2008				/* BMI2 shifts aren't better when shift count is already in rcx */
  2009				if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) {
  2010					/* shrx/sarx/shlx dst_reg, dst_reg, src_reg */
  2011					bool w = (BPF_CLASS(insn->code) == BPF_ALU64);
  2012					u8 op;
  2013	
  2014					switch (BPF_OP(insn->code)) {
  2015					case BPF_LSH:
  2016						op = 1; /* prefix 0x66 */
  2017						break;
  2018					case BPF_RSH:
  2019						op = 3; /* prefix 0xf2 */
  2020						break;
  2021					case BPF_ARSH:
  2022						op = 2; /* prefix 0xf3 */
  2023						break;
  2024					}
  2025	
  2026					emit_shiftx(&prog, dst_reg, src_reg, w, op);
  2027	
  2028					break;
  2029				}
  2030	
  2031				if (src_reg != BPF_REG_4) { /* common case */
  2032					/* Check for bad case when dst_reg == rcx */
  2033					if (dst_reg == BPF_REG_4) {
  2034						/* mov r11, dst_reg */
  2035						EMIT_mov(AUX_REG, dst_reg);
  2036						dst_reg = AUX_REG;
  2037					} else {
  2038						EMIT1(0x51); /* push rcx */
  2039					}
  2040					/* mov rcx, src_reg */
  2041					EMIT_mov(BPF_REG_4, src_reg);
  2042				}
  2043	
  2044				/* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
  2045				maybe_emit_1mod(&prog, dst_reg,
  2046						BPF_CLASS(insn->code) == BPF_ALU64);
  2047	
  2048				b3 = simple_alu_opcodes[BPF_OP(insn->code)];
  2049				EMIT2(0xD3, add_1reg(b3, dst_reg));
  2050	
  2051				if (src_reg != BPF_REG_4) {
  2052					if (insn->dst_reg == BPF_REG_4)
  2053						/* mov dst_reg, r11 */
  2054						EMIT_mov(insn->dst_reg, AUX_REG);
  2055					else
  2056						EMIT1(0x59); /* pop rcx */
  2057				}
  2058	
  2059				break;
  2060	
  2061			case BPF_ALU | BPF_END | BPF_FROM_BE:
  2062			case BPF_ALU64 | BPF_END | BPF_FROM_LE:
  2063				switch (imm32) {
  2064				case 16:
  2065					/* Emit 'ror %ax, 8' to swap lower 2 bytes */
  2066					EMIT1(0x66);
  2067					if (is_ereg(dst_reg))
  2068						EMIT1(0x41);
  2069					EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
  2070	
  2071					/* Emit 'movzwl eax, ax' */
  2072					if (is_ereg(dst_reg))
  2073						EMIT3(0x45, 0x0F, 0xB7);
  2074					else
  2075						EMIT2(0x0F, 0xB7);
  2076					EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
  2077					break;
  2078				case 32:
  2079					/* Emit 'bswap eax' to swap lower 4 bytes */
  2080					if (is_ereg(dst_reg))
  2081						EMIT2(0x41, 0x0F);
  2082					else
  2083						EMIT1(0x0F);
  2084					EMIT1(add_1reg(0xC8, dst_reg));
  2085					break;
  2086				case 64:
  2087					/* Emit 'bswap rax' to swap 8 bytes */
  2088					EMIT3(add_1mod(0x48, dst_reg), 0x0F,
  2089					      add_1reg(0xC8, dst_reg));
  2090					break;
  2091				}
  2092				break;
  2093	
  2094			case BPF_ALU | BPF_END | BPF_FROM_LE:
  2095				switch (imm32) {
  2096				case 16:
  2097					/*
  2098					 * Emit 'movzwl eax, ax' to zero extend 16-bit
  2099					 * into 64 bit
  2100					 */
  2101					if (is_ereg(dst_reg))
  2102						EMIT3(0x45, 0x0F, 0xB7);
  2103					else
  2104						EMIT2(0x0F, 0xB7);
  2105					EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
  2106					break;
  2107				case 32:
  2108					/* Emit 'mov eax, eax' to clear upper 32-bits */
  2109					if (is_ereg(dst_reg))
  2110						EMIT1(0x45);
  2111					EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
  2112					break;
  2113				case 64:
  2114					/* nop */
  2115					break;
  2116				}
  2117				break;
  2118	
  2119				/* speculation barrier */
  2120			case BPF_ST | BPF_NOSPEC:
  2121				EMIT_LFENCE();
  2122				break;
  2123	
  2124				/* ST: *(u8*)(dst_reg + off) = imm */
  2125			case BPF_ST | BPF_MEM | BPF_B:
  2126				if (is_ereg(dst_reg))
  2127					EMIT2(0x41, 0xC6);
  2128				else
  2129					EMIT1(0xC6);
  2130				goto st;
  2131			case BPF_ST | BPF_MEM | BPF_H:
  2132				if (is_ereg(dst_reg))
  2133					EMIT3(0x66, 0x41, 0xC7);
  2134				else
  2135					EMIT2(0x66, 0xC7);
  2136				goto st;
  2137			case BPF_ST | BPF_MEM | BPF_W:
  2138				if (is_ereg(dst_reg))
  2139					EMIT2(0x41, 0xC7);
  2140				else
  2141					EMIT1(0xC7);
  2142				goto st;
  2143			case BPF_ST | BPF_MEM | BPF_DW:
  2144				EMIT2(add_1mod(0x48, dst_reg), 0xC7);
  2145	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki