[PATCH bpf-next v2 0/2] bpf: Eliminate unnecessary clone/restore due to constants blinding

[PATCH bpf-next v2 0/2] bpf: Eliminate unnecessary clone/restore due to constants blinding

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

This series eliminates the unnecessary clone/restore overhead
introduced by patch d3e945223e0158c8 ("bpf: Move constants blinding
out of arch-specific JITs"), which moved constants blinding from JIT
code to general verifier code, and added env->insn_aux_data and subprog
starts duplication and restoration around constants blinding operations
to resolve the inconsistency between the metadata and instruction arrays.

Patch 1 eliminates the insn_aux_data duplication/restore. After JIT
failure, bpf_clear_insn_aux_data() is the only function that requires
insnsi and insn_aux_data to stay in sync as it accesses them using the
same size and index. However, the insnsi access is unnecessary as it is
checked to skip the second slot of an ldimm64 instruction which can be
absorbed into the jt check itself, so removing it eliminates the need
for the duplication/restore.

Patch 2 eliminates the prog clone and subprog duplication/restore
when the interpreter fallback is not available on JIT failure.

v2:
- Address UAF in bpf_fixup_call_args() (sashiko)

v1: https://lore.kernel.org/bpf/cover.1783775928.git.xukuohai@huaweicloud.com/

Xu Kuohai (2):
  bpf: Eliminate dup/restore of insn_aux_data
  bpf: Remove unnecessary dup/restore subprog_starts and prog clone

 include/linux/bpf_verifier.h |   1 +
 include/linux/filter.h       |  22 ++-----
 kernel/bpf/core.c            | 120 +++++++++++++++++------------------
 kernel/bpf/fixups.c          |  73 ++++++---------------
 kernel/bpf/verifier.c        |   4 +-
 5 files changed, 88 insertions(+), 132 deletions(-)

-- 
2.47.3

[PATCH bpf-next v2 1/2] bpf: Eliminate dup/restore of insn_aux_data

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

The dup/restore of insn_aux_data was introduced to resolve the
inconsistency between insnsi and insn_aux_data arrays, which occurs
on the failure path where insnsi was rolled back to the original
state before constants blinding, while insn_aux_data was not.

After JIT failure, there is only one user, bpf_clear_insn_aux_data(),
that requires insnsi and insn_aux_data to be synchronized. It accesses
both insnsi and insn_aux_data using the same array size and index.

However, the access to insnsi in bpf_clear_insn_aux_data() is not
necessary. It is checked to skip the second slot of an ldimm64 instruction,
whose jt is never set and can be absorbed into the jt check itself.

So remove the access to insnsi from bpf_clear_insn_aux_data(), and add a
specific length field for insn_aux_data to allow it to have a different
length from the insnsi array. Then remove dup/restore of insn_aux_data.

Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 include/linux/bpf_verifier.h |  1 +
 include/linux/filter.h       | 13 -------------
 kernel/bpf/core.c            | 16 ----------------
 kernel/bpf/fixups.c          | 36 ++----------------------------------
 kernel/bpf/verifier.c        |  4 ++--
 5 files changed, 5 insertions(+), 65 deletions(-)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 682c2cd3b844..cdd50049a0c1 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -948,6 +948,7 @@ struct bpf_verifier_env {
 	bool seen_direct_write;
 	bool seen_exception;
 	bool signature;
+	u32 insn_aux_data_len;
 	struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
 	const struct bpf_line_info *prev_linfo;
 	struct bpf_verifier_log log;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 32d5297c557e..5c072c1360eb 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1211,25 +1211,12 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
 #ifdef CONFIG_BPF_SYSCALL
 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
 				     const struct bpf_insn *patch, u32 len);
-struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env);
-void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
-			       struct bpf_insn_aux_data *orig_insn_aux);
 #else
 static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
 						   const struct bpf_insn *patch, u32 len)
 {
 	return ERR_PTR(-ENOTSUPP);
 }
-
-static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
-{
-	return NULL;
-}
-
-static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
-					     struct bpf_insn_aux_data *orig_insn_aux)
-{
-}
 #endif /* CONFIG_BPF_SYSCALL */
 
 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index e2076667b245..505d134cd264 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2634,22 +2634,10 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc
 {
 #ifdef CONFIG_BPF_JIT
 	struct bpf_prog *orig_prog;
-	struct bpf_insn_aux_data *orig_insn_aux;
 
 	if (!bpf_prog_need_blind(prog))
 		return bpf_int_jit_compile(env, prog);
 
-	if (env) {
-		/*
-		 * If env is not NULL, we are called from the end of bpf_check(), at this
-		 * point, only insn_aux_data is used after failure, so it should be restored
-		 * on failure.
-		 */
-		orig_insn_aux = bpf_dup_insn_aux_data(env);
-		if (!orig_insn_aux)
-			return prog;
-	}
-
 	orig_prog = prog;
 	prog = bpf_jit_blind_constants(env, prog);
 	/*
@@ -2662,8 +2650,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc
 	prog = bpf_int_jit_compile(env, prog);
 	if (prog->jited) {
 		bpf_jit_prog_release_other(prog, orig_prog);
-		if (env)
-			vfree(orig_insn_aux);
 		return prog;
 	}
 
@@ -2671,8 +2657,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc
 
 out_restore:
 	prog = orig_prog;
-	if (env)
-		bpf_restore_insn_aux_data(env, orig_insn_aux);
 #endif
 	return prog;
 }
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index a0bddada7964..2cdad2395b71 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -174,6 +174,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
 	if (cnt == 1)
 		return;
 	prog_len = new_prog->len;
+	env->insn_aux_data_len = prog_len;
 
 	memmove(data + off + cnt - 1, data + off,
 		sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
@@ -440,7 +441,6 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
 void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
 {
 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
-	struct bpf_insn *insns = env->prog->insnsi;
 	int end = start + len;
 	int i;
 
@@ -449,9 +449,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
 			kvfree(aux_data[i].jt);
 			aux_data[i].jt = NULL;
 		}
-
-		if (bpf_is_ldimm64(&insns[i]))
-			i++;
 	}
 }
 
@@ -464,7 +461,6 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
 	if (bpf_prog_is_offloaded(env->prog->aux))
 		bpf_prog_offload_remove_insns(env, off, cnt);
 
-	/* Should be called before bpf_remove_insns, as it uses prog->insnsi */
 	bpf_clear_insn_aux_data(env, off, cnt);
 
 	err = bpf_remove_insns(env->prog, off, cnt);
@@ -483,6 +479,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
 
 	memmove(aux_data + off,	aux_data + off + cnt,
 		sizeof(*aux_data) * (orig_prog_len - off - cnt));
+	env->insn_aux_data_len -= cnt;
 
 	return 0;
 }
@@ -1005,26 +1002,6 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s
 	env->subprog_info[env->subprog_cnt].start = env->prog->len;
 }
 
-struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
-{
-	size_t size;
-	void *new_aux;
-
-	size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);
-	new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT);
-	if (new_aux)
-		memcpy(new_aux, env->insn_aux_data, size);
-	return new_aux;
-}
-
-void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
-			       struct bpf_insn_aux_data *orig_insn_aux)
-{
-	/* the expanded elements are zero-filled, so no special handling is required */
-	vfree(env->insn_aux_data);
-	env->insn_aux_data = orig_insn_aux;
-}
-
 static int jit_subprogs(struct bpf_verifier_env *env)
 {
 	struct bpf_prog *prog = env->prog, **func, *tmp;
@@ -1299,7 +1276,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 	bool blinded = false;
 	struct bpf_insn *insn;
 	struct bpf_prog *prog, *orig_prog;
-	struct bpf_insn_aux_data *orig_insn_aux;
 	u32 *orig_subprog_starts;
 
 	if (env->subprog_cnt <= 1)
@@ -1307,14 +1283,8 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 
 	prog = orig_prog = env->prog;
 	if (bpf_prog_need_blind(prog)) {
-		orig_insn_aux = bpf_dup_insn_aux_data(env);
-		if (!orig_insn_aux) {
-			err = -ENOMEM;
-			goto out_cleanup;
-		}
 		orig_subprog_starts = bpf_dup_subprog_starts(env);
 		if (!orig_subprog_starts) {
-			vfree(orig_insn_aux);
 			err = -ENOMEM;
 			goto out_cleanup;
 		}
@@ -1334,7 +1304,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 	if (blinded) {
 		bpf_jit_prog_release_other(prog, orig_prog);
 		kvfree(orig_subprog_starts);
-		vfree(orig_insn_aux);
 	}
 
 	return 0;
@@ -1364,7 +1333,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 
 out_restore:
 	bpf_restore_subprog_starts(env, orig_subprog_starts);
-	bpf_restore_insn_aux_data(env, orig_insn_aux);
 	kvfree(orig_subprog_starts);
 out_cleanup:
 	/* cleanup main prog to be interpreted */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e6f35f4e715b..e31c41203bda 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -20122,7 +20122,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (!is_priv)
 		mutex_lock(&bpf_verifier_lock);
 
-	len = env->prog->len;
+	len = env->insn_aux_data_len = env->prog->len;
 	env->insn_aux_data =
 		__vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len),
 			  GFP_KERNEL_ACCOUNT | __GFP_ZERO);
@@ -20368,7 +20368,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	release_btfs(env);
 err_free_env:
 	if (env->insn_aux_data)
-		bpf_clear_insn_aux_data(env, 0, env->prog->len);
+		bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len);
 	vfree(env->insn_aux_data);
 	kvfree(env->fd_array);
 	bpf_stack_liveness_free(env);
-- 
2.47.3

[PATCH bpf-next v2 2/2] bpf: Remove unnecessary dup/restore subprog_starts and prog clone

[Xu Kuohai]

From: Xu Kuohai <xukuohai@huawei.com>

When JIT is required by the bpf program, the kernel rejects the program
on JIT failure rather than falling back to the interpreter. In this case,
the subprog_starts duplication/restore and bpf prog clone are unnecessary,
since they are only used to restore state for the interpreter. So remove
the duplication/restore for this case.

Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
 include/linux/filter.h |   9 ++--
 kernel/bpf/core.c      | 104 +++++++++++++++++++++++------------------
 kernel/bpf/fixups.c    |  37 +++++++--------
 3 files changed, 83 insertions(+), 67 deletions(-)

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 5c072c1360eb..2d1aa062566c 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1356,7 +1356,8 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 
 const char *bpf_jit_get_prog_name(struct bpf_prog *prog);
 
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog);
+int bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog **pprog,
+			    bool clone_needed, bool *cloned);
 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
 
 static inline bool bpf_prog_need_blind(const struct bpf_prog *prog)
@@ -1508,9 +1509,11 @@ static inline bool bpf_prog_need_blind(const struct bpf_prog *prog)
 }
 
 static inline
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
+int bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog **pprog,
+			    bool clone_needed, bool *cloned)
 {
-	return prog;
+	*cloned = false;
+	return 0;
 }
 
 static inline void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 505d134cd264..ab5da3ff941a 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1564,27 +1564,34 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
  * Now this function is used only to blind the main prog and must be invoked only when
  * bpf_prog_need_blind() returns true.
  */
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
+int bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog **pprog,
+			    bool clone_needed, bool *cloned)
 {
 	struct bpf_insn insn_buff[16], aux[2];
-	struct bpf_prog *clone, *tmp;
+	struct bpf_prog *prog, *orig_prog, *tmp;
 	int insn_delta, insn_cnt;
 	struct bpf_insn *insn;
 	int i, rewritten;
 
-	if (WARN_ON_ONCE(env && env->prog != prog))
-		return ERR_PTR(-EINVAL);
+	*cloned = false;
+	if (WARN_ON_ONCE(env && env->prog != *pprog))
+		return -EINVAL;
 
-	clone = bpf_prog_clone_create(prog, GFP_USER);
-	if (!clone)
-		return ERR_PTR(-ENOMEM);
+	prog = orig_prog = *pprog;
+	/* only clone the prog when we can fall back to the interpreter */
+	if (clone_needed) {
+		prog = bpf_prog_clone_create(orig_prog, GFP_USER);
+		if (!prog)
+			return -ENOMEM;
 
-	/* make sure bpf_patch_insn_data() patches the correct prog */
-	if (env)
-		env->prog = clone;
+		*cloned = true;
+		/* make sure bpf_patch_insn_data() patches the correct prog */
+		if (env)
+			env->prog = prog;
+	}
 
-	insn_cnt = clone->len;
-	insn = clone->insnsi;
+	insn_cnt = prog->len;
+	insn = prog->insnsi;
 
 	for (i = 0; i < insn_cnt; i++, insn++) {
 		if (bpf_pseudo_func(insn)) {
@@ -1605,42 +1612,49 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
 		    insn[1].code == 0)
 			memcpy(aux, insn, sizeof(aux));
 
-		rewritten = bpf_jit_blind_insn(insn, aux, insn_buff,
-						clone->aux->verifier_zext);
+		rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, prog->aux->verifier_zext);
 		if (!rewritten)
 			continue;
 
 		if (env)
 			tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
 		else
-			tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
+			tmp = bpf_patch_insn_single(prog, i, insn_buff, rewritten);
 
 		if (IS_ERR_OR_NULL(tmp)) {
-			if (env)
-				/* restore the original prog */
-				env->prog = prog;
-			/* Patching may have repointed aux->prog during
-			 * realloc from the original one, so we need to
-			 * fix it up here on error.
-			 */
-			bpf_jit_prog_release_other(prog, clone);
-			return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
+			if (*cloned) {
+				/* roll back to the original prog */
+				*pprog = orig_prog;
+				if (env)
+					env->prog = orig_prog;
+				/* Patching may have repointed aux->prog during
+				 * realloc from the original one, so we need to
+				 * fix it up here on error.
+				 */
+				bpf_jit_prog_release_other(orig_prog, prog);
+			} else {
+				/* just keep the latest successfully patched prog */
+				*pprog = prog;
+			}
+			return IS_ERR(tmp) ? PTR_ERR(tmp) : -ENOMEM;
 		}
 
-		clone = tmp;
+		prog = tmp;
 		insn_delta = rewritten - 1;
 
 		if (env)
-			env->prog = clone;
+			env->prog = prog;
 
 		/* Walk new program and skip insns we just inserted. */
-		insn = clone->insnsi + i + insn_delta;
+		insn = prog->insnsi + i + insn_delta;
 		insn_cnt += insn_delta;
 		i        += insn_delta;
 	}
 
-	clone->blinded = 1;
-	return clone;
+	prog->blinded = 1;
+	*pprog = prog;
+
+	return 0;
 }
 
 bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
@@ -2630,33 +2644,33 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
 	return select_interpreter;
 }
 
-static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
+static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog,
+					     bool jit_needed)
 {
 #ifdef CONFIG_BPF_JIT
+	int err;
+	bool cloned = false;
 	struct bpf_prog *orig_prog;
 
 	if (!bpf_prog_need_blind(prog))
 		return bpf_int_jit_compile(env, prog);
 
 	orig_prog = prog;
-	prog = bpf_jit_blind_constants(env, prog);
-	/*
-	 * If blinding was requested and we failed during blinding, we must fall
-	 * back to the interpreter.
-	 */
-	if (IS_ERR(prog))
-		goto out_restore;
+	err = bpf_jit_blind_constants(env, &prog, !jit_needed, &cloned);
+	if (err)
+		goto out;
 
 	prog = bpf_int_jit_compile(env, prog);
-	if (prog->jited) {
-		bpf_jit_prog_release_other(prog, orig_prog);
-		return prog;
+	if (cloned) {
+		if (prog->jited) {
+			bpf_jit_prog_release_other(prog, orig_prog);
+		} else {
+			bpf_jit_prog_release_other(orig_prog, prog);
+			prog = orig_prog;
+		}
 	}
 
-	bpf_jit_prog_release_other(orig_prog, prog);
-
-out_restore:
-	prog = orig_prog;
+out:
 #endif
 	return prog;
 }
@@ -2686,7 +2700,7 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct
 		if (*err)
 			return fp;
 
-		fp = bpf_prog_jit_compile(env, fp);
+		fp = bpf_prog_jit_compile(env, fp, jit_needed);
 		bpf_prog_jit_attempt_done(fp);
 		if (!fp->jited && jit_needed) {
 			*err = -ENOTSUPP;
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 2cdad2395b71..6da0f8d0b761 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -996,6 +996,8 @@ static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
 
 static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
 {
+	if (!orig_starts)
+		return;
 	for (int i = 0; i < env->subprog_cnt; i++)
 		env->subprog_info[i].start = orig_starts[i];
 	/* restore the start of fake 'exit' subprog as well */
@@ -1273,35 +1275,33 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 int bpf_jit_subprogs(struct bpf_verifier_env *env)
 {
 	int err, i;
-	bool blinded = false;
 	struct bpf_insn *insn;
 	struct bpf_prog *prog, *orig_prog;
-	u32 *orig_subprog_starts;
+	u32 *orig_subprog_starts = NULL;
+	bool cloned = false;
 
 	if (env->subprog_cnt <= 1)
 		return 0;
 
 	prog = orig_prog = env->prog;
 	if (bpf_prog_need_blind(prog)) {
-		orig_subprog_starts = bpf_dup_subprog_starts(env);
-		if (!orig_subprog_starts) {
-			err = -ENOMEM;
-			goto out_cleanup;
+		if (!prog->jit_required) {
+			orig_subprog_starts = bpf_dup_subprog_starts(env);
+			if (!orig_subprog_starts) {
+				err = -ENOMEM;
+				goto out_cleanup;
+			}
 		}
-		prog = bpf_jit_blind_constants(env, prog);
-		if (IS_ERR(prog)) {
-			err = -ENOMEM;
-			prog = orig_prog;
+		err = bpf_jit_blind_constants(env, &prog, !prog->jit_required, &cloned);
+		if (err)
 			goto out_restore;
-		}
-		blinded = true;
 	}
 
 	err = jit_subprogs(env);
 	if (err)
 		goto out_jit_err;
 
-	if (blinded) {
+	if (cloned) {
 		bpf_jit_prog_release_other(prog, orig_prog);
 		kvfree(orig_subprog_starts);
 	}
@@ -1309,13 +1309,13 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 	return 0;
 
 out_jit_err:
-	if (blinded) {
+	if (cloned) {
 		bpf_jit_prog_release_other(orig_prog, prog);
 		/* roll back to the clean original prog */
 		prog = env->prog = orig_prog;
 		goto out_restore;
 	} else {
-		if (err != -EFAULT) {
+		if (err != -EFAULT && !prog->jit_required) {
 			/*
 			 * We will fall back to interpreter mode when err is not -EFAULT, before
 			 * that, insn->off and insn->imm should be restored to their original
@@ -1344,8 +1344,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 int bpf_fixup_call_args(struct bpf_verifier_env *env)
 {
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
-	struct bpf_prog *prog = env->prog;
-	struct bpf_insn *insn = prog->insnsi;
+	struct bpf_insn *insn;
 	int depth;
 #endif
 	int i, err = 0;
@@ -1371,7 +1370,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 			return err;
 	}
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
-	if (prog->jit_required) {
+	if (env->prog->jit_required) {
 		verbose(env, "program requires BPF JIT compiler but it is not available\n");
 		return -EINVAL;
 	}
@@ -1388,7 +1387,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
 		return -EINVAL;
 	}
-	for (i = 0; i < prog->len; i++, insn++) {
+	for (i = 0, insn = env->prog->insnsi; i < env->prog->len; i++, insn++) {
 		if (bpf_pseudo_func(insn)) {
 			/* When JIT fails the progs with callback calls
 			 * have to be rejected, since interpreter doesn't support them yet.
-- 
2.47.3