* [PATCH bpf-next 01/49] bpf: Look for bpf_clone_redirect() once in check_recursion()
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 02/49] bpf: Make stack liveness faster Alexei Starovoitov
` (47 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
check_recursion() scans all insns of the program for each recursive
subprog. There is no limit on subprogs now. So the cost is subprogs x
insns, and no signal ends that loop. Scan once.
100000 static subprogs of 4 insns in one cycle BPF_PROG_LOAD
before 34 s
after 11 s
Fixes: f7e93aa4a34f ("bpf: Remove BPF_MAX_SUBPROGS")
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
kernel/bpf/verifier.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1a8e5a108a6..48b8a790d6e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5485,6 +5485,7 @@ static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog);
*/
static int check_recursion(struct bpf_verifier_env *env, struct bpf_call_graph *cg)
{
+ bool reenters = bpf_prog_reenters_datapath(env->prog);
int j, s;
for_each_marked_subprog(cg, s, CG_LIVE) {
@@ -5506,7 +5507,7 @@ static int check_recursion(struct bpf_verifier_env *env, struct bpf_call_graph *
return -EINVAL;
}
/* It runs on top of its own frames. The half of the stack is not enough. */
- if (bpf_prog_reenters_datapath(env->prog)) {
+ if (reenters) {
verbose(env, "recursive %s() in a program that calls bpf_clone_redirect()\n",
bpf_subprog_name(env, s));
return -EINVAL;
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 02/49] bpf: Make stack liveness faster
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 01/49] bpf: Look for bpf_clone_redirect() once in check_recursion() Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:01 ` sashiko-bot
2026-11-20 8:24 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 03/49] bpf: Delete the old main pass of the verifier Alexei Starovoitov
` (46 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
compute_subprog_args() follows the pointers to the stack. Each round
visits all insns and joins all slots, one by one. Most of that work
finds no change. Do only what can find a change:
old new
each round visits all insns only the insns whose input changed
first visit: join each slot memcpy()
later visit: join each slot memcmp() first
half slots to slots: by bits by words
The fixed point is the same.
load time, ms
program before after
pyperf600 222 124
pyperf100 126 64
strobemeta 153 116
profiler1 kprobe__vfs_link 18 8
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
kernel/bpf/liveness.c | 60 ++++++++++++++++++++++++++++---------------
1 file changed, 40 insertions(+), 20 deletions(-)
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index cfa416f7620..bd186d3c201 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -752,11 +752,19 @@ static int cmp_instances(const void *pa, const void *pb)
/* OR the 8-byte slots touched by a half-slot (4-byte) mask into @slots. */
static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask, u32 nbits)
{
- u32 slot;
+ unsigned long x;
+ u32 i;
- for (slot = 0; slot < MAX_BPF_STACK_SLOTS && slot * 2 + 1 < nbits; slot++)
- if (test_bit(slot * 2, mask) || test_bit(slot * 2 + 1, mask))
- __set_bit(slot, slots);
+ nbits = min(nbits, FRAME_HALF_SPIS);
+ for (i = 0; i < nbits; i += 32) {
+ x = bitmap_read(mask, i, min(nbits - i, 32));
+ x = (x | x >> 1) & 0x55555555;
+ x = (x | x >> 1) & 0x33333333;
+ x = (x | x >> 2) & 0x0f0f0f0f;
+ x = (x | x >> 4) & 0x00ff00ff;
+ x = (x | x >> 8) & 0x0000ffff;
+ slots[BIT_WORD(i / 2)] |= x << (i / 2 % BITS_PER_LONG);
+ }
}
/*
@@ -858,7 +866,7 @@ static int print_instances(struct bpf_verifier_env *env)
struct arg_track {
union {
s16 off[MAX_ARG_OFFSETS]; /* byte offsets; off_cnt says how many */
- u32 mask; /* arg bitmask when arg == ARG_IMPRECISE */
+ u64 mask; /* arg bitmask if ARG_IMPRECISE. As wide as off[] for memcmp() */
};
s8 frame; /* absolute frame index, or enum arg_track_state */
s8 off_cnt; /* 0 = offset-imprecise, 1-4 = # of precise offsets */
@@ -912,7 +920,7 @@ static void verbose_arg_track(struct bpf_verifier_env *env, struct arg_track *at
switch (at->frame) {
case ARG_NONE: verbose(env, "_"); break;
case ARG_UNVISITED: verbose(env, "?"); break;
- case ARG_IMPRECISE: verbose(env, "IMP%x", at->mask); break;
+ case ARG_IMPRECISE: verbose(env, "IMP%llx", at->mask); break;
default:
/* frame >= 0: absolute frame index */
if (at->off_cnt == 0) {
@@ -1103,9 +1111,18 @@ static bool arg_track_join_all(struct bpf_verifier_env *env, int idx, int target
bool changed = false;
int r;
+ if (!arg_is_visited(&in[BPF_REG_FP])) {
+ memcpy(in, out, MAX_AT_TRACK_REGS * sizeof(*in));
+ memcpy(stack_in, stack_out, nslots * sizeof(*stack_in));
+ return true;
+ }
for (r = 0; r < MAX_AT_TRACK_REGS; r++)
changed |= arg_track_join(env, idx, target, r, &in[r], out[r]);
- for (r = 0; r < nslots; r++)
+ same = !memcmp(stack_in, stack_out, nslots * sizeof(*stack_in));
+ /* a change pays for the next visit of @target too */
+ bpf_add_cost(env, MAX_AT_TRACK_REGS + nslots,
+ same && !changed ? BPF_COST_COPY : BPF_COST_XREG);
+ for (r = 0; r < nslots && !same; r++)
changed |= arg_track_join(env, idx, target, -r - 1, &stack_in[r], stack_out[r]);
return changed;
}
@@ -2078,8 +2095,8 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS];
struct arg_track unvisited = { .frame = ARG_UNVISITED };
struct arg_track none = { .frame = ARG_NONE };
- bool changed;
int i, p, r, err = -ENOMEM;
+ unsigned long *changed = NULL;
at_in = kvmalloc_objs(*at_in, len, BPF_VERIFIER_GFP);
if (!at_in)
@@ -2090,7 +2107,8 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
goto err_free;
at_stack_out = kvmalloc_objs(*at_stack_out, nslots, BPF_VERIFIER_GFP);
- if (!at_stack_out)
+ changed = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
+ if (!at_stack_out || !changed)
goto err_free;
for (i = 0; i < len; i++) {
@@ -2123,9 +2141,9 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
if (env->log.level & BPF_LOG_LEVEL2)
verbose(env, "subprog#%d: analyzing (depth %d)...\n", subprog, depth);
- /* Forward fixed-point iteration in reverse post order */
+ /* Forward fixed-point iteration in reverse post order, over insns whose input changed */
+ __set_bit(0, changed);
redo:
- changed = false;
for (p = po_end - 1; p >= po_start; p--) {
int idx = env->cfg.insn_postorder[p];
int i = idx - start;
@@ -2136,17 +2154,17 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
err = -EINTR;
if (bpf_must_abort())
goto err_free;
- if (!arg_is_visited(&at_in[i][0]) && !arg_is_visited(&at_in[i][1]))
+ if (!__test_and_clear_bit(i, changed))
continue;
memcpy(at_out, at_in[i], sizeof(at_out));
memcpy(at_stack_out, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*at_stack_out));
ti = bpf_head_guard_target(env, idx) - start;
- if (ti >= 0)
- changed |= arg_track_join_all(env, idx, start + ti, at_in[ti],
- &at_stack_in[(size_t)ti * nslots],
- at_out, at_stack_out, nslots);
+ if (ti >= 0 && arg_track_join_all(env, idx, start + ti, at_in[ti],
+ &at_stack_in[(size_t)ti * nslots],
+ at_out, at_stack_out, nslots))
+ __set_bit(ti, changed);
arg_track_xfer(env, insn, idx, at_out, at_stack_out, nslots,
at_stack_arg_entry, instance, callsites);
@@ -2162,12 +2180,13 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
if (target < start || target >= end)
continue;
ti = target - start;
- changed |= arg_track_join_all(env, idx, target, at_in[ti],
- &at_stack_in[(size_t)ti * nslots],
- at_out, at_stack_out, nslots);
+ if (arg_track_join_all(env, idx, target, at_in[ti],
+ &at_stack_in[(size_t)ti * nslots],
+ at_out, at_stack_out, nslots))
+ __set_bit(ti, changed);
}
}
- if (changed)
+ if (!bitmap_empty(changed, len))
goto redo;
/* Record memory accesses using converged at_in (RPO skips dead code) */
@@ -2210,6 +2229,7 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
err = 0;
err_free:
+ kvfree(changed);
kvfree(at_stack_out);
kvfree(at_stack_in);
kvfree(at_in);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 02/49] bpf: Make stack liveness faster
2026-11-20 7:41 ` [PATCH bpf-next 02/49] bpf: Make stack liveness faster Alexei Starovoitov
@ 2026-11-20 8:01 ` sashiko-bot
2026-11-20 8:24 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:01 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [High] bpf: build failure in arg_track_join_all() due to undeclared
identifiers
--
commit cea430c3bc4cf95a99331bfb62eb0f7db9d5efe1
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Make stack liveness faster
This commit speeds up BPF stack liveness analysis by visiting only
instructions whose input changed, using memcpy() and memcmp() for slot
joins, and converting half-slots to slots by words.
> diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
> index cfa416f762097..bd186d3c201a3 100644
> --- a/kernel/bpf/liveness.c
> +++ b/kernel/bpf/liveness.c
[ ... ]
> @@ -1103,9 +1111,18 @@ static bool arg_track_join_all(struct bpf_verifier_env *env, int idx, int target
> bool changed = false;
> int r;
>
> + if (!arg_is_visited(&in[BPF_REG_FP])) {
> + memcpy(in, out, MAX_AT_TRACK_REGS * sizeof(*in));
> + memcpy(stack_in, stack_out, nslots * sizeof(*stack_in));
> + return true;
> + }
> for (r = 0; r < MAX_AT_TRACK_REGS; r++)
> changed |= arg_track_join(env, idx, target, r, &in[r], out[r]);
> - for (r = 0; r < nslots; r++)
> + same = !memcmp(stack_in, stack_out, nslots * sizeof(*stack_in));
> + /* a change pays for the next visit of @target too */
> + bpf_add_cost(env, MAX_AT_TRACK_REGS + nslots,
> + same && !changed ? BPF_COST_COPY : BPF_COST_XREG);
> + for (r = 0; r < nslots && !same; r++)
> changed |= arg_track_join(env, idx, target, -r - 1, &stack_in[r], stack_out[r]);
> return changed;
> }
[Severity: High]
Does arg_track_join_all() fail to build here because same, bpf_add_cost(),
BPF_COST_COPY, and BPF_COST_XREG are undeclared? These identifiers are
only declared and defined later in the series in commit "bpf: Limit the
cost and the memory of a load", breaking compilation and git bisection at
this commit.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=2
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 02/49] bpf: Make stack liveness faster
2026-11-20 7:41 ` [PATCH bpf-next 02/49] bpf: Make stack liveness faster Alexei Starovoitov
2026-11-20 8:01 ` sashiko-bot
@ 2026-11-20 8:24 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:24 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2558 bytes --]
> bpf: Make stack liveness faster
TL;DR: arg_track_join_all() appears not to compile: 'same', bpf_add_cost() and
BPF_COST_* are undeclared until a later patch, breaking bisection; cost
accounting is also missing from the commit message.
> diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
> --- a/kernel/bpf/liveness.c
> +++ b/kernel/bpf/liveness.c
[ ... ]
> @@ -1103,9 +1111,18 @@ static bool arg_track_join_all(struct bpf_verifier_env *env, int idx, int target
> bool changed = false;
> int r;
>
> + if (!arg_is_visited(&in[BPF_REG_FP])) {
> + memcpy(in, out, MAX_AT_TRACK_REGS * sizeof(*in));
> + memcpy(stack_in, stack_out, nslots * sizeof(*stack_in));
> + return true;
> + }
> for (r = 0; r < MAX_AT_TRACK_REGS; r++)
> changed |= arg_track_join(env, idx, target, r, &in[r], out[r]);
> - for (r = 0; r < nslots; r++)
> + same = !memcmp(stack_in, stack_out, nslots * sizeof(*stack_in));
> + /* a change pays for the next visit of @target too */
> + bpf_add_cost(env, MAX_AT_TRACK_REGS + nslots,
> + same && !changed ? BPF_COST_COPY : BPF_COST_XREG);
> + for (r = 0; r < nslots && !same; r++)
> changed |= arg_track_join(env, idx, target, -r - 1, &stack_in[r], stack_out[r]);
> return changed;
> }
Does this compile? arg_track_join_all() assigns and reads 'same', but the
only locals declared are 'bool changed = false;' and 'int r;'.
bpf_add_cost(), BPF_COST_COPY and BPF_COST_XREG also don't appear to be
defined anywhere in the tree at this commit. A grep for bpf_add_cost and
BPF_COST_ only finds these two lines in kernel/bpf/liveness.c.
Since liveness.o is built whenever CONFIG_BPF_SYSCALL=y, this looks like it
would fail with "'same' undeclared", an implicit declaration of
bpf_add_cost(), and "'BPF_COST_COPY' undeclared". The cost accounting and
the comment "a change pays for the next visit of @target too" aren't
mentioned in the commit message either.
This breaks bisection from this commit until a later patch in the series,
ecbd98bcc45a ("bpf: Limit the cost and the memory of a load"), changes the
declaration to 'bool changed = false, same;' and adds
BPF_COST_SCAN/COPY/XREG/INSN/STATE and bpf_add_cost() to
include/linux/bpf_verifier.h.
Could 'bool same' be declared here and the bpf_add_cost() call moved into
ecbd98bcc45a so that each patch builds?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 03/49] bpf: Delete the old main pass of the verifier
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 01/49] bpf: Look for bpf_clone_redirect() once in check_recursion() Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 02/49] bpf: Make stack liveness faster Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 04/49] bpf: Delete scalar evolution Alexei Starovoitov
` (45 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The main pass decides at each join point if two states become one. It
decides before it knows if the join does harm. So it guesses, with one
table or one rule for each shape of code:
what decides where
8 tables of "need" flow_compute_need_const()
kinds of values bpf_flow_compute_kinds()
bounds for widening flow_compute_rungs()
rules for two states flow_join_makes_up(), flow_*_differs()
walks of loops flow_try_walk()
caps on states 1024 at an insn, 256 for each insn, ...
A valid program that no rule fits is rejected, or it runs into a cap.
This series changes the order. Join first, and decide after an error.
An insn whose check fails is "unproven". The next run can prove it:
join all states --> error ? --no--> accept
^ | yes
| v
keep apart what the unproven insn depends on
Delete the tables, the rules, the caps, the work list, the join, the
walks, the compare of registers and references, and the code that
builds the log from kept states. flow.c keeps the order of the insns
and the places for may_goto.
The patch only deletes. Later patches add the new code as whole
functions. The kernel builds again with "bpf: Verify in passes".
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 142 --
kernel/bpf/fixups.c | 1 -
kernel/bpf/flow.c | 3104 ----------------------------------
kernel/bpf/join.c | 621 -------
kernel/bpf/log.c | 47 -
kernel/bpf/states.c | 281 ---
kernel/bpf/verifier.c | 563 ------
7 files changed, 4759 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index f45c18b3cfa..b0f520c168d 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -412,72 +412,6 @@ struct bpf_loop_iters {
bool spill_base;
};
-/* Where may_goto that the verifier adds goes, see check_loop_guard() */
-enum bpf_guard_edge {
- BPF_NO_GUARD_EDGE,
- BPF_GUARD_LEAVES_SUBPROG,
- BPF_GUARD_LEAVES_LOOP,
-};
-
-struct loop_stack_entry {
- struct bpf_verifier_state *entry_state;
- u32 min_header_count;
- u32 max_header_count;
- u32 loop_id:23;
- u32 terminates:1;
- /* the loop had may_goto when the state entered it, see check_loop_guard() */
- u32 guarded:1;
- /*
- * The iterations are kept apart: states with different @trip are not
- * joined for as long as they are in the loop.
- */
- u32 apart:1;
- /* max_header_count follows from the state at the entry, not from patience */
- u32 counted:1;
- /* the walk goes on for as long as each iteration decides a jump that leaves the loop */
- u32 probe:1;
- /* this one did */
- u32 decided:1;
- /*
- * The state is kept apart from one with the other value of a flag, see
- * flow_join_makes_up(). The test of the flag is decided in both, and decides nothing.
- */
- u32 flag_apart:1;
- /*
- * The first iteration of a counted loop is kept apart. The state that comes back
- * to the head is made to stand for the rest.
- */
- u32 first:1;
- /*
- * How many times the state came back to the head of the loop. With @apart that is
- * the iteration that it is in. A state that stands for many iterations is a join, and has
- * the largest number of the states that it is made of.
- */
- u32 trip;
- /* the number of references that the state had when it entered the loop */
- u32 refs;
- /*
- * Which of the states that entered the loop this one comes from. What comes back to
- * the head is for that state, see flow_merge(). A join has the number of one of the two.
- */
- u32 entry_id;
- /*
- * A walk on patience: which one it is, and how many insns may be looked at in it, see
- * env->walk_insns. The size of the loop says nothing about the functions that it calls.
- */
- u32 walk_id;
- u32 walk_patience;
- int iv_reg;
- struct scalar iv_range;
- int count_insn;
-};
-
-/* The state is in a walk on patience, which may not get to the end of the loop */
-static inline bool bpf_loop_walk_may_give_up(const struct loop_stack_entry *entry)
-{
- return entry->apart && entry->terminates && !entry->counted && !entry->first;
-}
-
#define LOOP_STACK_SIZE 16
/* state of the program:
@@ -529,8 +463,6 @@ struct bpf_func_state {
* | number of simulations is tracked in frame N
*/
u32 callback_depth;
- /* Which of the states that got to the call this one comes from, see flow_merge() */
- u32 callback_entry_id;
/* Instructions processed in this frame and callees on the current path. */
u32 insns_subtotal;
/*
@@ -828,23 +760,6 @@ struct bpf_loop_exit {
int to; /* instruction outside the loop */
};
-/* The places that the table of a loop knows: registers, then stack slots, fp-8 first */
-#define BPF_LOOP_SLOT 64
-#define BPF_LOOP_PLACES 128
-
-/*
- * Where the bounds of the number in @place may come to rest at the head of a loop: the lowest and
- * the highest upper bound, the highest and the lowest lower bound that the jumps of the loop
- * let come back to the head, see flow_compute_rungs().
- */
-struct bpf_loop_rungs {
- s64 max[2];
- s64 min[2];
- bool has_max;
- bool has_min;
- u8 place;
-};
-
struct bpf_loop {
struct bpf_backedge backedges[MAX_BACKEDGES];
/* edges exiting from this loop, includes edges from nested loops */
@@ -861,19 +776,8 @@ struct bpf_loop {
bool rt_bounded;
/* insns in the loop, nested loops included */
u32 nr_insns;
- /*
- * a walk on patience whose jumps were all decided, or the two trips of a loop with calls,
- * didn't get to the end of the loop: there is no other try
- */
- bool walk_failed;
- /* functions of the program are called in the loop */
- bool has_calls;
/* not rt_bounded: there are places for may_goto, see guard_to in bpf_insn_aux_data */
bool has_guards;
- /* and it is going to be there, see bpf_flow_loop_gets_guards() */
- bool guarded;
- /* a state went through it without the edges of may_goto, see bpf_flow_loop_gets_guards() */
- bool no_guard_edges;
/*
* Loop-entry registers used to compute stack addresses for loads,
* stores and calls requiring fixed stack offsets, in this and nested
@@ -891,10 +795,6 @@ struct bpf_loop {
*/
unsigned long linear_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
unsigned long linear32_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- /* the places that insns of the loop write, as far as the insns tell */
- unsigned long writes[BITS_TO_LONGS(BPF_LOOP_PLACES)];
- struct bpf_loop_rungs *rungs;
- u32 rungs_cnt;
};
struct bpf_insn_aux_data {
@@ -936,9 +836,6 @@ struct bpf_insn_aux_data {
DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS);
DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS);
u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */
- /* regs that hold a number or a pointer to arena; that, or a pointer to a plain map value */
- u16 plain_regs;
- u16 value_regs;
int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
u32 seen; /* this insn was processed by the verifier at env->pass_cnt */
u64 nospec:1; /* do not execute this instruction speculatively */
@@ -956,7 +853,6 @@ struct bpf_insn_aux_data {
u64 guard_at_header:1; /* there is no better place than the head of the loop */
u64 guard_no_unwind:1; /* it cannot unwind from here, at least sometimes */
u64 guard_unset_reg:1; /* where it goes a register is read that is not set here */
- u64 guard_edge_dropped:1; /* an edge of may_goto from here was dropped */
u64 need_scev:1;
/* a jump in a loop: the edge that is not taken, bit 0, or the other one leaves the loop */
u64 edge_leaves:2;
@@ -1310,8 +1206,6 @@ struct bpf_verifier_env {
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_loop_gets_guards() */
u32 loop_entry_cnt; /* see loop_stack_entry.entry_id, callback_entry_id */
- u32 walk_cnt; /* see loop_stack_entry.walk_id */
- u32 *walk_insns; /* the insns that were looked at in each of them */
u32 max_arena_stack_depth; /* of all call chains */
struct bpf_verifier_state *cur_state; /* current verifier state */
struct bpf_map **used_maps; /* array of map's used by eBPF program */
@@ -1450,7 +1344,6 @@ struct bpf_verifier_env {
struct bpf_leave_cond *leave_conds;
struct scev *scev;
struct bpf_flow *flow;
- struct bpf_join_state *join;
};
static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@@ -1509,15 +1402,6 @@ __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
char __user *log_buf, u32 log_size);
void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos);
-
-/* The end of the log, taken out of it by bpf_vlog_stash() */
-struct bpf_vlog_stash {
- char *buf;
- u64 len, lost_start, lost_end;
-};
-
-int bpf_vlog_stash(struct bpf_verifier_log *log, u64 pos, struct bpf_vlog_stash *stash);
-void bpf_vlog_unstash(struct bpf_verifier_log *log, struct bpf_vlog_stash *stash);
int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual);
__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
@@ -1621,44 +1505,19 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
struct bpf_verifier_state *cur);
-/* join.c */
-int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
- const struct bpf_verifier_state *a, const struct bpf_verifier_state *b,
- bool widen);
-void bpf_join_free(struct bpf_verifier_env *env);
-
/* flow.c */
int bpf_flow_prepare(struct bpf_verifier_env *env);
void bpf_flow_reset(struct bpf_verifier_env *env);
void bpf_flow_free(struct bpf_verifier_env *env);
-bool bpf_flow_is_point(struct bpf_verifier_env *env, int insn_idx);
-struct bpf_verifier_state *bpf_flow_keep_state(struct bpf_verifier_env *env,
- const struct bpf_verifier_state *st);
-int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- int insn_idx, int prev_insn_idx);
-int bpf_flow_guard_edge(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- int insn_idx, int prev_insn_idx, enum bpf_guard_edge edge);
-int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx);
-void bpf_flow_print_way(struct bpf_verifier_env *env, u64 log_pos, int head, int i);
int bpf_flow_call(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
struct bpf_verifier_state *entry, int subprog, int call_idx);
int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int exit_idx);
void bpf_flow_next_is_no_bound(struct bpf_verifier_env *env, int insn_idx, int h);
bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env);
-void bpf_flow_loop_gets_guards(struct bpf_verifier_env *env, int h);
-void bpf_flow_new_round(struct bpf_verifier_env *env);
int bpf_flow_place_guards(struct bpf_verifier_env *env);
-int bpf_flow_compute_kinds(struct bpf_verifier_env *env);
-u16 bpf_flow_flag_regs(struct bpf_verifier_env *env, int idx);
-bool bpf_flow_is_needed(struct bpf_verifier_env *env, int idx, u32 r);
-/* verifier.c, for flow.c */
-int bpf_flow_loop_leave(struct bpf_verifier_env *env, bool *enters);
-int bpf_flow_loop_enter(struct bpf_verifier_env *env, bool *backedge, bool *drop, bool *ends_walk);
int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
const struct bpf_verifier_state *exit, int call_idx);
int bpf_insert_loop_guards(struct bpf_verifier_env *env);
-u16 bpf_call_const_arg_regs(struct bpf_verifier_env *env, struct bpf_insn *insn);
-u16 bpf_call_checked_arg_regs(struct bpf_verifier_env *env, struct bpf_insn *insn);
bool bpf_insn_is_iter_next(struct bpf_verifier_env *env, struct bpf_insn *insn);
bool bpf_insn_leaves_loop(struct bpf_verifier_env *env, int h, int n);
int bpf_compute_leave_conds(struct bpf_verifier_env *env);
@@ -2133,7 +1992,6 @@ static inline bool bpf_is_kfunc_sleepable(struct bpf_call_arg_meta *meta)
return meta->kfunc_flags & KF_SLEEPABLE;
}
bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta);
-bool bpf_is_cast_kfunc(const struct bpf_call_arg_meta *meta);
struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem);
int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off);
bool bpf_insn_is_cond_jump(u8 code);
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 4d9b6798152..aca3c7a53b6 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -755,7 +755,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
for (i = start; i < end; i++) {
if (aux_data[i].loop) {
kvfree(aux_data[i].loop->exits);
- kvfree(aux_data[i].loop->rungs);
kvfree(aux_data[i].loop);
aux_data[i].loop = NULL;
}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 70766ee48eb..79d1be515f2 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -1,28 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-only
-/*
- * The main pass of the verifier as a fixpoint over the control flow graph.
- *
- * Every insn that starts a basic block has a set of states. The state that
- * arrives there is
- * - dropped, if it is within a state of the set,
- * - joined with a state of the set, if there is a join, see bpf_states_join(),
- * that doesn't make up what neither of them has, see flow_join_makes_up(),
- * - added to the set otherwise.
- * What is new in the set is pending. Pending states are processed in the order
- * of the program in which a loop is done with before what follows it, see
- * flow_compute_order(). The pass ends when nothing is pending.
- *
- * Loops. The state that enters a loop is made to stand for all iterations with
- * what scalar evolution knows about the loop, see bpf_flow_loop_enter().
- * The state that comes back to the loop head is expected to be within it.
- * If it is not, the two are joined, what grew is forgotten, and the loop is
- * done again.
- *
- * Functions. A static function that gets nothing but numbers and pointers to
- * what is not the stack is verified on its own, for the states at its entry
- * and not for every chain of calls. The call waits. What comes out of the
- * function goes to all calls that wait for the states it comes from.
- */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
@@ -32,157 +8,11 @@
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
-/*
- * States at an insn that have no join: kinds of values, references, chains of calls.
- * A state that arrives is compared with those that it may be related to, see flow_may_relate().
- * At worst that is all of them, and the work is the square of it.
- */
-#define BPF_FLOW_MAX_STATES 1024
-/*
- * All states in the sets at a time: that many for every insn of the prog, and some. Not counted:
- * calls that wait, what came out of a function, the states that loops were entered with.
- */
-#define BPF_FLOW_LIVE_PER_INSN 16
-#define BPF_FLOW_LIVE_BASE 4096
-/* All states that were ever there. It bounds the work by the size of the prog. */
-#define BPF_FLOW_TOTAL_PER_INSN 256
-#define BPF_FLOW_TOTAL_BASE 65536
-/*
- * What comes out of a function is joined that many times. Then it is widened, like what
- * comes back to a loop head: the exit states have to stop changing.
- */
-#define BPF_FLOW_MAX_JOINS 16
-/* With fewer states at an insn it's faster to compare them than to look at their shapes. */
-#define BPF_FLOW_MIN_SHAPES 8
-/* bpf_loop() with that many iterations at most is walked. */
-#define BPF_FLOW_MAX_CALLBACKS 8
-/* What a callback returns to the helper with is joined that many times before it is widened. */
-#define BPF_FLOW_CALLBACK_JOINS 4
-
-enum flow_cycle {
- FLOW_NO_CYCLE,
- /* what grew is forgotten */
- FLOW_CYCLE,
- /* after a while. A flag that a callback sets doesn't grow any further. */
- FLOW_CALLBACK_CYCLE,
-};
-
-/* See flow_may_relate() */
-enum flow_word {
- FLOW_CHAIN,
- FLOW_CTX,
- FLOW_LAYOUT,
- FLOW_PTRS,
- FLOW_OFFS,
- FLOW_SHAPE_WORDS,
-};
-
-struct flow_shape {
- u32 word[FLOW_SHAPE_WORDS];
- /* how much of the packet is checked, the sum over the pointers to the packet */
- u32 pkt_range;
- /* what comes after FLOW_CTX is computed */
- bool places;
-};
-
-struct flow_state {
- struct bpf_verifier_state st;
- /* states at the insn, see struct flow_set, calls of struct flow_entry */
- struct list_head node;
- struct rb_node rb; /* pending states */
- int prev_insn_idx;
- u32 seq;
- u32 from; /* seq of the last state on the way to it that is kept */
- struct flow_shape shape;
- u16 joins;
- bool pending;
- /* the order of pending states, see flow_set_pending() */
- u8 key_len;
- u32 key[MAX_CALL_FRAMES + 1];
-};
-
-/* what came out of a function that is verified on its own */
-struct flow_exit {
- struct list_head node;
- struct bpf_verifier_state st;
- u16 joins;
-};
-
-/*
- * Every call that waits makes an entry of the function: the state at its first insn and what
- * becomes of it. Entries whose states met are one set: what comes out of the function for one of
- * them is for the calls of all of them. One entry stands for the set and has its lists.
- */
-struct flow_entry {
- struct flow_entry *up; /* towards the one that stands for the set: up == itself */
- struct list_head calls; /* struct flow_state: the calls that wait */
- struct list_head exits; /* struct flow_exit: what came out */
-};
-
-struct flow_func {
- int exit_idx; /* all exits are compared as if they were here */
-};
-
-/* States at an insn, in the order in which they came. */
-struct flow_set {
- struct list_head list;
-};
-
-struct flow_united {
- u32 a, b;
-};
-
struct flow_kept {
struct list_head node;
struct bpf_verifier_state st;
};
-/* What the checks ahead of an insn look at, see enum flow_need_kind */
-struct flow_need {
- u64 slots; /* of the stack, fp-8 is bit 0 */
- u16 regs;
-};
-
-/* see flow_compute_need_const() */
-enum flow_need_kind {
- FLOW_NEED_ARG,
- FLOW_NEED_STACK_OFF,
- FLOW_NEED_PTR_OFF,
- FLOW_NEED_JUMP,
- FLOW_NEED_FLAG,
- FLOW_NEED_VALUE_OFF,
- FLOW_NEED_BASE,
- FLOW_NEED_JUMP_ALU,
- FLOW_NEED_KINDS,
-};
-
-struct bpf_flow {
- /* computed once */
- u32 len; /* of the prog during the pass */
- u32 nr_funcs;
- u32 *pos; /* place of the insn in the order */
- u32 *rank; /* callers go first */
- struct flow_need *need[FLOW_NEED_KINDS];
- unsigned long *point; /* insn starts a basic block */
- unsigned long *keep; /* states that were processed are compared with */
-
- /* of the pass */
- struct flow_set *states; /* per insn */
- struct flow_set *enter; /* per insn: the states that enter the loop at its head */
- u32 *nr_states; /* per insn: the two sets are counted together */
- struct rb_root_cached pending;
- struct flow_func *funcs;
- struct flow_entry **entries; /* bpf_verifier_state.flow_entry is the index, 0 is none */
- u32 entry_cnt, entry_cap;
- struct flow_united *united;
- u32 united_cnt, united_cap;
- struct list_head kept;
- struct bpf_verifier_state tmp;
- u32 seq;
- u32 from; /* flow_state.from of what comes from the current state */
- u32 live_states;
-};
-
/*
* The order. Reverse post order is a topological order of the graph without
* back edges, but it may put what follows a loop between the insns of
@@ -428,45 +258,6 @@ bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env)
return false;
}
-/*
- * A state is in the loop @h without a count, so the loop gets may_goto. At run time may_goto is
- * there for everybody, also for those who know the number of trips or got to the end of the loop
- * trip by trip. It can be out of budget in the first trip, because of a loop before this one.
- *
- * So a loop has may_goto or it has not, and a state takes the edges of may_goto if the loop had
- * it when the state entered. If a state entered without it, with a count or on a walk that may
- * give up, all is done again, and the loop has may_goto from the start.
- */
-void bpf_flow_loop_gets_guards(struct bpf_verifier_env *env, int h)
-{
- struct bpf_loop *loop = env->insn_aux_data[h].loop;
-
- if (loop->guarded)
- return;
- loop->guarded = true;
- if (loop->no_guard_edges)
- env->guards_late = true;
-}
-
-/* The main prog and the global functions are about to be verified, maybe not for the first time */
-void bpf_flow_new_round(struct bpf_verifier_env *env)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- int i;
-
- env->restart = false;
- env->guards_late = false;
- for (i = 0; i < env->prog->len; i++) {
- aux[i].guard_edge_dropped = false;
- if (!aux[i].loop)
- continue;
- aux[i].loop->no_guard_edges = false;
- /* what the last round found out */
- if (aux[i].loop->rt_bounded)
- aux[i].loop->rt_bounded = flow_loop_is_bounded(env, i);
- }
-}
-
/*
* Which loops have something that stops them at run time, and where may_goto goes in the others.
* Returns 1 if a place is at the head of a loop.
@@ -493,9 +284,6 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
return -E2BIG;
}
aux[h].loop->nr_insns++;
- if (bpf_pseudo_call(&env->prog->insnsi[i]) ||
- bpf_is_callx(&env->prog->insnsi[i]))
- aux[h].loop->has_calls = true;
}
}
@@ -517,2895 +305,3 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
kvfree(seen);
return ret;
}
-
-/* The slot that @insn spills to or fills from, if it's one that struct flow_need has a bit for */
-static int flow_fp_slot(const struct bpf_insn *insn, u8 base)
-{
- u8 mode = BPF_MODE(insn->code);
-
- if (base != BPF_REG_FP || (mode != BPF_MEM && mode != BPF_MEMSX) ||
- insn->off >= 0 || insn->off < -64 * BPF_REG_SIZE || insn->off % BPF_REG_SIZE)
- return -1;
- return -insn->off / BPF_REG_SIZE - 1;
-}
-
-/*
- * A number is added to a pointer to the stack. A store at an offset that is not known
- * forgets what the slots it may hit hold, pointers above all.
- */
-static u16 flow_stack_off_regs(struct bpf_verifier_env *env, int idx)
-{
- u16 ptrs = env->insn_aux_data[idx].stack_ptrs;
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- u8 op = BPF_OP(insn->code);
-
- if (BPF_CLASS(insn->code) != BPF_ALU64 || BPF_SRC(insn->code) != BPF_X ||
- (op != BPF_ADD && op != BPF_SUB))
- return 0;
- if ((ptrs & BIT(insn->dst_reg)) && !(ptrs & BIT(insn->src_reg)))
- return BIT(insn->src_reg);
- if (op == BPF_ADD && (ptrs & BIT(insn->src_reg)) && !(ptrs & BIT(insn->dst_reg)))
- return BIT(insn->dst_reg);
- return 0;
-}
-
-/*
- * A jump that compares a register with a constant, which may be in the other register.
- * A call does the same: the verifier compares a size, or what bpf_set_retval() or bpf_throw()
- * takes, with what is allowed. So does the exit of the prog, see flow_compute_need_const().
- */
-static u16 flow_tested_regs(struct bpf_verifier_env *env, int idx)
-{
- struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
- u16 consts = aux->const_reg_mask | aux->const_reg_wide_mask;
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
-
- if (bpf_helper_call(insn) || bpf_pseudo_kfunc_call(insn))
- return bpf_call_checked_arg_regs(env, insn);
- if ((class != BPF_JMP && class != BPF_JMP32) ||
- op == BPF_JA || op == BPF_CALL || op == BPF_EXIT || op == BPF_JCOND)
- return 0;
- if (BPF_SRC(insn->code) == BPF_K || (consts & BIT(insn->src_reg)))
- return BIT(insn->dst_reg);
- return consts & BIT(insn->dst_reg) ? BIT(insn->src_reg) : 0;
-}
-
-/* A jump that compares a register with 0 or 1 */
-u16 bpf_flow_flag_regs(struct bpf_verifier_env *env, int idx)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
-
- if (BPF_SRC(insn->code) != BPF_K || !bpf_insn_is_cond_jump(insn->code) ||
- BPF_OP(insn->code) == BPF_JCOND)
- return 0;
- if (BPF_OP(insn->code) == BPF_JSET ? !(insn->imm & 1) : (u32)insn->imm > 1)
- return 0;
- return BIT(insn->dst_reg);
-}
-
-/* With or without sign extension */
-static bool flow_is_copy(const struct bpf_insn *insn)
-{
- return BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && !insn->imm &&
- (!insn->off || insn->off == 8 || insn->off == 16 || insn->off == 32);
-}
-
-static bool flow_is_shift(const struct bpf_insn *insn)
-{
- u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
-
- return (class == BPF_ALU || class == BPF_ALU64) &&
- (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH);
-}
-
-/*
- * The places of a function that hold a value: the registers, the arguments on the stack that it
- * got and those that it hands over, the slots of the stack that struct flow_need has a bit for.
- */
-enum {
- FLOW_ARG_IN = 16,
- FLOW_ARG_OUT = FLOW_ARG_IN + MAX_STACK_ARG_SLOTS,
- FLOW_SLOT = 64,
- FLOW_PLACES = 128,
-};
-static_assert(FLOW_ARG_OUT + MAX_STACK_ARG_SLOTS <= FLOW_SLOT);
-
-enum flow_kind {
- FLOW_IS_PLAIN, /* a number or a pointer to arena: nothing checks where it points to */
- FLOW_IS_VALUE, /* that, or a pointer to a map value that holds nothing but numbers */
- FLOW_IS_CTX, /* the context, as the prog or a global function got it */
- FLOW_KINDS,
- FLOW_IS_OTHER = FLOW_KINDS,
-};
-
-/* What the places hold for sure in front of an insn */
-struct flow_kinds {
- unsigned long is[FLOW_KINDS][BITS_TO_LONGS(FLOW_PLACES)];
-};
-
-static void flow_kind_set(struct flow_kinds *k, u32 place, enum flow_kind kind)
-{
- int i;
-
- for (i = 0; i < FLOW_KINDS; i++)
- __assign_bit(place, k->is[i], i == kind);
- if (kind == FLOW_IS_PLAIN)
- __set_bit(place, k->is[FLOW_IS_VALUE]);
-}
-
-static void flow_kind_copy(struct flow_kinds *to, u32 place, const struct flow_kinds *from,
- u32 from_place)
-{
- int i;
-
- for (i = 0; i < FLOW_KINDS; i++)
- __assign_bit(place, to->is[i], test_bit(from_place, from->is[i]));
-}
-
-/* What holds on both ways to an insn. Returns true if that is less than before. */
-static bool flow_kinds_meet(struct flow_kinds *to, const struct flow_kinds *from)
-{
- unsigned long lost = 0;
- int i, j;
-
- for (i = 0; i < FLOW_KINDS; i++) {
- for (j = 0; j < ARRAY_SIZE(to->is[i]); j++) {
- lost |= to->is[i][j] & ~from->is[i][j];
- to->is[i][j] &= from->is[i][j];
- }
- }
- return lost;
-}
-
-/* The first insn of a function that is called from @k */
-static bool flow_kinds_enter(struct flow_kinds *to, const struct flow_kinds *k)
-{
- struct flow_kinds e = {};
- int i;
-
- /* r12 points to arena. The rest is not set. */
- bitmap_fill(e.is[FLOW_IS_PLAIN], FLOW_PLACES);
- bitmap_fill(e.is[FLOW_IS_VALUE], FLOW_PLACES);
- flow_kind_set(&e, BPF_REG_FP, FLOW_IS_OTHER);
- for (i = BPF_REG_1; i <= BPF_REG_5; i++)
- flow_kind_copy(&e, i, k, i);
- for (i = 0; i < MAX_STACK_ARG_SLOTS; i++)
- flow_kind_copy(&e, FLOW_ARG_IN + i, k, FLOW_ARG_OUT + i);
- return flow_kinds_meet(to, &e);
-}
-
-/* The place that @insn loads from or stores to through r10 or r11, -1 if it's something else */
-static int flow_mem_place(const struct bpf_insn *insn, u8 base)
-{
- int n = (abs(insn->off) - 1) / BPF_REG_SIZE;
-
- if (!insn->off)
- return -1;
- if (base == BPF_REG_FP && insn->off < 0 && n < 64)
- return FLOW_SLOT + n;
- if (base == BPF_REG_PARAMS && n < MAX_STACK_ARG_SLOTS)
- return (insn->off < 0 ? FLOW_ARG_OUT : FLOW_ARG_IN) + n;
- return -1;
-}
-
-static enum flow_kind flow_map_value_kind(const struct bpf_map *map)
-{
- if (map->map_type == BPF_MAP_TYPE_ARENA)
- return FLOW_IS_PLAIN;
- /* what is at a known offset of a read-only map is known: a number, a string, a function */
- if (map->map_flags & BPF_F_RDONLY_PROG)
- return FLOW_IS_OTHER;
- return IS_ERR_OR_NULL(map->record) && !map->inner_map_meta ? FLOW_IS_VALUE : FLOW_IS_OTHER;
-}
-
-/* What @insn loads through the register @base */
-static enum flow_kind flow_loaded_kind(struct bpf_verifier_env *env, const struct flow_kinds *k,
- const struct bpf_insn *insn, u8 base)
-{
- struct bpf_insn_access_aux info = { .reg_type = SCALAR_VALUE };
- int size = bpf_size_to_bytes(BPF_SIZE(insn->code));
- bool tstamp = env->prog->tstamp_type_access, valid;
-
- /* a number that is good for a load is an address in arena */
- if (test_bit(base, k->is[FLOW_IS_VALUE]))
- return FLOW_IS_PLAIN;
- if (size == BPF_REG_SIZE && !test_bit(base, k->is[FLOW_IS_CTX]))
- return FLOW_IS_OTHER;
- valid = env->ops->is_valid_access &&
- env->ops->is_valid_access(insn->off, size, BPF_READ, env->prog, &info);
- /* it notes what the prog reads, and this is a question */
- env->prog->tstamp_type_access = tstamp;
- if (valid && info.reg_type == PTR_TO_PACKET_END)
- bpf_find_containing_subprog(env, insn - env->prog->insnsi)->loads_pkt_end = true;
- /*
- * Only the context has pointers that are shorter than a register. A base that nothing is
- * known about can be the context: it came out of a struct, or only some paths have it.
- */
- if (size < BPF_REG_SIZE)
- return valid && info.reg_type != SCALAR_VALUE ? FLOW_IS_OTHER : FLOW_IS_PLAIN;
- return valid && info.reg_type == SCALAR_VALUE ? FLOW_IS_PLAIN : FLOW_IS_OTHER;
-}
-
-/* What a helper or a kfunc returns */
-static enum flow_kind flow_returned_kind(struct bpf_verifier_env *env, int idx)
-{
- struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- const struct bpf_func_proto *fn;
- struct bpf_call_arg_meta meta;
- const struct btf_type *t;
-
- if (bpf_pseudo_kfunc_call(insn)) {
- if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0)
- return FLOW_IS_OTHER;
- t = btf_type_skip_modifiers(meta.btf, meta.func_proto->type, NULL);
- if (!btf_type_is_ptr(t))
- return FLOW_IS_PLAIN;
- /* void * is a number, unless the verifier knows the kfunc: it may fail, or casts */
- t = btf_type_skip_modifiers(meta.btf, t->type, NULL);
- return btf_type_is_void(t) && !(meta.kfunc_flags & KF_RET_NULL) &&
- !bpf_is_cast_kfunc(&meta) ? FLOW_IS_PLAIN : FLOW_IS_OTHER;
- }
- if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)
- return FLOW_IS_OTHER;
- if (fn->ret_type == RET_INTEGER || fn->ret_type == RET_VOID)
- return FLOW_IS_PLAIN;
- if (base_type(fn->ret_type) == RET_PTR_TO_MAP_VALUE &&
- (aux->const_reg_map_mask & BIT(BPF_REG_1)))
- return flow_map_value_kind(env->used_maps[aux->const_reg_vals[BPF_REG_1]]);
- return FLOW_IS_OTHER;
-}
-
-/*
- * @k is what holds in front of the insn at @idx. Make it what holds behind it.
- * @ret: the registers that are FLOW_IS_PLAIN at all exits of a subprog.
- * Returns true if less is known than before at the entry or about the exits of a subprog.
- */
-static bool flow_kinds_step(struct bpf_verifier_env *env, struct flow_kinds *kinds, u16 *ret,
- struct flow_kinds *k, int idx)
-{
- struct bpf_subprog_info *sub = env->subprog_info;
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
- u8 dst = insn->dst_reg, src = insn->src_reg;
- bool wide = BPF_SIZE(insn->code) == BPF_DW && BPF_MODE(insn->code) == BPF_MEM &&
- !(insn->off % BPF_REG_SIZE);
- bool lost = false;
- u16 regs = U16_MAX;
- int i, s;
-
- if (class == BPF_ALU64 && (op == BPF_ADD || op == BPF_SUB)) {
- __clear_bit(dst, k->is[FLOW_IS_CTX]);
- for (i = 0; BPF_SRC(insn->code) == BPF_X && i < FLOW_IS_CTX; i++)
- if (!test_bit(src, k->is[i]))
- __clear_bit(dst, k->is[i]);
- } else if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && !insn->off) {
- flow_kind_copy(k, dst, k, src);
- } else if (class == BPF_ALU || class == BPF_ALU64) {
- /*
- * addr_space_cast too. It leaves a pointer that is not in arena as it is, and
- * a compiler casts what it cannot know. Nearly all of that is in arena.
- */
- flow_kind_set(k, dst, FLOW_IS_PLAIN);
- } else if (class == BPF_LDX) {
- s = flow_mem_place(insn, src);
- if (wide && s >= 0)
- flow_kind_copy(k, dst, k, s);
- else
- flow_kind_set(k, dst, flow_loaded_kind(env, k, insn, src));
- } else if (class == BPF_STX || class == BPF_ST) {
- i = bpf_atomic_load_reg(insn);
- if (i >= 0)
- flow_kind_set(k, i, flow_loaded_kind(env, k, insn,
- insn->imm == BPF_LOAD_ACQ ? src : dst));
- s = flow_mem_place(insn, dst);
- if (class == BPF_STX && wide && s >= 0)
- flow_kind_copy(k, s, k, src);
- else if (s >= 0)
- flow_kind_set(k, s, FLOW_IS_PLAIN);
- } else if (bpf_is_ldimm64(insn)) {
- if (src == BPF_PSEUDO_MAP_VALUE || src == BPF_PSEUDO_MAP_IDX_VALUE)
- flow_kind_set(k, dst, flow_map_value_kind(
- env->used_maps[env->insn_aux_data[idx].map_index]));
- else
- flow_kind_set(k, dst, src ? FLOW_IS_OTHER : FLOW_IS_PLAIN);
- } else if (insn->code == (BPF_JMP | BPF_EXIT) && !bpf_is_unwind_exit(insn)) {
- s = bpf_find_containing_subprog(env, idx) - sub;
- lost = ret[s] & ~k->is[FLOW_IS_PLAIN][0];
- ret[s] &= k->is[FLOW_IS_PLAIN][0];
- } else if (class == BPF_LD || (class == BPF_JMP && op == BPF_CALL)) {
- s = bpf_pseudo_call(insn) ? bpf_find_subprog(env, idx + insn->imm + 1) : -1;
- if (s >= 0 && !bpf_subprog_is_global(env, s)) {
- lost = flow_kinds_enter(&kinds[sub[s].start], k);
- regs = ret[s];
- } else if (bpf_is_callx(insn)) {
- regs = 0;
- }
- /* a subprog may return a pair in R0 and R2, the rest is not set */
- regs |= ~(BIT(BPF_REG_0) | BIT(BPF_REG_2));
- for (i = BPF_REG_0; i <= BPF_REG_5; i++)
- flow_kind_set(k, i, regs & BIT(i) ? FLOW_IS_PLAIN : FLOW_IS_OTHER);
- for (i = 0; i < MAX_STACK_ARG_SLOTS; i++)
- flow_kind_set(k, FLOW_ARG_OUT + i, FLOW_IS_PLAIN);
- if (bpf_helper_call(insn) || bpf_pseudo_kfunc_call(insn))
- flow_kind_set(k, BPF_REG_0, flow_returned_kind(env, idx));
- }
- return lost;
-}
-
-/*
- * Find the registers that hold a number or a pointer to arena, and those that point to the value
- * of a map that holds nothing but numbers at worst, whichever way the prog gets to an insn:
- * plain_regs and value_regs of struct bpf_insn_aux_data. The others may hold any pointer.
- *
- * Forwards, from "all is known" down, until nothing changes. Arguments go from the callers to
- * the callees and results the other way, so the subprogs are looked at in both orders in turn.
- *
- * Nothing depends on it but which states are kept apart. A pointer that is taken for
- * a number costs a rule. A number that is taken for a pointer costs states. So it's fine to
- * miss a store through a pointer to the stack, and to take what callx calls for unknown.
- *
- * The edges that the verifier adds, see bpf_head_guard_target(): may_goto at the head of a loop
- * takes what holds there to its target. A subprog that is out of budget goes to an exit with 0
- * in place of the result, a number.
- */
-int bpf_flow_compute_kinds(struct bpf_verifier_env *env)
-{
- struct bpf_subprog_info *sub = env->subprog_info;
- int len = env->prog->len, cnt = env->subprog_cnt, i, j, n, s, t, idx, err = 0;
- struct flow_kinds *kinds, k = {};
- struct bpf_iarray *succ;
- bool lost = true;
- u16 *ret;
-
- kinds = kvmalloc_array(len, sizeof(*kinds), BPF_VERIFIER_GFP);
- ret = kvmalloc_array(env->subprog_cnt, sizeof(*ret), BPF_VERIFIER_GFP);
- if (!kinds || !ret) {
- kvfree(kinds);
- kvfree(ret);
- return -ENOMEM;
- }
- memset(kinds, 0xff, len * sizeof(*kinds));
- memset(ret, 0xff, env->subprog_cnt * sizeof(*ret));
-
- /* what the kernel, a helper, callx or another prog calls: only the context is known */
- for (s = 0; s < env->subprog_cnt; s++) {
- if (s && !bpf_subprog_is_global(env, s) && !sub[s].addr_taken)
- continue;
- flow_kinds_enter(&kinds[sub[s].start], &k);
- for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
- if (sub[s].args[i].arg_type == ARG_PTR_TO_CTX ||
- (!s && !i && env->prog->type != BPF_PROG_TYPE_EXT))
- flow_kind_set(&kinds[sub[s].start], BPF_REG_1 + i, FLOW_IS_CTX);
- }
-
- for (n = 0; lost; n++) {
- lost = false;
- /* a chain of calls takes a round for each link */
- if (bpf_must_abort()) {
- err = -EINTR;
- break;
- }
- for (j = 0; j < cnt; j++) {
- /* it has callees first */
- s = env->subprog_topo_order[n & 1 ? j : cnt - 1 - j];
- for (i = sub[s + 1].postorder_start; i-- > sub[s].postorder_start;) {
- idx = env->cfg.insn_postorder[i];
- k = kinds[idx];
- lost |= flow_kinds_step(env, kinds, ret, &k, idx);
- succ = bpf_insn_successors(env, idx);
- iarray_for_each(t, succ)
- lost |= flow_kinds_meet(&kinds[t], &k);
- t = bpf_head_guard_target(env, idx);
- if (t >= 0)
- lost |= flow_kinds_meet(&kinds[t], &kinds[idx]);
- }
- }
- }
- for (i = 0; i < len; i++) {
- env->insn_aux_data[i].plain_regs = kinds[i].is[FLOW_IS_PLAIN][0];
- env->insn_aux_data[i].value_regs = kinds[i].is[FLOW_IS_VALUE][0];
- }
- kvfree(kinds);
- kvfree(ret);
- return err;
-}
-
-/*
- * A number is added to what may be a pointer whose offset the verifier looks at:
- * - An object of the kernel, the context, a socket take a known offset only.
- * - The packet: the offset was compared with the end of the packet before the states met.
- * - The value of a map that has a timer, a lock, a pointer in the middle.
- * - The value of a read-only map: what is loaded from a known offset is known.
- * Not a value that holds nothing but numbers. Only the lowest and the highest offset
- * count there, and the join of two constants has both. @any_value: that one too.
- */
-static u16 flow_ptr_off_regs(struct bpf_verifier_env *env, int idx, bool any_value)
-{
- struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- u16 both = BIT(insn->dst_reg) | BIT(insn->src_reg), ptrs;
- u8 op = BPF_OP(insn->code);
-
- if (BPF_CLASS(insn->code) != BPF_ALU64 || BPF_SRC(insn->code) != BPF_X ||
- (op != BPF_ADD && op != BPF_SUB) || (aux->stack_ptrs & both))
- return 0;
- ptrs = both & ~(any_value ? aux->plain_regs : aux->value_regs);
- if (op == BPF_SUB)
- ptrs &= BIT(insn->dst_reg);
- /* the one that may be the pointer makes the other one the number */
- return ptrs && ptrs != both ? both ^ ptrs : ptrs;
-}
-
-/* The base of a load or a store, unless it's a number or a pointer to arena for sure */
-static u16 flow_base_regs(struct bpf_verifier_env *env, int idx)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- u8 class = BPF_CLASS(insn->code);
- u16 regs = 0;
-
- if (class == BPF_LDX || bpf_atomic_is_load_acq(insn))
- regs = BIT(insn->src_reg);
- else if (class == BPF_ST || class == BPF_STX)
- regs = BIT(insn->dst_reg);
- return regs & ~env->insn_aux_data[idx].plain_regs & GENMASK(BPF_REG_9, BPF_REG_0);
-}
-
-/*
- * Some arguments of kfuncs and helpers have to be known constants: the id of a type,
- * the size of what is reserved in a ring buffer, the size of a slice of a dynptr.
- * A compiler makes one call of
- * if (x) p = bpf_rdonly_cast(q, ID); else p = bpf_rdonly_cast(q, 0);
- * and one store of
- * if (x) a[n++] = p; else b[m++] = p;
- * where a and b are on the stack.
- * and one load of
- * x = ipv6 ? *(u16 *)(pkt + 54) : *(u16 *)(pkt + 34); y = z ? task->pid : task->tgid;
- * The join of two constants is not a constant. Find the registers and the slots of the stack
- * that end up in such arguments, or in offsets in the stack or from a pointer,
- * like live registers are found: backwards, until nothing changes.
- * States in which they differ are not joined.
- * Nothing depends on it but what is known: it's fine to miss a store through a pointer.
- *
- * FLOW_NEED_JUMP: those that a jump ahead compares with a constant, see flow_join_makes_up().
- * As they are, or extended: without 32-bit insns a compiler tests
- * r7 = r8; r7 <<= 32; r7 >>= 32;
- * and goes on with r8.
- * A slot that is filled through another pointer to the stack than r10 is not one of them, and
- * flow_null_differs() counts on that.
- *
- * FLOW_NEED_FLAG: those of which a jump tells 0 from 1, see flow_join_makes_up(). As they are,
- * extended, or masked: a bool is tested as "w1 &= 1; if w1 != 0". Not through a call.
- * The jump ends the basic block of the insn, or a block that this block leads to. Not further:
- * every state that is kept apart goes through all that is on the way to the jump once more.
- * So there are two rounds. In the first one nothing comes from the blocks ahead. What it finds
- * at the first insn of a block is what the second one takes at the end of the blocks that lead
- * there. In both a block depends on nothing that the round itself computes for another block,
- * so the order in which the insns are looked at doesn't matter.
- *
- * The last three kinds keep no states apart. They are for bpf_flow_is_needed(), which asks
- * whether anything ahead looks at a number that a loop changes from iteration to iteration.
- * FLOW_NEED_VALUE_OFF: FLOW_NEED_PTR_OFF, and the offset in a map value that holds nothing but
- * numbers. It has to be in the value. It can be an argument, or what a function returns:
- * return v->hist[count_bits(x)];
- * So can an offset in the stack. FLOW_NEED_STACK_OFF doesn't follow it through a return, so this
- * table has it too.
- * FLOW_NEED_BASE: the base of a load or a store, as it is, behind a cast, moved by a constant:
- * p = NULL; for (i = 0; i < 3; i++) { if (p) n += p->len; p = next(); }
- * FLOW_NEED_JUMP_ALU: FLOW_NEED_JUMP, and what the tested number is made of step by step:
- * for (i = 0; i < 3; i++) x /= 10; if (x < 8) ...
- * In x op= y that is x. y is not followed.
- * What is stored through a pointer to the stack other than r10 counts as tested. It's a result
- * for the caller, and no table follows it from the store to the fill:
- * count_bits(x, &n); return v->hist[n];
- * These two go through callx, to every function whose address is taken.
- */
-static int flow_compute_need_const(struct bpf_verifier_env *env, struct flow_need **pneed,
- enum flow_need_kind kind)
-{
- struct bpf_insn *insns = env->prog->insnsi, *insn;
- struct flow_need in, out, *need, *from, *ahead = NULL;
- struct bpf_iarray *succ;
- bool jump = kind == FLOW_NEED_JUMP || kind == FLOW_NEED_JUMP_ALU;
- bool callx = kind == FLOW_NEED_BASE || kind == FLOW_NEED_JUMP_ALU;
- bool flag = kind == FLOW_NEED_FLAG, second = false;
- bool changed = true, any = false, keep, from_src, reg;
- int i, s, idx, slot, err = 0;
- u16 *gen, callx_regs = 0, pair = BIT(BPF_REG_0) | BIT(BPF_REG_2);
- u8 *ret, class, op;
-
- need = kvcalloc(env->prog->len, sizeof(*need), BPF_VERIFIER_GFP);
- gen = kvcalloc(env->prog->len, sizeof(u16), BPF_VERIFIER_GFP);
- /* the callers need what the subprog returns: r0, and r2 of a pair */
- ret = kvcalloc(env->subprog_cnt, sizeof(u8), BPF_VERIFIER_GFP);
- if (flag)
- ahead = kvcalloc(env->prog->len, sizeof(*need), BPF_VERIFIER_GFP);
- *pneed = need;
- if (!need || !gen || !ret || (flag && !ahead)) {
- kvfree(ahead);
- kvfree(gen);
- kvfree(ret);
- return -ENOMEM;
- }
- for (i = 0; i < env->prog->len; i++) {
- if (jump)
- gen[i] = flow_tested_regs(env, i);
- else if (flag)
- gen[i] = bpf_flow_flag_regs(env, i);
- else if (kind == FLOW_NEED_STACK_OFF)
- gen[i] = flow_stack_off_regs(env, i);
- else if (kind == FLOW_NEED_PTR_OFF || kind == FLOW_NEED_VALUE_OFF)
- gen[i] = flow_ptr_off_regs(env, i, kind == FLOW_NEED_VALUE_OFF);
- else if (kind == FLOW_NEED_BASE)
- gen[i] = flow_base_regs(env, i);
- else if (bpf_helper_call(&insns[i]) || bpf_pseudo_kfunc_call(&insns[i]))
- gen[i] = bpf_call_const_arg_regs(env, &insns[i]);
- if (kind == FLOW_NEED_JUMP_ALU && BPF_CLASS(insns[i].code) == BPF_STX &&
- insns[i].dst_reg != BPF_REG_FP &&
- (env->insn_aux_data[i].stack_ptrs & BIT(insns[i].dst_reg)))
- gen[i] = BIT(insns[i].src_reg);
- if (kind == FLOW_NEED_VALUE_OFF)
- gen[i] |= flow_stack_off_regs(env, i);
- any |= gen[i];
- }
- if (jump) {
- /* the verifier compares what the prog and its callbacks return with a range */
- ret[0] = BIT(BPF_REG_0);
- /* no insn takes the address of this one. It's 0 if there is none */
- ret[env->exception_callback_subprog] = BIT(BPF_REG_0);
- for (i = 0; i < env->prog->len; i++) {
- if (!bpf_pseudo_func(&insns[i]))
- continue;
- s = bpf_find_subprog(env, i + insns[i].imm + 1);
- if (s >= 0)
- ret[s] = BIT(BPF_REG_0);
- }
- any = true;
- }
-again:
- while (any && changed) {
- changed = false;
- if (bpf_must_abort()) {
- err = -EINTR;
- break;
- }
- for (s = 0; callx && s < env->subprog_cnt; s++)
- if (env->subprog_info[s].addr_taken)
- callx_regs |= need[env->subprog_info[s].start].regs;
- for (i = 0; i < env->cfg.cur_postorder; i++) {
- idx = env->cfg.insn_postorder[i];
- insn = &insns[idx];
- class = BPF_CLASS(insn->code);
- succ = bpf_insn_successors(env, idx);
- out.regs = 0;
- out.slots = 0;
- from = flag && env->insn_aux_data[idx].bb_end ? ahead : need;
- for (s = 0; s < succ->cnt; s++) {
- out.regs |= from[succ->items[s]].regs;
- out.slots |= from[succ->items[s]].slots;
- }
- in = out;
- if (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) {
- in.regs &= ~GENMASK(BPF_REG_5, BPF_REG_0);
- if (bpf_is_callx(insn))
- in.regs |= callx_regs & GENMASK(BPF_REG_5, BPF_REG_1);
- s = idx + insn->imm + 1;
- if (bpf_pseudo_call(insn) && !flag &&
- !bpf_subprog_is_global(env, bpf_find_subprog(env, s))) {
- if (kind != FLOW_NEED_PTR_OFF)
- in.regs |= need[s].regs &
- GENMASK(BPF_REG_5, BPF_REG_1);
- s = bpf_find_subprog(env, s);
- /*
- * Not for an index where states are kept apart: that is
- * one of many numbers.
- */
- if ((kind == FLOW_NEED_ARG || kind == FLOW_NEED_VALUE_OFF ||
- jump) &&
- (out.regs & pair & ~ret[s])) {
- ret[s] |= out.regs & pair;
- changed = true;
- }
- }
- } else if (insn->code == (BPF_JMP | BPF_EXIT)) {
- s = bpf_find_containing_subprog(env, idx) - env->subprog_info;
- in.regs |= ret[s];
- } else if (class == BPF_LDX || class == BPF_LD) {
- slot = class == BPF_LDX ? flow_fp_slot(insn, insn->src_reg) : -1;
- if (slot >= 0 && (out.regs & BIT(insn->dst_reg)))
- in.slots |= BIT_ULL(slot);
- in.regs &= ~BIT(insn->dst_reg);
- } else if (class == BPF_STX || class == BPF_ST) {
- slot = flow_fp_slot(insn, insn->dst_reg);
- if (slot >= 0 && (out.slots & BIT_ULL(slot))) {
- in.slots &= ~BIT_ULL(slot);
- if (class == BPF_STX)
- in.regs |= BIT(insn->src_reg);
- }
- } else if ((class == BPF_ALU || class == BPF_ALU64) &&
- (out.regs & BIT(insn->dst_reg))) {
- op = BPF_OP(insn->code);
- reg = BPF_SRC(insn->code) == BPF_X;
- /* the result is known when what it is made of is */
- keep = op != BPF_MOV;
- from_src = reg;
- if (jump || flag)
- from_src = flow_is_copy(insn);
- if (kind == FLOW_NEED_JUMP || flag)
- keep = flow_is_shift(insn) && !reg;
- if (flag && op == BPF_AND && !reg)
- keep = insn->imm & 1;
- if (kind == FLOW_NEED_BASE) {
- keep = !reg && (op == BPF_ADD || op == BPF_SUB);
- from_src = reg && op == BPF_MOV;
- }
- if (!keep)
- in.regs &= ~BIT(insn->dst_reg);
- if (from_src)
- in.regs |= BIT(insn->src_reg);
- }
- in.regs |= gen[idx];
- if (in.regs != need[idx].regs || in.slots != need[idx].slots) {
- need[idx] = in;
- changed = true;
- }
- }
- }
- if (flag && !err && !second) {
- memcpy(ahead, need, env->prog->len * sizeof(*need));
- second = changed = true;
- goto again;
- }
- /*
- * An offset that is an argument: not through the call, see above, and not where the calls
- * meet. A function would be looked at once for each offset that it's called with.
- */
- for (s = 0; kind == FLOW_NEED_PTR_OFF && s < env->subprog_cnt; s++)
- need[env->subprog_info[s].start].regs = 0;
- kvfree(ahead);
- kvfree(gen);
- kvfree(ret);
- if (!any) {
- kvfree(need);
- *pneed = NULL;
- }
- return err;
-}
-
-/*
- * Does anything ahead of the insn at @idx look at what @r holds there? @r is a register or a stack
- * slot of the function as scev.c counts them. See flow_compute_need_const(): it's a guess.
- */
-bool bpf_flow_is_needed(struct bpf_verifier_env *env, int idx, u32 r)
-{
- const struct flow_need *need;
- int i;
-
- /* no table has a bit for it */
- if (r >= MAX_BPF_REG + 64)
- return true;
- for (i = 0; i < FLOW_NEED_KINDS; i++) {
- need = env->flow->need[i];
- if (need && (r < MAX_BPF_REG ? need[idx].regs & BIT(r) :
- need[idx].slots & BIT_ULL(r - MAX_BPF_REG)))
- return true;
- }
- return false;
-}
-
-/*
- * A number that grows from one iteration of a loop to the next is not followed step by step. The
- * bound that moved is put where it may come to rest, see scalar_widen(). The jumps of
- * the loop tell where that is:
- * do { a[i] = 0; i += 3; } while (i < 30); 29 at most comes back to the head
- * while (i < 30) { a[i] = 0; i += 3; } 32
- * loop { if (...) { if (n >= 64) break; a[n++] = x; } } 64
- *
- * One iteration is looked at, from the head to the jumps back. What a register or a stack slot
- * holds is written as root + constant. A root is what a place held at the head, or what was made
- * in a place since. A copy, an extension, a spill and a fill keep the root. += and -= of a constant
- * change the constant. Everything else makes a new root. Where paths meet two constants become
- * a range of them, and two roots a new one.
- * A jump that compares root + c with a constant bounds the root on each of its edges that stays in
- * the loop. What comes back to the head in a place is root + c again: that, with the bounds of
- * the root, is where the place may come to rest.
- *
- * It's a guess. The state that comes back to the head is compared with it like with any other
- * state. So it's fine to take a 32-bit add for an add, to miss a store through a pointer, to mix up
- * two numbers that are made in one place, and to know nothing of the edges that the verifier adds.
- */
-struct flow_sym {
- s32 lo, hi;
- u8 root;
-};
-
-#define FLOW_NO_PLACE U8_MAX
-/* How many of them the look at one loop may take. A loop that needs more has no table. */
-#define BPF_FLOW_MAX_SYMS (4 << 20)
-
-static_assert(FLOW_SLOT == BPF_LOOP_SLOT && FLOW_PLACES == BPF_LOOP_PLACES);
-
-/* @insn makes the place *@to a copy of the place *@from, extended or cut to its low bits */
-static bool flow_copies_place(const struct bpf_insn *insn, u8 *to, u8 *from)
-{
- u8 class = BPF_CLASS(insn->code);
- int slot;
-
- if (class == BPF_ALU || class == BPF_ALU64) {
- *to = insn->dst_reg;
- *from = insn->src_reg;
- return flow_is_copy(insn) && *to < BPF_REG_FP && *from < BPF_REG_FP;
- }
- if (class != BPF_LDX && class != BPF_STX)
- return false;
- slot = flow_fp_slot(insn, class == BPF_LDX ? insn->src_reg : insn->dst_reg);
- if (slot < 0)
- return false;
- *to = class == BPF_LDX ? insn->dst_reg : FLOW_SLOT + slot;
- *from = class == BPF_LDX ? FLOW_SLOT + slot : insn->src_reg;
- return *to != BPF_REG_FP && *from != BPF_REG_FP;
-}
-
-/* An ALU insn that leaves a small number as it is: an extension, or half of one */
-static bool flow_extends(const struct bpf_insn *insn)
-{
- if (BPF_SRC(insn->code) != BPF_K)
- return false;
- if (flow_is_shift(insn))
- return insn->imm == 32 || insn->imm == 48 || insn->imm == 56;
- return BPF_OP(insn->code) == BPF_AND &&
- (insn->imm == 0xff || insn->imm == 0xffff || insn->imm == -1);
-}
-
-static void flow_written_places(const struct bpf_insn *insn, unsigned long *places)
-{
- u8 class = BPF_CLASS(insn->code);
- int p;
-
- bitmap_zero(places, FLOW_PLACES);
- if (class == BPF_ALU || class == BPF_ALU64 || class == BPF_LDX || bpf_is_ldimm64(insn)) {
- __set_bit(insn->dst_reg, places);
- } else if (class == BPF_LD || (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL)) {
- bitmap_set(places, BPF_REG_0, BPF_REG_5 + 1);
- } else if (class == BPF_ST || class == BPF_STX) {
- p = bpf_atomic_load_reg(insn);
- if (p >= 0)
- __set_bit(p, places);
- p = flow_mem_place(insn, insn->dst_reg);
- if (p >= FLOW_SLOT)
- __set_bit(p, places);
- }
-}
-
-/*
- * The jump at @idx compares the register *@reg with a constant: it's taken if "reg op k".
- * The constant has 32 bits, so does what is added to it below: nothing wraps.
- */
-static bool flow_jump_cmp(struct bpf_verifier_env *env, int idx, u8 *reg, u8 *op, s64 *k)
-{
- const struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
- const struct bpf_insn *insn = &env->prog->insnsi[idx];
- u8 class = BPF_CLASS(insn->code);
-
- *op = BPF_OP(insn->code);
- if ((class != BPF_JMP && class != BPF_JMP32) || *op == BPF_JA || *op == BPF_CALL ||
- *op == BPF_EXIT || *op == BPF_JCOND || *op == BPF_JSET)
- return false;
- *reg = insn->dst_reg;
- if (BPF_SRC(insn->code) == BPF_K) {
- *k = insn->imm;
- } else if (aux->const_reg_mask & BIT(insn->src_reg)) {
- *k = aux->const_reg_vals[insn->src_reg];
- } else if (aux->const_reg_mask & BIT(insn->dst_reg)) {
- *k = aux->const_reg_vals[insn->dst_reg];
- *reg = insn->src_reg;
- *op = bpf_flip_opcode(*op);
- } else {
- return false;
- }
- if (class == BPF_JMP32)
- *k = *op == BPF_JSGT || *op == BPF_JSGE || *op == BPF_JSLT || *op == BPF_JSLE ?
- (s64)(s32)*k : (s64)(u32)*k;
- return *reg < BPF_REG_FP;
-}
-
-static void flow_add_max(struct bpf_loop_rungs *r, s64 max)
-{
- r->max[0] = r->has_max ? min(r->max[0], max) : max;
- r->max[1] = r->has_max ? max(r->max[1], max) : max;
- r->has_max = true;
-}
-
-static void flow_add_min(struct bpf_loop_rungs *r, s64 min)
-{
- r->min[0] = r->has_min ? max(r->min[0], min) : min;
- r->min[1] = r->has_min ? min(r->min[1], min) : min;
- r->has_min = true;
-}
-
-/* "x op k" holds. What does that say about the root of @x? */
-static void flow_note_cmp(struct bpf_loop_rungs *root, const struct flow_sym *x, u8 op, s64 k)
-{
- bool has_max = true, has_min = true;
- s64 max = k, min = k;
-
- switch (op) {
- case BPF_JEQ:
- break;
- case BPF_JNE:
- /* it cuts off the end of a range */
- max = k - 1;
- min = k + 1;
- break;
- case BPF_JLT:
- max = k - 1;
- fallthrough;
- case BPF_JLE:
- /* the ranges are signed: nothing is said by a constant above S64_MAX */
- min = 0;
- has_max = max >= 0;
- has_min = max >= 0;
- break;
- case BPF_JGT:
- min = k + 1;
- fallthrough;
- case BPF_JGE:
- has_max = false;
- has_min = k >= 0;
- break;
- case BPF_JSLT:
- max = k - 1;
- fallthrough;
- case BPF_JSLE:
- has_min = false;
- break;
- case BPF_JSGT:
- min = k + 1;
- fallthrough;
- case BPF_JSGE:
- has_max = false;
- break;
- default:
- return;
- }
- if (has_max)
- flow_add_max(root, max - x->lo);
- if (has_min)
- flow_add_min(root, min - x->hi);
-}
-
-/* @cur is what the @n places hold in front of @insn. Make it what they hold behind it. */
-static void flow_sym_step(const struct bpf_insn *insn, struct flow_sym *cur, const u8 *idx, u32 n)
-{
- u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code), to, from;
- DECLARE_BITMAP(written, FLOW_PLACES);
- struct flow_sym *x;
- s64 c = insn->imm;
- int p;
-
- /* both are among the places or none is, see flow_compute_loop_rungs() */
- if (flow_copies_place(insn, &to, &from)) {
- if (idx[to] != FLOW_NO_PLACE)
- cur[idx[to]] = cur[idx[from]];
- return;
- }
- if ((class == BPF_ALU || class == BPF_ALU64) && idx[insn->dst_reg] != FLOW_NO_PLACE) {
- x = &cur[idx[insn->dst_reg]];
- if (flow_extends(insn))
- return;
- if (op == BPF_SUB)
- c = -c;
- if ((op == BPF_ADD || op == BPF_SUB) && BPF_SRC(insn->code) == BPF_K &&
- x->lo + c >= S32_MIN && x->hi + c <= S32_MAX) {
- x->lo += c;
- x->hi += c;
- return;
- }
- }
- flow_written_places(insn, written);
- for_each_set_bit(p, written, FLOW_PLACES)
- if (idx[p] != FLOW_NO_PLACE)
- cur[idx[p]] = (struct flow_sym){ .root = n + idx[p] };
-}
-
-/* One more path leads to where @to holds */
-static void flow_sym_meet(struct flow_sym *to, const struct flow_sym *from, u32 n)
-{
- u32 j;
-
- if (to[0].root == FLOW_NO_PLACE) {
- memcpy(to, from, n * sizeof(*to));
- return;
- }
- for (j = 0; j < n; j++) {
- if (to[j].root != from[j].root) {
- to[j] = (struct flow_sym){ .root = n + j };
- continue;
- }
- to[j].lo = min(to[j].lo, from[j].lo);
- to[j].hi = max(to[j].hi, from[j].hi);
- }
-}
-
-static u8 flow_place_set(u8 *set, u8 p)
-{
- while (set[p] != p) {
- set[p] = set[set[p]];
- p = set[p];
- }
- return p;
-}
-
-/* @at: the insns in the order. The insns of a loop are next to each other, the head first. */
-static int flow_compute_loop_rungs(struct bpf_verifier_env *env, const u32 *at, u32 total, int h)
-{
- const struct bpf_insn *insns = env->prog->insnsi, *insn;
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_loop_rungs *roots = NULL, *rungs, *r;
- u8 set[FLOW_PLACES], idx[FLOW_PLACES], to, from, reg, op;
- u32 first = env->flow->pos[h], nr, n = 0, back_cnt = 0, cnt = 0, j, k;
- DECLARE_BITMAP(tested, FLOW_PLACES) = {};
- DECLARE_BITMAP(moves, FLOW_PLACES) = {};
- struct flow_sym *syms, *cur, *back, *x;
- struct bpf_loop *loop = aux[h].loop;
- const u32 *pos = env->flow->pos;
- struct bpf_iarray *succ;
- int i, p, t, err = -ENOMEM;
- bool carry = false;
- s64 c;
-
- nr = min(loop->nr_insns, total - first);
-
- /*
- * The places to look at: what a jump compares with a constant, and what that is copied
- * from and to. One of them has to change in the loop.
- */
- for (p = 0; p < FLOW_PLACES; p++)
- set[p] = p;
- for (k = 0; k < nr; k++) {
- i = at[first + k];
- if (flow_copies_place(&insns[i], &to, &from))
- set[flow_place_set(set, to)] = flow_place_set(set, from);
- else if (flow_jump_cmp(env, i, ®, &op, &c))
- __set_bit(reg, tested);
- }
- for_each_set_bit(p, tested, FLOW_PLACES)
- __set_bit(flow_place_set(set, p), moves);
- bitmap_copy(tested, moves, FLOW_PLACES);
- bitmap_zero(moves, FLOW_PLACES);
- for_each_set_bit(p, loop->writes, FLOW_PLACES)
- __set_bit(flow_place_set(set, p), moves);
- for (p = 0; p < FLOW_PLACES; p++) {
- i = flow_place_set(set, p);
- idx[p] = test_bit(i, tested) && test_bit(i, moves) ? n++ : FLOW_NO_PLACE;
- }
- if (!n || (u64)n * nr > BPF_FLOW_MAX_SYMS)
- return 0;
-
- /* for each insn, for the current insn, for each jump back */
- syms = kvmalloc_array((size_t)nr + 1 + loop->backedges_cnt, n * sizeof(*syms),
- BPF_VERIFIER_GFP);
- /* for each root, and the result for each place */
- roots = kvcalloc(3 * n, sizeof(*roots), BPF_VERIFIER_GFP);
- if (!syms || !roots)
- goto out;
- memset(syms, 0xff, (size_t)nr * n * sizeof(*syms));
- cur = syms + (size_t)nr * n;
- back = cur + n;
- rungs = roots + 2 * n;
- for (j = 0; j < n; j++)
- syms[j] = (struct flow_sym){ .root = j };
-
- for (k = 0; k < nr; k++) {
- i = at[first + k];
- insn = &insns[i];
- if (!carry)
- memcpy(cur, syms + (size_t)k * n, n * sizeof(*cur));
- /* nothing but a jump back leads here */
- for (j = 0; !carry && j < n; j++)
- if (cur[j].root == FLOW_NO_PLACE)
- cur[j] = (struct flow_sym){ .root = n + j };
- /* a loop in the loop: nobody knows how many times it does what it does */
- if (k && aux[i].loop)
- for_each_set_bit(p, aux[i].loop->writes, FLOW_PLACES)
- if (idx[p] != FLOW_NO_PLACE)
- cur[idx[p]] = (struct flow_sym){ .root = n + idx[p] };
-
- if (flow_jump_cmp(env, i, ®, &op, &c) && idx[reg] != FLOW_NO_PLACE) {
- x = &cur[idx[reg]];
- if (pos[i + insn->off + 1] - first < nr)
- flow_note_cmp(&roots[x->root], x, op, c);
- if (pos[i + 1] - first < nr)
- flow_note_cmp(&roots[x->root], x, bpf_rev_opcode(op), c);
- }
- flow_sym_step(insn, cur, idx, n);
-
- succ = bpf_insn_successors(env, i);
- carry = succ->cnt == 1 && !test_bit(succ->items[0], env->flow->point) &&
- pos[succ->items[0]] == first + k + 1;
- if (carry)
- continue;
- iarray_for_each(t, succ) {
- if (t == h && back_cnt < loop->backedges_cnt)
- memcpy(back + (size_t)back_cnt++ * n, cur, n * sizeof(*cur));
- else if (pos[t] > pos[i] && pos[t] - first < nr)
- flow_sym_meet(syms + (size_t)(pos[t] - first) * n, cur, n);
- if (t == h || pos[t] > pos[i] || pos[t] - first >= nr)
- continue;
- /* jump back of an inner loop: what it adds moves the bounds of the root */
- for (j = 0; j < n; j++) {
- r = &roots[cur[j].root];
- if (r->has_max && cur[j].hi > 0)
- flow_add_max(r, r->max[1] + cur[j].hi);
- if (r->has_min && cur[j].lo < 0)
- flow_add_min(r, r->min[1] + cur[j].lo);
- }
- }
- }
-
- for (k = 0; k < back_cnt; k++) {
- for (j = 0; j < n; j++) {
- x = &back[(size_t)k * n + j];
- r = &roots[x->root];
- /* it is what it was */
- if (x->root == j && !x->lo && !x->hi)
- continue;
- if (r->has_max) {
- flow_add_max(&rungs[j], r->max[0] + x->hi);
- flow_add_max(&rungs[j], r->max[1] + x->hi);
- }
- if (r->has_min) {
- flow_add_min(&rungs[j], r->min[0] + x->lo);
- flow_add_min(&rungs[j], r->min[1] + x->lo);
- }
- }
- }
- for (j = 0; j < n; j++)
- cnt += rungs[j].has_max || rungs[j].has_min;
- err = 0;
- if (!cnt)
- goto out;
- loop->rungs = kvcalloc(cnt, sizeof(*loop->rungs), BPF_VERIFIER_GFP);
- if (!loop->rungs) {
- err = -ENOMEM;
- goto out;
- }
- for (p = 0; p < FLOW_PLACES; p++) {
- if (idx[p] == FLOW_NO_PLACE || (!rungs[idx[p]].has_max && !rungs[idx[p]].has_min))
- continue;
- loop->rungs[loop->rungs_cnt] = rungs[idx[p]];
- loop->rungs[loop->rungs_cnt++].place = p;
- }
-out:
- kvfree(syms);
- kvfree(roots);
- return err;
-}
-
-static int flow_compute_rungs(struct bpf_verifier_env *env)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- u32 total = env->cfg.cur_postorder, depth, k, *at;
- DECLARE_BITMAP(written, FLOW_PLACES);
- struct bpf_loop *loop;
- int h, i, err = 0;
-
- for (h = 0; h < env->prog->len && !aux[h].loop; h++)
- ;
- if (h == env->prog->len)
- return 0;
- at = kvmalloc_array(total, sizeof(*at), BPF_VERIFIER_GFP);
- if (!at)
- return -ENOMEM;
- for (k = 0; k < total; k++) {
- i = env->cfg.insn_postorder[k];
- at[env->flow->pos[i]] = i;
- flow_written_places(&env->prog->insnsi[i], written);
- for (h = bpf_loop_at_index(env, i); h >= 0; h = aux[h].loop_header)
- bitmap_or(aux[h].loop->writes, aux[h].loop->writes, written, FLOW_PLACES);
- }
- for (h = 0; h < env->prog->len && !err; h++) {
- loop = aux[h].loop;
- /* the order says little about a loop with more than one entry */
- if (!loop || loop->irreducible || loop->backedges_overflow)
- continue;
- /* the main pass doesn't get that deep */
- depth = 0;
- for (i = aux[h].loop_header; i >= 0; i = aux[i].loop_header)
- depth++;
- if (depth >= LOOP_STACK_SIZE)
- continue;
- if (bpf_must_abort()) {
- err = -EINTR;
- break;
- }
- err = flow_compute_loop_rungs(env, at, total, h);
- }
- kvfree(at);
- return err;
-}
-
-int bpf_flow_prepare(struct bpf_verifier_env *env)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_insn *insns = env->prog->insnsi;
- /* the rest is for loops */
- int len = env->prog->len, kinds = FLOW_NEED_VALUE_OFF, i, s, err;
- struct bpf_iarray *succ;
- struct bpf_flow *flow;
- u8 *preds;
-
- flow = kzalloc_obj(*flow, BPF_VERIFIER_GFP);
- if (!flow)
- return -ENOMEM;
- env->flow = flow;
- INIT_LIST_HEAD(&flow->kept);
- flow->pending = RB_ROOT_CACHED;
-
- flow->pos = kvcalloc(len, sizeof(u32), BPF_VERIFIER_GFP);
- flow->rank = kvcalloc(env->subprog_cnt + 1, sizeof(u32), BPF_VERIFIER_GFP);
- flow->point = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
- flow->keep = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
- flow->states = kvmalloc_array(len, sizeof(*flow->states), BPF_VERIFIER_GFP);
- flow->enter = kvmalloc_array(len, sizeof(*flow->enter), BPF_VERIFIER_GFP);
- flow->nr_states = kvcalloc(len, sizeof(u32), BPF_VERIFIER_GFP);
- flow->funcs = kvcalloc(env->subprog_cnt + 1, sizeof(*flow->funcs), BPF_VERIFIER_GFP);
- preds = kvcalloc(len, 1, BPF_VERIFIER_GFP);
- if (!flow->pos || !flow->rank || !flow->point || !flow->keep || !flow->states ||
- !flow->enter ||
- !flow->nr_states || !flow->funcs || !preds) {
- kvfree(preds);
- return -ENOMEM;
- }
- for (i = 0; i < len; i++) {
- INIT_LIST_HEAD(&flow->states[i].list);
- INIT_LIST_HEAD(&flow->enter[i].list);
- }
- for (i = 0; i <= env->subprog_cnt; i++)
- flow->funcs[i].exit_idx = -1;
- /* from here on there is something for bpf_flow_reset() to look at */
- flow->len = len;
- flow->nr_funcs = env->subprog_cnt + 1;
-
- /* subprog_topo_order[] has callees first */
- for (i = 0; i < env->subprog_cnt; i++)
- flow->rank[env->subprog_topo_order[i]] = env->subprog_cnt - i;
-
- for (i = 0; i < len; i++) {
- succ = bpf_insn_successors(env, i);
- iarray_for_each(s, succ) {
- if (preds[s] < 2)
- preds[s]++;
- if (aux[i].bb_end)
- __set_bit(s, flow->point);
- }
- /* where the callees come back to */
- if (bpf_pseudo_call(&insns[i]) || bpf_is_callx(&insns[i])) {
- if (i + 1 < len) {
- __set_bit(i + 1, flow->point);
- __set_bit(i + 1, flow->keep);
- }
- }
- if (aux[i].calls_callback) {
- __set_bit(i, flow->point);
- __set_bit(i, flow->keep);
- }
- if (aux[i].loop) {
- __set_bit(i, flow->point);
- __set_bit(i, flow->keep);
- kinds = FLOW_NEED_KINDS;
- }
- if (bpf_is_ldimm64(&insns[i]))
- i++;
- }
- for (i = 0; i < len; i++)
- if (preds[i] > 1)
- __set_bit(i, flow->keep);
- for (i = 0; i < env->subprog_cnt; i++) {
- __set_bit(env->subprog_info[i].start, flow->point);
- __set_bit(env->subprog_info[i].start, flow->keep);
- }
- kvfree(preds);
-
- err = flow_compute_order(env, flow);
- if (err)
- return err;
- for (i = 0; !err && i < kinds; i++)
- err = flow_compute_need_const(env, &flow->need[i], i);
- err = err ?: flow_compute_rungs(env);
- if (err)
- return err;
- return 0;
-}
-
-static void flow_free_state(struct bpf_verifier_env *env, struct flow_state *s)
-{
- bpf_free_verifier_state(&s->st, false);
- kfree(s);
- env->flow->live_states--;
-}
-
-/* What a pass leaves behind. */
-void bpf_flow_reset(struct bpf_verifier_env *env)
-{
- struct bpf_flow *flow = env->flow;
- struct flow_state *s, *sn;
- struct flow_kept *k, *kn;
- struct flow_exit *x, *xn;
- struct flow_entry *e;
- int i;
-
- if (!flow || !flow->states || !flow->enter)
- return;
- for (i = 0; i < flow->len; i++) {
- list_for_each_entry_safe(s, sn, &flow->states[i].list, node) {
- list_del(&s->node);
- flow_free_state(env, s);
- }
- list_for_each_entry_safe(s, sn, &flow->enter[i].list, node) {
- list_del(&s->node);
- flow_free_state(env, s);
- }
- flow->nr_states[i] = 0;
- }
- flow->pending = RB_ROOT_CACHED;
- for (i = 0; i < flow->nr_funcs && flow->funcs; i++)
- flow->funcs[i].exit_idx = -1;
- for (i = 1; i <= flow->entry_cnt; i++) {
- e = flow->entries[i];
- list_for_each_entry_safe(s, sn, &e->calls, node) {
- bpf_free_verifier_state(&s->st, false);
- kfree(s);
- }
- list_for_each_entry_safe(x, xn, &e->exits, node) {
- bpf_free_verifier_state(&x->st, false);
- kfree(x);
- }
- kfree(e);
- }
- list_for_each_entry_safe(k, kn, &flow->kept, node) {
- bpf_free_verifier_state(&k->st, false);
- kfree(k);
- }
- INIT_LIST_HEAD(&flow->kept);
- bpf_free_verifier_state(&flow->tmp, false);
- memset(&flow->tmp, 0, sizeof(flow->tmp));
- flow->entry_cnt = 0;
- flow->united_cnt = 0;
-}
-
-void bpf_flow_free(struct bpf_verifier_env *env)
-{
- struct bpf_flow *flow = env->flow;
- int i;
-
- if (!flow)
- return;
- bpf_flow_reset(env);
- kvfree(flow->pos);
- kvfree(flow->rank);
- for (i = 0; i < FLOW_NEED_KINDS; i++)
- kvfree(flow->need[i]);
- kvfree(flow->point);
- kvfree(flow->keep);
- kvfree(flow->states);
- kvfree(flow->enter);
- kvfree(flow->nr_states);
- kvfree(flow->funcs);
- kvfree(flow->entries);
- kvfree(flow->united);
- kfree(flow);
- env->flow = NULL;
-}
-
-bool bpf_flow_is_point(struct bpf_verifier_env *env, int insn_idx)
-{
- return test_bit(insn_idx, env->flow->point);
-}
-
-/* A copy of @st that lives as long as the pass. */
-struct bpf_verifier_state *bpf_flow_keep_state(struct bpf_verifier_env *env,
- const struct bpf_verifier_state *st)
-{
- struct flow_kept *k = kzalloc_obj(*k, BPF_VERIFIER_GFP);
-
- if (!k)
- return NULL;
- list_add(&k->node, &env->flow->kept);
- if (bpf_copy_verifier_state(&k->st, st))
- return NULL;
- return &k->st;
-}
-
-/*
- * Both are in a loop whose iterations are kept apart: an iteration is done with before
- * the next one begins. A back edge in the middle of the loop would run ahead otherwise and
- * leave the rest of every iteration behind, see flow_earlier_trip(). The state that stands
- * for all iterations comes last.
- */
-static int flow_cmp_trips(const struct bpf_func_state *fa, const struct bpf_func_state *fb)
-{
- const struct loop_stack_entry *ea, *eb;
- u32 i, ta, tb;
-
- for (i = 0; i < fa->loop_stack_cnt && i < fb->loop_stack_cnt; i++) {
- ea = &fa->loop_stack[i];
- eb = &fb->loop_stack[i];
- if (ea->loop_id != eb->loop_id)
- break;
- ta = ea->apart ? ea->trip : U32_MAX;
- tb = eb->apart ? eb->trip : U32_MAX;
- if (ta != tb)
- return ta < tb ? -1 : 1;
- }
- return 0;
-}
-
-static bool flow_less(struct rb_node *na, const struct rb_node *nb)
-{
- const struct flow_state *a = rb_entry(na, struct flow_state, rb);
- const struct flow_state *b = rb_entry(nb, struct flow_state, rb);
- int cmp;
- u32 i;
-
- for (i = 0; i < a->key_len && i < b->key_len; i++) {
- /* key[i] is the place in frame[i - 1] */
- if (i) {
- cmp = flow_cmp_trips(a->st.frame[i - 1], b->st.frame[i - 1]);
- if (cmp)
- return cmp < 0;
- }
- if (a->key[i] != b->key[i])
- return a->key[i] < b->key[i];
- }
- if (a->key_len != b->key_len)
- return a->key_len < b->key_len;
- return a->seq < b->seq;
-}
-
-/*
- * Callers first, so that the calls of a function get to it before it is looked
- * at. Then by the place of the call in every frame: what is called in the frames of
- * its callers is done with before what follows the call. What follows the call of
- * a function that is verified on its own goes on with every state that comes out.
- */
-static void flow_set_pending(struct bpf_flow *flow, struct flow_state *s)
-{
- const struct bpf_verifier_state *st = &s->st;
- u32 i;
-
- s->key[0] = flow->rank[st->frame[0]->subprogno];
- /* the place in the function makes the rank of the function redundant below */
- for (i = 0; i < st->curframe && i + 2 < ARRAY_SIZE(s->key); i++)
- s->key[i + 1] = flow->pos[st->frame[i + 1]->callsite];
- s->key[i + 1] = flow->pos[st->insn_idx];
- s->key_len = i + 2;
- s->pending = true;
- rb_add_cached(&s->rb, &flow->pending, flow_less);
-}
-
-static void flow_unlink(struct bpf_verifier_env *env, struct flow_state *s)
-{
- struct bpf_flow *flow = env->flow;
-
- if (s->pending)
- rb_erase_cached(&s->rb, &flow->pending);
- s->pending = false;
- list_del(&s->node);
- flow->nr_states[s->st.insn_idx]--;
-}
-
-/* The entry that stands for the set that entry @id is in */
-static struct flow_entry *flow_entry_find(struct bpf_flow *flow, u32 id)
-{
- struct flow_entry *e = flow->entries[id];
-
- while (e->up != e) {
- e->up = e->up->up;
- e = e->up;
- }
- return e;
-}
-
-static int flow_entry_new(struct bpf_flow *flow)
-{
- struct flow_entry **entries, *e;
- u32 cap;
-
- if (flow->entry_cnt + 1 >= flow->entry_cap) {
- cap = max(64U, flow->entry_cap * 2);
- entries = kvrealloc(flow->entries, cap * sizeof(*entries), BPF_VERIFIER_GFP);
- if (!entries)
- return -ENOMEM;
- flow->entries = entries;
- flow->entry_cap = cap;
- }
- e = kmalloc_obj(*e, BPF_VERIFIER_GFP);
- if (!e)
- return -ENOMEM;
- e->up = e;
- INIT_LIST_HEAD(&e->calls);
- INIT_LIST_HEAD(&e->exits);
- flow->entries[++flow->entry_cnt] = e;
- return flow->entry_cnt;
-}
-
-/* It delivers states, so it is done when nothing is in the middle of that. */
-static int flow_unite(struct bpf_flow *flow, u32 a, u32 b)
-{
- struct flow_united *u;
- u32 cap;
-
- if (!a || !b || flow_entry_find(flow, a) == flow_entry_find(flow, b))
- return 0;
- if (flow->united_cnt == flow->united_cap) {
- cap = max(16U, flow->united_cap * 2);
- u = kvrealloc(flow->united, cap * sizeof(*u), BPF_VERIFIER_GFP);
- if (!u)
- return -ENOMEM;
- flow->united = u;
- flow->united_cap = cap;
- }
- u = &flow->united[flow->united_cnt++];
- u->a = a;
- u->b = b;
- return 0;
-}
-
-/*
- * What is known about a counted loop comes from the state it was entered
- * with. @t stands for @s only if that holds for these states too.
- */
-static bool flow_loops_within(struct bpf_verifier_env *env, const struct bpf_verifier_state *t,
- const struct bpf_verifier_state *s)
-{
- const struct loop_stack_entry *lt, *ls;
- u32 fr, i;
-
- for (fr = 0; fr <= t->curframe; fr++) {
- if (t->frame[fr]->loop_stack_cnt != s->frame[fr]->loop_stack_cnt)
- return false;
- for (i = 0; i < t->frame[fr]->loop_stack_cnt; i++) {
- lt = &t->frame[fr]->loop_stack[i];
- ls = &s->frame[fr]->loop_stack[i];
- if (!lt->terminates || lt->entry_state == ls->entry_state)
- continue;
- if (!ls->terminates ||
- !bpf_state_subsumes(env, lt->entry_state, ls->entry_state))
- return false;
- }
- }
- return true;
-}
-
-static bool flow_within(struct bpf_verifier_env *env, struct bpf_verifier_state *t,
- struct bpf_verifier_state *s)
-{
- return t->curframe == s->curframe &&
- t->frame[0]->subprogno == s->frame[0]->subprogno &&
- /* an async callback returns to the kernel, a call of the function to its caller */
- !t->flow_entry == !s->flow_entry &&
- bpf_state_subsumes(env, t, s) && flow_loops_within(env, t, s);
-}
-
-/*
- * How many iterations is @t behind @s, in the loop in which that is the most? Only the loops
- * whose iterations are kept apart are looked at.
- * 0 if both are in the same iteration of every loop, U32_MAX if @t is ahead of @s in a loop.
- */
-static u32 flow_trips_behind(const struct bpf_verifier_state *t, const struct bpf_verifier_state *s)
-{
- const struct loop_stack_entry *lt, *ls;
- u32 fr, i, behind = 0;
-
- if (t->curframe != s->curframe)
- return U32_MAX;
- for (fr = 0; fr <= t->curframe; fr++) {
- if (t->frame[fr]->loop_stack_cnt != s->frame[fr]->loop_stack_cnt)
- return U32_MAX;
- for (i = 0; i < t->frame[fr]->loop_stack_cnt; i++) {
- lt = &t->frame[fr]->loop_stack[i];
- ls = &s->frame[fr]->loop_stack[i];
- if (!lt->apart && !ls->apart)
- continue;
- if (lt->trip > ls->trip)
- return U32_MAX;
- behind = max(behind, ls->trip - lt->trip);
- }
- }
- return behind;
-}
-
-/*
- * A state is compared with the states of its own iteration and of the two before it. The older ones
- * are gone, see flow_earlier_trip(), but for what is pending or waits: every iteration of a loop
- * calls a recursive function, and what comes out of it goes to all the calls at once.
- */
-static bool flow_trips_near(const struct bpf_verifier_state *t, const struct bpf_verifier_state *s)
-{
- return flow_trips_behind(t, s) <= 2;
-}
-
-/*
- * @s is within @t, and from now on @t stands for it.
- *
- * A walk on patience gets to the end of a loop when no state comes back to the head any more.
- * Then the loop ends at run time too: the number of the iteration grows at the head, nothing makes
- * it smaller, and there is a largest one.
- * If @t is of an earlier iteration than @s that is not true any more. What becomes of @t got to @s
- * and is @t again: as far as the verifier can see the program goes around for ever. A spin lock
- * is such a loop. So the loop gets may_goto.
- *
- * It's asked for every loop that @t is behind in, not only for the outermost one. This is the only
- * way for the number of a loop to get smaller. In a walk it doesn't change in another way but at
- * the head: a state never stands for a state of an earlier iteration, see loop_stack_safe(), and it
- * takes the place of, or is joined with, states of its own iterations only, see flow_merge().
- * Where the iterations are not kept apart a join has the larger number, and a state may stand
- * for one with a smaller number. Such loop has a count or may_goto, or something stops it, since
- * the state entered it.
- */
-static int flow_cycle_closed(struct bpf_verifier_env *env, const struct bpf_verifier_state *t,
- const struct bpf_verifier_state *s)
-{
- const struct loop_stack_entry *lt, *ls;
- struct bpf_loop *loop;
- u32 fr, i;
-
- for (fr = 0; fr <= t->curframe; fr++) {
- for (i = 0; i < t->frame[fr]->loop_stack_cnt; i++) {
- lt = &t->frame[fr]->loop_stack[i];
- ls = &s->frame[fr]->loop_stack[i];
- if (lt->trip >= ls->trip)
- continue;
- /* counted: what comes back is expected to be within what is at the head */
- if (lt->terminates && !bpf_loop_walk_may_give_up(lt))
- continue;
- loop = env->insn_aux_data[lt->loop_id].loop;
- /* it goes around for as long as the iterator has elements or may_goto budget */
- if (loop->rt_bounded)
- continue;
- if (!loop->has_guards) {
- verbose(env, "loop at insn %d never exits\n", lt->loop_id);
- return -EINVAL;
- }
- bpf_flow_loop_gets_guards(env, lt->loop_id);
- }
- }
- return 0;
-}
-
-/*
- * @t is of an earlier iteration of a loop whose iterations are kept apart than @s.
- * What comes after @s is of later iterations too, so @t is not going to be
- * of use. If something of its iteration does come it is looked at again.
- *
- * Not so where a state stands for many iterations. States of different kinds are at the head of
- * such loop. One of them comes to rest early, and what comes around is within it for as long as
- * the others take.
- */
-static bool flow_earlier_trip(const struct bpf_verifier_state *t,
- const struct bpf_verifier_state *s)
-{
- const struct bpf_func_state *ft, *fs;
- u32 fr, i;
-
- if (t->curframe != s->curframe)
- return false;
- for (fr = 0; fr <= t->curframe; fr++) {
- ft = t->frame[fr];
- fs = s->frame[fr];
- if (ft->subprogno != fs->subprogno || ft->callsite != fs->callsite ||
- ft->loop_stack_cnt != fs->loop_stack_cnt)
- return false;
- for (i = 0; i < ft->loop_stack_cnt; i++) {
- if (ft->loop_stack[i].loop_id != fs->loop_stack[i].loop_id ||
- ft->loop_stack[i].apart != fs->loop_stack[i].apart)
- return false;
- if (ft->loop_stack[i].apart &&
- ft->loop_stack[i].trip != fs->loop_stack[i].trip)
- return ft->loop_stack[i].trip + 1 < fs->loop_stack[i].trip;
- }
- }
- return false;
-}
-
-/* Can flow_earlier_trip() be true for a state and @s? */
-static bool flow_has_earlier_trips(const struct bpf_verifier_state *s)
-{
- const struct loop_stack_entry *entry;
- u32 fr, i;
-
- for (fr = 0; fr <= s->curframe; fr++) {
- for (i = 0; i < s->frame[fr]->loop_stack_cnt; i++) {
- entry = &s->frame[fr]->loop_stack[i];
- if (entry->apart && entry->trip > 1)
- return true;
- }
- }
- return false;
-}
-
-/* @flag: 0 and 1 */
-static bool flow_two_consts(const struct bpf_reg_state *ra, const struct bpf_reg_state *rb,
- bool flag)
-{
- return ra->type == SCALAR_VALUE && rb->type == SCALAR_VALUE &&
- tnum_is_const(ra->var_off) && tnum_is_const(rb->var_off) &&
- ra->var_off.value != rb->var_off.value &&
- (!flag || (ra->var_off.value | rb->var_off.value) == 1);
-}
-
-/* Constants that have to stay constants, see flow_compute_need_const() */
-static bool flow_needed_consts_differ(const struct bpf_flow *flow, enum flow_need_kind kind,
- const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- const struct flow_need *needs = flow->need[kind], *need;
- const struct bpf_stack_state *sa, *sb;
- const struct bpf_func_state *fa, *fb;
- bool flag = kind == FLOW_NEED_FLAG;
- u32 fr, i, nr;
-
- if (!needs || a->curframe != b->curframe)
- return false;
- for (fr = 0; fr <= a->curframe; fr++) {
- /* the callers need what they need when the call returns */
- need = &needs[fr == a->curframe ? a->insn_idx : a->frame[fr + 1]->callsite + 1];
- fa = a->frame[fr];
- fb = b->frame[fr];
- for (i = BPF_REG_0; need->regs && i < BPF_REG_FP; i++)
- if ((need->regs & BIT(i)) &&
- flow_two_consts(&fa->regs[i], &fb->regs[i], flag))
- return true;
- nr = min(bpf_stack_nr_slots(fa), bpf_stack_nr_slots(fb));
- for (i = 0; i < nr && i < 64 && need->slots >> i; i++) {
- sa = bpf_stack_slot(fa, i);
- sb = bpf_stack_slot(fb, i);
- if ((need->slots & BIT_ULL(i)) &&
- bpf_is_spilled_scalar_reg(sa) && bpf_is_spilled_scalar_reg(sb) &&
- flow_two_consts(&sa->spilled_ptr, &sb->spilled_ptr, flag))
- return true;
- }
- }
- return false;
-}
-
-/* flow_shape_ptr() hashes a struct scalar up to its padding: there is no hole in front of it */
-static_assert(offsetofend(struct scalar, step) == sizeof(struct tnum) + sizeof(struct cnum64) +
- sizeof(struct cnum32) + 2 * sizeof(u16));
-
-/* regsafe() and join_reg() say what each type needs */
-static void flow_shape_ptr(struct flow_shape *sh, u32 place, const struct bpf_reg_state *reg)
-{
- u32 ids = (reg->id != 0) | (reg->parent_id != 0) << 1 | (reg->map_uid != 0) << 2;
- bool fixed_part = true, exact = false, has_off = false, var_off = false;
- u32 *ptrs = &sh->word[FLOW_PTRS];
- u64 off = reg->var_off.value;
-
- switch (base_type(reg->type)) {
- case NOT_INIT:
- case SCALAR_VALUE:
- return;
- case PTR_TO_ARENA:
- /* the offset is not tracked */
- fixed_part = false;
- ids = 0;
- break;
- case PTR_TO_PACKET_META:
- case PTR_TO_PACKET:
- fixed_part = false;
- ids &= 1;
- /* behind the end of the packet, or at the end: the same in both states */
- if (reg->range < 0)
- ids |= -reg->range << 1;
- else
- sh->pkt_range += reg->range;
- if (reg->id) {
- /* displacements between pointers that share an id are kept */
- has_off = true;
- off = reg->r64.base;
- } else if (reg->range >= 0) {
- /* join_reg() takes two such constants for two places in the packet */
- has_off = true;
- var_off = !tnum_is_const(reg->var_off) || off > MAX_PACKET_OFF;
- }
- break;
- case PTR_TO_INSN:
- /* a range of offsets is a set of targets */
- ids = 0;
- break;
- case PTR_TO_MAP_KEY:
- case PTR_TO_MAP_VALUE:
- case PTR_TO_MEM:
- case PTR_TO_BUF:
- case PTR_TO_TP_BUFFER:
- case PTR_TO_STACK:
- case PTR_TO_BTF_ID:
- has_off = true;
- var_off = !tnum_is_const(reg->var_off);
- break;
- default:
- exact = true;
- }
- sh->word[FLOW_LAYOUT] = jhash_1word(place << 1 | var_off, sh->word[FLOW_LAYOUT]);
- *ptrs = jhash_1word(reg->type, *ptrs + ids);
- if (fixed_part)
- *ptrs = jhash2((const u32 *)®->raw, sizeof(reg->raw) / sizeof(u32), *ptrs);
- if (exact)
- *ptrs = jhash2((const u32 *)scalar_of(reg),
- offsetofend(struct scalar, step) / sizeof(u32), *ptrs);
- if (has_off && !var_off)
- sh->word[FLOW_OFFS] = jhash_2words(lower_32_bits(off), upper_32_bits(off),
- sh->word[FLOW_OFFS]);
-}
-
-/* stacksafe() and join_special_slot() say what each kind needs */
-static void flow_shape_special(struct flow_shape *sh, u32 place, const struct bpf_stack_state *slot)
-{
- const struct bpf_reg_state *reg = &slot->spilled_ptr;
- u32 kind = slot->slot_type[BPF_REG_SIZE - 1] << 2 | (reg->id != 0);
- u32 *ptrs = &sh->word[FLOW_PTRS];
-
- switch (slot->slot_type[BPF_REG_SIZE - 1]) {
- case STACK_DYNPTR:
- kind |= (reg->parent_id != 0) << 1;
- *ptrs = jhash_3words(kind, reg->dynptr.type, reg->dynptr.first_slot, *ptrs);
- break;
- case STACK_ITER:
- *ptrs = jhash_3words(lower_32_bits((unsigned long)reg->iter.btf),
- upper_32_bits((unsigned long)reg->iter.btf),
- reg->iter.btf_id, *ptrs + kind);
- *ptrs = jhash_3words(reg->type, reg->iter.state, reg->iter.loops, *ptrs);
- break;
- case STACK_IRQ_FLAG:
- *ptrs = jhash_2words(kind, reg->irq.kfunc_class, *ptrs);
- break;
- }
- sh->word[FLOW_LAYOUT] = jhash_1word(place << 1, sh->word[FLOW_LAYOUT]);
-}
-
-/* The part of the shape that is made of the registers and the stack, when it is asked for */
-static void flow_shape_places(struct bpf_verifier_env *env, struct flow_state *s)
-{
- struct bpf_verifier_state *st = &s->st;
- const struct bpf_stack_state *slot;
- const struct bpf_func_state *frame;
- struct flow_shape *sh = &s->shape;
- u32 fr, i, args;
- u16 live;
-
- sh->places = true;
- for (fr = 0; fr <= st->curframe; fr++) {
- frame = st->frame[fr];
- /* a state that does a call again is not cleaned, dead registers are not compared */
- live = env->insn_aux_data[bpf_frame_insn_idx(st, fr)].live_regs_before;
- for (i = BPF_REG_0; i < BPF_REG_FP; i++)
- if (live & BIT(i))
- flow_shape_ptr(sh, fr << 10 | i, &frame->regs[i]);
- for (i = 0; i < bpf_stack_nr_slots(frame); i++) {
- slot = bpf_stack_slot(frame, i);
- switch (slot->slot_type[BPF_REG_SIZE - 1]) {
- case STACK_SPILL:
- flow_shape_ptr(sh, fr << 10 | (i + 16), &slot->spilled_ptr);
- break;
- case STACK_DYNPTR:
- case STACK_ITER:
- case STACK_IRQ_FLAG:
- flow_shape_special(sh, fr << 10 | (i + 16), slot);
- break;
- }
- }
- /* incoming arguments are not compared with outgoing ones: other places */
- args = 16 + MAX_BPF_STACK_SLOTS;
- if (bpf_stack_args_are_incoming(frame))
- args += MAX_STACK_ARG_SLOTS;
- for (i = 0; i < frame->out_stack_arg_cnt; i++)
- flow_shape_ptr(sh, fr << 10 | (i + args), &frame->stack_arg_regs[i]);
- }
-}
-
-/* The part of the shape that is not */
-static void flow_shape_ctx(const struct bpf_verifier_state *st, struct flow_shape *sh)
-{
- const struct bpf_reference_state *ref;
- const struct loop_stack_entry *entry;
- const struct bpf_func_state *frame;
- u32 chain = st->curframe, ctx, fr, i;
-
- ctx = jhash_3words(st->in_sleepable | st->callers_unknown << 1 |
- st->unwound << 2 | st->changed_pkt_data << 3 |
- (st->active_irq_id != 0) << 4 | (st->active_lock_id != 0) << 5,
- st->acquired_refs, st->active_locks, 0);
- ctx = jhash_2words(st->active_preempt_locks, st->active_rcu_locks, ctx);
- ctx = jhash_2words(lower_32_bits((unsigned long)st->active_lock_ptr),
- upper_32_bits((unsigned long)st->active_lock_ptr), ctx);
- for (i = 0; i < st->acquired_refs; i++) {
- ref = &st->refs[i];
- ctx = jhash_1word(ref->type, ctx);
- if (ref->type != REF_TYPE_PTR && ref->type != REF_TYPE_IRQ)
- ctx = jhash_2words(lower_32_bits((unsigned long)ref->ptr),
- upper_32_bits((unsigned long)ref->ptr), ctx);
- }
- for (fr = 0; fr <= st->curframe; fr++) {
- frame = st->frame[fr];
- chain = jhash_3words(frame->subprogno, frame->callsite, frame->loop_stack_cnt,
- chain);
- ctx = jhash_2words(frame->entry_refs, frame->in_rcu_callback_fn, ctx);
- ctx = jhash_2words(frame->entry_rcu_locks, frame->entry_preempt_locks, ctx);
- for (i = 0; i < frame->loop_stack_cnt; i++) {
- entry = &frame->loop_stack[i];
- chain = jhash_1word(entry->loop_id << 1 | entry->apart, chain);
- ctx = jhash_3words(entry->counted << 3 | entry->guarded << 2 |
- entry->first << 1 | entry->terminates,
- entry->max_header_count,
- entry->counted ? entry->trip : 0, ctx);
- }
- }
- memset(sh, 0, sizeof(*sh));
- sh->word[FLOW_CHAIN] = chain;
- sh->word[FLOW_CTX] = ctx;
-}
-
-/* Are the words up to @last the same? */
-static __always_inline bool flow_same_shape(const struct flow_state *a, const struct flow_state *b,
- enum flow_word last)
-{
- u32 i;
-
- for (i = 0; i <= last; i++)
- if (a->shape.word[i] != b->shape.word[i])
- return false;
- return true;
-}
-
-/*
- * A state that arrives is compared with the states at the insn: is it within one of them, see
- * bpf_state_subsumes(), does it have a join with one of them, see bpf_states_join()? Most of
- * them have nothing to do with it. The shape of a state tells that before the registers are
- * compared or, for a join, copied. It has what must be the same in two states that are related:
- *
- * FLOW_CHAIN the chain of calls and the loops that every frame is in
- * FLOW_CTX what is known about every loop, the iteration of a counted one,
- * the flags of the state and of its frames, locks and references
- * FLOW_LAYOUT which registers, stack slots and stack arguments hold a pointer, a dynptr,
- * an iterator or an irq flag, which of the pointers are at a constant offset
- * FLOW_PTRS what the pointers point to and which of their ids are set, the kind of the others
- * FLOW_OFFS the constant offsets
- *
- * Nothing about numbers is in it.
- *
- * States that differ in one of the first two words are not related. Neither are states of
- * the same layout that differ in one of the last two: a pointer is related to a pointer to the same
- * thing only, and two constant offsets are two places in it.
- * States of different layout may be related, whatever the last two words are: nothing in
- * a register or in a slot stands for a pointer there, and a range of offsets stands for a constant,
- * or is joined with it.
- *
- * A word is a hash. It is a filter and proves nothing: the states that get through it go
- * through all the comparisons. So it's fine to look at the first two words only: @places.
- */
-static bool flow_may_relate(struct bpf_verifier_env *env, struct flow_state *t,
- struct flow_state *s, bool places)
-{
- if (!flow_same_shape(t, s, FLOW_CTX))
- return false;
- if (!places)
- return true;
- if (!t->shape.places)
- flow_shape_places(env, t);
- if (!s->shape.places)
- flow_shape_places(env, s);
- return !flow_same_shape(t, s, FLOW_LAYOUT) || flow_same_shape(t, s, FLOW_OFFS);
-}
-
-/*
- * A pointer to the packet stands for one with at least as many checked bytes, and without
- * widening it's joined with one that has as many. In states of the same layout that holds for
- * the sums too. Returns 0 if the sums are the same, or say nothing.
- */
-static int flow_cmp_pkt_range(const struct flow_state *t, const struct flow_state *s)
-{
- if (!t->shape.places || !s->shape.places || !flow_same_shape(t, s, FLOW_LAYOUT) ||
- t->shape.pkt_range == s->shape.pkt_range)
- return 0;
- return t->shape.pkt_range < s->shape.pkt_range ? -1 : 1;
-}
-
-/*
- * bpf_loop(N, ...) with a small N: the calls of the callback are looked at one by one, like
- * the iterations of a loop that is walked. The call stops to enter the callback when
- * callback_depth gets to N.
- */
-static bool flow_callback_depth_differs(struct bpf_verifier_env *env,
- const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- u32 max = BPF_FLOW_MAX_CALLBACKS, fr;
- const struct bpf_func_state *fa, *fb;
- struct bpf_insn *insn;
-
- if (a->curframe != b->curframe)
- return false;
- /* at the call, and in the callback and what it calls */
- for (fr = 0; fr <= a->curframe; fr++) {
- insn = &env->prog->insnsi[fr == a->curframe ? a->insn_idx :
- a->frame[fr + 1]->callsite];
- if (!bpf_helper_call(insn) || insn->imm != BPF_FUNC_loop)
- continue;
- fa = a->frame[fr];
- fb = b->frame[fr];
- if (fa->callback_depth != fb->callback_depth &&
- fa->regs[BPF_REG_1].type == SCALAR_VALUE &&
- reg_umax(&fa->regs[BPF_REG_1]) <= max &&
- fb->regs[BPF_REG_1].type == SCALAR_VALUE &&
- reg_umax(&fb->regs[BPF_REG_1]) <= max)
- return true;
- }
- return false;
-}
-
-static const struct bpf_reg_state *flow_slot_number(const struct bpf_func_state *frame, u32 i)
-{
- const struct bpf_stack_state *slot = bpf_stack_slot(frame, i);
-
- if (slot->slot_type[0] != STACK_SPILL || slot->slot_type[BPF_REG_SIZE - 1] != STACK_SPILL ||
- slot->spilled_ptr.type != SCALAR_VALUE)
- return NULL;
- return &slot->spilled_ptr;
-}
-
-/* Returns true when the answer is known */
-static bool flow_null_pair(const struct bpf_reg_state *ra, const struct bpf_reg_state *rb,
- bool *a_null, bool *b_null)
-{
- if (tnum_equals_const(ra->var_off, 0) && !tnum_is_const(rb->var_off)) {
- *a_null = true;
- if (reg_umin(rb) == 1 && reg_umax(rb) == U64_MAX)
- return true;
- }
- if (tnum_equals_const(rb->var_off, 0) && !tnum_is_const(ra->var_off)) {
- *b_null = true;
- if (reg_umin(ra) == 1 && reg_umax(ra) == U64_MAX)
- return true;
- }
- return *a_null && *b_null;
-}
-
-/*
- * Two things of which one is there: "if (hdr->ipv4) ... else if (hdr->ipv6)". The one that
- * is not there is 0. What the other one is may be unknown, e.g. a pointer to the packet
- * after the packet was changed. (0, x) is not within (y, 0) and the join (?, ?) forgets that
- * it's one or the other.
- * The same for a number that was compared with 0: 0 on one path, not 0 on the other.
- *
- * Only where a jump ahead can see it: FLOW_NEED_JUMP, see flow_join_makes_up(). What no jump
- * looks at again sends no state another way, and every such number doubles the states.
- *
- * for (i = 0; i < n; i++) { p = a[i]; if (!p) continue; put(p); }
- * a[] is on the stack. The jump finds out about the slot that p was filled from too. Nothing
- * looks at a[i] again, but the loop reads through a pointer that moves, so all of a[] is live
- * until the loop is done: 2^n states if they stay apart.
- * They don't, because FLOW_NEED_JUMP has only what is filled through r10, see flow_fp_slot().
- * It cannot tell a[i] from a[i + 1]. With what is filled through other pointers to the stack in
- * it, all of a[] is tested in all of the loop.
- */
-static bool flow_null_differs(const struct bpf_flow *flow, const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- const struct bpf_func_state *fa, *fb;
- const struct bpf_reg_state *ra, *rb;
- const struct flow_need *need = flow->need[FLOW_NEED_JUMP];
- bool a_null = false, b_null = false;
- u32 i;
-
- if (!need || a->curframe != b->curframe)
- return false;
- need += a->insn_idx;
- fa = a->frame[a->curframe];
- fb = b->frame[b->curframe];
- for (i = BPF_REG_0; i < BPF_REG_FP; i++) {
- ra = &fa->regs[i];
- rb = &fb->regs[i];
- if ((need->regs & BIT(i)) &&
- ra->type == SCALAR_VALUE && rb->type == SCALAR_VALUE &&
- flow_null_pair(ra, rb, &a_null, &b_null))
- return true;
- }
- for (i = 0; i < 64 && i < bpf_stack_nr_slots(fa) && i < bpf_stack_nr_slots(fb); i++) {
- if (!(need->slots & BIT_ULL(i)))
- continue;
- ra = flow_slot_number(fa, i);
- rb = flow_slot_number(fb, i);
- if (ra && rb && flow_null_pair(ra, rb, &a_null, &b_null))
- return true;
- }
- return false;
-}
-
-static const struct bpf_reg_state *flow_number_at(const struct bpf_func_state *frame, u8 loc)
-{
- const struct bpf_reg_state *reg;
-
- if (loc <= BPF_REG_FP)
- reg = &frame->regs[loc];
- else if (loc - BPF_REG_FP - 1 < bpf_stack_nr_slots(frame) &&
- bpf_is_spilled_scalar_reg(bpf_stack_slot(frame, loc - BPF_REG_FP - 1)))
- reg = &bpf_stack_slot(frame, loc - BPF_REG_FP - 1)->spilled_ptr;
- else
- return NULL;
- return reg->type == SCALAR_VALUE ? reg : NULL;
-}
-
-/*
- * A register is set in one of the states and not in the other. Only live registers are in
- * a state, so something ahead may read it. The state in which it's not set doesn't get there:
- * a jump sends it another way. The join would not know which way to go, and would get
- * there without the register. There is nothing to gain from such a join but a state less.
- */
-static bool flow_unset_differs(const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- u32 fr, i;
-
- for (fr = 0; fr <= a->curframe; fr++)
- for (i = BPF_REG_0; i < BPF_REG_FP; i++)
- if ((a->frame[fr]->regs[i].type == NOT_INIT) !=
- (b->frame[fr]->regs[i].type == NOT_INIT))
- return true;
- return false;
-}
-
-static bool flow_disjoint(const struct bpf_reg_state *a, const struct bpf_reg_state *b)
-{
- return cnum64_is_empty(cnum64_intersect(a->r64, b->r64)) ||
- cnum32_is_empty(cnum32_intersect(a->r32, b->r32)) ||
- ((a->var_off.value ^ b->var_off.value) & ~(a->var_off.mask | b->var_off.mask));
-}
-
-/* The smallest range that has both has numbers that are in neither */
-static bool flow_gap(const struct bpf_reg_state *a, const struct bpf_reg_state *b)
-{
- if (cnum64_is_empty(cnum64_intersect(a->r64, b->r64)) &&
- a->r64.size + b->r64.size + 1 > a->r64.size &&
- cnum64_union(a->r64, b->r64).size > a->r64.size + b->r64.size + 1)
- return true;
- return cnum32_is_empty(cnum32_intersect(a->r32, b->r32)) &&
- cnum32_union(a->r32, b->r32).size > (u64)a->r32.size + b->r32.size + 1;
-}
-
-/*
- * Numbers always have a join. It must not make up what neither state has where a jump ahead
- * can see it. FLOW_NEED_JUMP has the registers and the stack slots that a jump ahead compares
- * with a constant, or the verifier does at a call or at the exit.
- *
- * if (ret < 0) goto err; if (ret > 0) goto skip; use(p);
- * ret = [-2, -1] and ret = [1, 2], both without p, meet in front of that. [-2, 2] has 0, gets
- * through both jumps, and there is no p. So two ranges that leave a gap are not joined. Ranges
- * that touch, 0 and 1, -1 and 0, or have something in common make up no number.
- *
- * err = update(); ret = err < 0 ? DROP : 0; ... if (err < 0) goto out; ... if (ret < 0) ...
- * (err >= 0, ret = 0) and (err < 0, ret = DROP) meet. The join has err < 0 with ret = 0. So
- * two numbers, each with nothing in common between the states, are not joined either.
- *
- * fatal = err == -ENOMEM; ... if (fatal) bpf_set_retval(err);
- * (fatal = 0, err = 1) and (fatal = 1, err = -ENOMEM) meet. The same: the join has
- * fatal = 1 with err = 1, which bpf_set_retval() doesn't take.
- *
- * ok = i < 64 && in[i]; if (x) n++; if (ok) a[i] = 1;
- * (ok = 0, i >= 64), (ok = 0, i < 64) and (ok = 1, i < 64) meet. 0 and 1 touch, and i < 64 comes
- * with either, so none of the above keeps them apart. The join has ok = 1 with i >= 64. A flag is
- * made to send states different ways. So 0 and 1 are not joined where a jump that is not far
- * tells one from the other: FLOW_NEED_FLAG.
- * while (more) { ...; more = next(&x); }
- * That jump can be the one that leaves a loop. It is decided in both states, one stays and one
- * leaves. It's the jump that set the flag, which went both ways, once more: no sign that the
- * loop gets anywhere. A walk on patience has to know, see mark_loop_exit_decided().
- *
- * The states go on to the jumps on their own. Behind the last one the numbers are dead, and what
- * is left is joined.
- */
-static bool flow_join_makes_up(const struct bpf_flow *flow, struct bpf_verifier_state *a,
- struct bpf_verifier_state *b)
-{
- struct bpf_func_state *fa = a->frame[a->curframe], *fb = b->frame[b->curframe];
- const struct bpf_reg_state *ra, *rb, *one_a = NULL, *one_b = NULL;
- const struct flow_need *tested = flow->need[FLOW_NEED_JUMP], *need;
- u32 fr, i;
-
- if (!tested)
- return false;
- for (fr = 0; fr <= a->curframe; fr++) {
- /* the callers go on when the call returns */
- need = &tested[fr == a->curframe ? a->insn_idx : a->frame[fr + 1]->callsite + 1];
- for (i = BPF_REG_0; i < BPF_REG_FP + 1 + 64; i++) {
- if (i <= BPF_REG_FP ? !(need->regs & BIT(i)) :
- !(need->slots & BIT_ULL(i - BPF_REG_FP - 1)))
- continue;
- ra = flow_number_at(a->frame[fr], i);
- rb = flow_number_at(b->frame[fr], i);
- if (!ra || !rb || !flow_disjoint(ra, rb))
- continue;
- if (flow_gap(ra, rb))
- return true;
- /* copies of a scalar count once: same range and tnum in each state */
- if (one_a && (!reg_value_equal(ra, one_a) || !reg_value_equal(rb, one_b)))
- return true;
- one_a = ra;
- one_b = rb;
- }
- }
- if (!flow_needed_consts_differ(flow, FLOW_NEED_FLAG, a, b))
- return false;
- for (i = 0; i < fa->loop_stack_cnt; i++)
- fa->loop_stack[i].flag_apart = fa->loop_stack[i].probe;
- for (i = 0; i < fb->loop_stack_cnt; i++)
- fb->loop_stack[i].flag_apart = fb->loop_stack[i].probe;
- return true;
-}
-
-/*
- * @s came back to the head of a loop, or from a callback to the helper that calls it. @t is there.
- * Do they come from two states that entered the cycle?
- */
-static bool flow_entries_differ(const struct bpf_verifier_state *t,
- const struct bpf_verifier_state *s, enum flow_cycle cycle)
-{
- const struct bpf_func_state *ft = t->frame[t->curframe], *fs = s->frame[s->curframe];
- u32 cnt = fs->loop_stack_cnt;
-
- if (cycle == FLOW_CALLBACK_CYCLE)
- return ft->callback_entry_id != fs->callback_entry_id;
- return cnt && ft->loop_stack_cnt == cnt &&
- ft->loop_stack[cnt - 1].entry_id != fs->loop_stack[cnt - 1].entry_id;
-}
-
-/*
- * States that have a join are kept apart while there is room: half of every limit.
- * After that they are joined, which may lose what the prog needs, and does not run into the limit.
- */
-static bool flow_has_room(const struct bpf_verifier_env *env, u32 nr_states)
-{
- u64 len = env->prog->len;
-
- return nr_states < BPF_FLOW_MAX_STATES / 2 &&
- env->flow->live_states < (BPF_FLOW_LIVE_PER_INSN * len + BPF_FLOW_LIVE_BASE) / 2 &&
- env->total_states < (BPF_FLOW_TOTAL_PER_INSN * len + BPF_FLOW_TOTAL_BASE) / 2;
-}
-
-/* The state of @set behind @t, the first one if @t is NULL, that @s may be related to */
-static struct flow_state *flow_next_related(struct bpf_verifier_env *env,
- const struct flow_set *set, struct flow_state *s,
- struct flow_state *t, bool places)
-{
- t = t ? list_next_entry(t, node) : list_first_entry(&set->list, struct flow_state, node);
- list_for_each_entry_from(t, &set->list, node)
- if (flow_may_relate(env, t, s, places))
- return t;
- return NULL;
-}
-
-static void flow_link(struct bpf_verifier_env *env, struct flow_set *set, struct flow_state *s)
-{
- list_add_tail(&s->node, &set->list);
- env->flow->nr_states[s->st.insn_idx]++;
-}
-
-static void flow_sweep_trip(struct bpf_verifier_env *env, struct flow_state *t,
- const struct flow_state *s)
-{
- if (t->pending || !flow_earlier_trip(&t->st, &s->st))
- return;
- flow_unlink(env, t);
- flow_free_state(env, t);
-}
-
-/* States of earlier iterations than @s go away, see flow_earlier_trip() */
-static void flow_sweep_trips(struct bpf_verifier_env *env, struct flow_set *set,
- const struct flow_state *s)
-{
- struct flow_state *t, *tn;
-
- if (!flow_has_earlier_trips(&s->st))
- return;
- list_for_each_entry_safe(t, tn, &set->list, node)
- if (flow_same_shape(t, s, FLOW_CHAIN))
- flow_sweep_trip(env, t, s);
-}
-
-/*
- * Put *@ps into @set, one of the sets of states at its insn. Returns
- * 1 if it is within a state of the set. It is gone then.
- * 0 if it is in the set now, as it was or joined with states of the set.
- * @cycle: it came around a cycle, what grew is forgotten.
- */
-static int flow_merge(struct bpf_verifier_env *env, struct flow_set *set,
- struct flow_state **ps, enum flow_cycle cycle)
-{
- struct bpf_flow *flow = env->flow;
- struct flow_state *s = *ps, *t, *tn;
- int insn_idx = s->st.insn_idx, err;
- u32 *nr_states = &flow->nr_states[insn_idx];
- bool places, widen_all, widen;
- /* the set of the states that enter a loop is a stop on the way, see bpf_flow_deliver() */
- bool log = (env->log.level & BPF_LOG_LEVEL2) && set == &flow->states[insn_idx];
- const char *what = NULL;
- int prev = s->prev_insn_idx;
-
-again:
- if (bpf_must_abort()) {
- err = -EINTR;
- goto err_free;
- }
-
- /* again after a join: an offset may not be a constant any more, a pointer may be gone */
- flow_shape_ctx(&s->st, &s->shape);
- places = *nr_states >= BPF_FLOW_MIN_SHAPES;
- for (t = flow_next_related(env, set, s, NULL, places); t;
- t = flow_next_related(env, set, s, t, places)) {
- if (flow_cmp_pkt_range(t, s) > 0 || !flow_trips_near(&t->st, &s->st) ||
- !flow_within(env, &t->st, &s->st))
- continue;
- if (env->log.level & BPF_LOG_LEVEL2)
- verbose(env, "from %d to %d: within\n", prev, insn_idx);
- err = flow_cycle_closed(env, &t->st, &s->st) ?:
- flow_unite(flow, s->st.flow_entry, t->st.flow_entry) ?: 1;
- goto err_free;
- }
-
- flow_sweep_trips(env, set, s);
-
- widen_all = cycle == FLOW_CYCLE;
- for (t = flow_next_related(env, set, s, NULL, places); t; t = tn) {
- tn = flow_next_related(env, set, s, t, places);
- /*
- * Iterations that are kept apart have no join. Neither does a state take the place
- * of a state of another iteration that it covers: that one is done with, or has its
- * own way to go, see flow_cycle_closed().
- * The shape is a hash, and what follows goes through the frames of both.
- */
- if (flow_trips_behind(&t->st, &s->st))
- continue;
- widen = widen_all || (cycle == FLOW_CALLBACK_CYCLE &&
- max(s->joins, t->joins) >= BPF_FLOW_CALLBACK_JOINS);
- /*
- * ret = 0; if (...) { denied++; ret = -EPERM; }
- * bpf_for(i, 0, n) { ... } return ret;
- * Two states enter the loop. What comes around it with 0 is for the state with 0.
- * Next to -EPERM it looks like a number that grew, and would be forgotten. The same
- * for the header of a packet at pkt + 34 and at pkt + 54. No rule has to tell which
- * numbers that holds for: a state knows where it comes from.
- * If it has no join with that state it is at the head on its own from now on.
- * The same for bpf_loop() in place of the loop.
- */
- if (widen && flow_has_room(env, *nr_states) &&
- flow_entries_differ(&t->st, &s->st, cycle))
- continue;
- if (!widen && flow_has_room(env, *nr_states) &&
- flow_null_differs(flow, &t->st, &s->st))
- continue;
- if (!widen && flow_needed_consts_differ(flow, FLOW_NEED_ARG, &t->st, &s->st))
- continue;
- if (flow_callback_depth_differs(env, &t->st, &s->st))
- continue;
- /* what grows in a loop is not replaced with what is larger, it is forgotten */
- if (!widen && flow_cmp_pkt_range(s, t) <= 0 && flow_within(env, &s->st, &t->st)) {
- err = flow_unite(flow, s->st.flow_entry, t->st.flow_entry);
- if (err)
- goto err_free;
- /*
- * What a callback comes back with and covers what is there grew, like
- * what is joined with it. It counts, or nothing ends "*ctx += 1".
- */
- s->joins = max(s->joins, t->joins) + (cycle == FLOW_CALLBACK_CYCLE);
- flow_unlink(env, t);
- flow_free_state(env, t);
- what = what ?: "joined";
- continue;
- }
- if (!widen && flow_cmp_pkt_range(s, t))
- continue;
- if (flow_has_room(env, *nr_states) &&
- (flow_unset_differs(&t->st, &s->st) ||
- (!widen && flow_join_makes_up(flow, &t->st, &s->st))))
- continue;
- /*
- * Around a loop too:
- * u8 stack[10], depth = 0;
- * loop {
- * if (...) { if (depth >= 10) break; stack[depth++] = x; }
- * else { depth--; if (depth >= 10) break; x = stack[depth]; }
- * }
- * depth is in a 32-bit register. A compiler tests 'depth & 255' and computes it
- * anew for the index. What is found out about a copy says nothing about
- * a range, and everything about a constant.
- */
- if (flow_has_room(env, *nr_states) &&
- flow_needed_consts_differ(flow, FLOW_NEED_STACK_OFF, &t->st, &s->st))
- continue;
- if (!widen && flow_has_room(env, *nr_states) &&
- flow_needed_consts_differ(flow, FLOW_NEED_PTR_OFF, &t->st, &s->st))
- continue;
- err = bpf_states_join(env, &flow->tmp, &t->st, &s->st, widen);
- if (err < 0)
- goto err_free;
- if (err)
- continue;
- what = widen ? "widened" : "joined";
- err = flow_unite(flow, s->st.flow_entry, t->st.flow_entry);
- if (err)
- goto err_free;
- flow->tmp.flow_entry = t->st.flow_entry;
- flow->tmp.last_insn_idx = s->prev_insn_idx;
- s->joins = max(s->joins, t->joins) + 1;
- bpf_free_verifier_state(&s->st, false);
- s->st = flow->tmp;
- memset(&flow->tmp, 0, sizeof(flow->tmp));
- flow_unlink(env, t);
- flow_free_state(env, t);
- /* the join may be within, or have a join with, what is left in the set */
- goto again;
- }
-
- if (*nr_states >= BPF_FLOW_MAX_STATES) {
- verbose(env, "more than %d states that cannot be joined at insn %d\n",
- BPF_FLOW_MAX_STATES, insn_idx);
- err = -E2BIG;
- goto err_free;
- }
- if (flow->live_states > BPF_FLOW_LIVE_PER_INSN * env->prog->len + BPF_FLOW_LIVE_BASE ||
- env->total_states >
- BPF_FLOW_TOTAL_PER_INSN * (u64)env->prog->len + BPF_FLOW_TOTAL_BASE) {
- verbose(env, "too many states for %d insns: %d, %d at a time\n",
- env->prog->len, env->total_states, flow->live_states);
- err = -E2BIG;
- goto err_free;
- }
- /*
- * The state is printed when its turn comes. That says enough about a state that
- * is new, and about a state that goes on to the next insn.
- */
- if (prev >= 0 && insn_idx == prev + 1 + bpf_is_ldimm64(&env->prog->insnsi[prev]) &&
- what && !strcmp(what, "joined"))
- what = NULL;
- if (log && what)
- verbose(env, "from %d to %d: %s\n", prev, insn_idx, what);
- flow_link(env, set, s);
- env->max_states_per_insn = max(env->max_states_per_insn, *nr_states);
- return 0;
-
-err_free:
- flow_free_state(env, s);
- return err;
-}
-
-static struct flow_state *flow_new_state(struct bpf_verifier_env *env, int prev_insn_idx)
-{
- struct flow_state *s = kzalloc_obj(*s, BPF_VERIFIER_GFP);
- struct bpf_flow *flow = env->flow;
-
- if (!s)
- return NULL;
- s->prev_insn_idx = prev_insn_idx;
- s->seq = ++flow->seq;
- s->from = flow->from;
- env->total_states++;
- flow->live_states++;
- env->peak_states = max(env->peak_states, flow->live_states);
- return s;
-}
-
-/* What @st holds is taken and @st is left empty. What doesn't matter at @insn_idx goes. */
-static struct flow_state *flow_arrive(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- int insn_idx, int prev_insn_idx, bool *enters)
-{
- struct bpf_verifier_state *cur = env->cur_state;
- int cur_idx = env->insn_idx, err;
- struct flow_state *s;
-
- if (insn_idx < 0 || insn_idx >= env->prog->len) {
- verbose(env, "invalid insn idx %d insn_cnt %d\n", insn_idx, env->prog->len);
- return ERR_PTR(-EFAULT);
- }
-
- s = flow_new_state(env, prev_insn_idx);
- if (!s)
- return ERR_PTR(-ENOMEM);
- s->st = *st;
- memset(st, 0, sizeof(*st));
- s->st.insn_idx = insn_idx;
- s->st.last_insn_idx = prev_insn_idx;
-
- env->cur_state = &s->st;
- env->insn_idx = insn_idx;
- err = bpf_flow_loop_leave(env, enters);
- if (!err)
- err = bpf_clean_verifier_state(env, &s->st);
- env->cur_state = cur;
- env->insn_idx = cur_idx;
- if (err) {
- flow_free_state(env, s);
- return ERR_PTR(err);
- }
- return s;
-}
-
-/* Is that where may_goto in front of @insn_idx goes, as far as it's known? */
-static bool flow_guard_edge_is_real(struct bpf_verifier_env *env, int insn_idx, u8 edge)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
-
- return aux[bpf_loop_at_index(env, insn_idx)].loop->guarded &&
- aux[insn_idx].guard_no_unwind == (edge == BPF_GUARD_LEAVES_LOOP);
-}
-
-/*
- * @st gets to @insn_idx if the loop has may_goto in front of @prev_insn_idx and that is where it
- * goes, see check_loop_guard(). If that is not so, or not known yet, @st is dropped. If it turns
- * out to be so later all is done again.
- */
-int bpf_flow_guard_edge(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- int insn_idx, int prev_insn_idx, enum bpf_guard_edge edge)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
-
- if (flow_guard_edge_is_real(env, prev_insn_idx, edge)) {
- aux[prev_insn_idx].loop_guard = true;
- return bpf_flow_deliver(env, st, insn_idx, prev_insn_idx);
- }
- /* it's not going to leave the subprog any more */
- if (edge == BPF_GUARD_LEAVES_SUBPROG && aux[prev_insn_idx].guard_no_unwind)
- return 0;
- aux[prev_insn_idx].guard_edge_dropped = true;
- aux[bpf_loop_at_index(env, prev_insn_idx)].loop->no_guard_edges = true;
- return 0;
-}
-
-/*
- * @s came back to the head of a loop, and the walk through the iterations ends, see
- * bpf_flow_loop_enter(). From now on a state stands for many iterations. The first such state is
- * the join of @s with the state that it comes from, one iteration ago. Both were seen, and what
- * differs between them is what the loop changes. If they have no join @s goes on as it is.
- * A state of another walk, which began with another state, has nothing to do with @s.
- */
-static int flow_end_walk(struct bpf_verifier_env *env, struct flow_set *set, struct flow_state *s)
-{
- struct bpf_func_state *frame = s->st.frame[s->st.curframe];
- struct loop_stack_entry *entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
- u32 top = frame->loop_stack_cnt - 1, walk = entry->walk_id;
- struct bpf_flow *flow = env->flow;
- int insn_idx = s->st.insn_idx, err;
- struct flow_state *t;
-
- list_for_each_entry_reverse(t, &set->list, node) {
- /*
- * Some rules of flow_merge() hold here too. One of them: the iteration has set
- * a register, see flow_unset_differs(). The join would be without it.
- */
- if (flow_trips_behind(&t->st, &s->st) ||
- t->st.frame[t->st.curframe]->loop_stack[top].walk_id != walk ||
- (flow_has_room(env, flow->nr_states[insn_idx]) &&
- (flow_unset_differs(&t->st, &s->st) ||
- flow_needed_consts_differ(flow, FLOW_NEED_STACK_OFF, &t->st, &s->st) ||
- flow_needed_consts_differ(flow, FLOW_NEED_PTR_OFF, &t->st, &s->st))))
- continue;
- err = bpf_states_join(env, &flow->tmp, &t->st, &s->st, true);
- if (err < 0)
- return err;
- if (err)
- continue;
- err = flow_unite(flow, s->st.flow_entry, t->st.flow_entry);
- if (err)
- return err;
- flow->tmp.flow_entry = t->st.flow_entry;
- flow->tmp.last_insn_idx = s->prev_insn_idx;
- bpf_free_verifier_state(&s->st, false);
- s->st = flow->tmp;
- memset(&flow->tmp, 0, sizeof(flow->tmp));
- if (env->log.level & BPF_LOG_LEVEL2)
- verbose(env, "from %d to %d: widened\n", s->prev_insn_idx, insn_idx);
- break;
- }
-
- frame = s->st.frame[s->st.curframe];
- entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
- entry->guarded = env->insn_aux_data[insn_idx].loop->guarded;
- entry->apart = false;
- entry->probe = false;
- entry->decided = false;
- entry->terminates = false;
- entry->trip++;
- entry->max_header_count = 0;
- return 0;
-}
-
-/*
- * @s goes into the set of the states at its insn. It is taken.
- * @enters: it enters a loop.
- */
-static int flow_add(struct bpf_verifier_env *env, struct flow_state *s, enum flow_cycle cycle,
- bool enters)
-{
- struct bpf_verifier_state *cur = env->cur_state;
- int cur_idx = env->insn_idx, insn_idx = s->st.insn_idx, err;
- const struct loop_stack_entry *entry;
- const struct bpf_func_state *frame;
- bool backedge = false, drop = false, ends_walk = false;
- struct bpf_flow *flow = env->flow;
-
- env->cur_state = &s->st;
- env->insn_idx = insn_idx;
- err = bpf_flow_loop_enter(env, &backedge, &drop, &ends_walk);
- env->cur_state = cur;
- env->insn_idx = cur_idx;
- if (!err && ends_walk)
- err = flow_end_walk(env, &flow->states[insn_idx], s);
- if (!err && enters) {
- frame = s->st.frame[s->st.curframe];
- entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
- /* it has a count, see bpf_flow_loop_gets_guards() */
- if (entry->terminates && !entry->guarded && !bpf_loop_walk_may_give_up(entry))
- env->insn_aux_data[insn_idx].loop->no_guard_edges = true;
- }
- if (err || drop) {
- flow_free_state(env, s);
- return err;
- }
-
- err = flow_merge(env, &flow->states[insn_idx], &s, backedge ? FLOW_CYCLE : cycle);
- if (err)
- return err < 0 ? err : 0;
- flow_set_pending(flow, s);
- return 0;
-}
-
-/*
- * @st gets to @insn_idx. What it holds is taken and @st is left empty.
- */
-int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- int insn_idx, int prev_insn_idx)
-{
- struct bpf_flow *flow = env->flow;
- struct bpf_subprog_info *sub;
- struct flow_state *s, *e;
- enum flow_cycle cycle;
- bool enters = false;
- int err;
-
- s = flow_arrive(env, st, insn_idx, prev_insn_idx, &enters);
- if (IS_ERR(s))
- return PTR_ERR(s);
-
- /*
- * Cycles that have no loop head: the callback that came back to the helper
- * that calls it, and calls that go through the entry of a function.
- */
- sub = bpf_find_containing_subprog(env, insn_idx);
- cycle = FLOW_NO_CYCLE;
- if (sub->recursive && sub->start == insn_idx)
- cycle = FLOW_CYCLE;
- else if (env->insn_aux_data[insn_idx].calls_callback && prev_insn_idx >= 0 &&
- env->prog->insnsi[prev_insn_idx].code == (BPF_JMP | BPF_EXIT))
- cycle = FLOW_CALLBACK_CYCLE;
- else if (env->insn_aux_data[insn_idx].calls_callback)
- s->st.frame[s->st.curframe]->callback_entry_id = ++env->loop_entry_cnt;
-
- /*
- * What is known about a loop is derived from the state that enters it. So
- * the states that enter are a set of their own, and what gets to
- * the set at the head of the loop is derived from what is new in it.
- */
- if (enters) {
- err = flow_merge(env, &flow->enter[insn_idx], &s, cycle);
- if (err)
- return err < 0 ? err : 0;
- cycle = FLOW_NO_CYCLE;
- e = s;
- s = flow_new_state(env, prev_insn_idx);
- if (!s)
- return -ENOMEM;
- err = bpf_copy_verifier_state(&s->st, &e->st);
- if (err) {
- flow_free_state(env, s);
- return err;
- }
- }
- return flow_add(env, s, cycle, enters);
-}
-
-static void flow_free_call(struct flow_state *w)
-{
- list_del(&w->node);
- bpf_free_verifier_state(&w->st, false);
- kfree(w);
-}
-
-/* @x came out of the function that the call @w waits for. @w may be gone afterwards. */
-static int flow_apply_exit(struct bpf_verifier_env *env, struct flow_state *w,
- struct flow_exit *x)
-{
- int call_idx = w->st.insn_idx, err;
- struct bpf_verifier_state st = {};
- struct flow_state *s;
-
- err = bpf_copy_verifier_state(&st, &w->st);
- if (!err)
- err = bpf_flow_return(env, &st, &x->st, call_idx);
- if (!err)
- err = bpf_flow_deliver(env, &st, call_idx + 1, call_idx);
- bpf_free_verifier_state(&st, false);
- if (err != 1)
- return err;
-
- /*
- * The call is verified in the context of the caller: it doesn't wait any more, it is
- * done again. Not with bpf_flow_deliver(): the state was at the insn before, it may
- * be there still.
- */
- s = flow_new_state(env, w->st.last_insn_idx);
- if (!s)
- return -ENOMEM;
- s->st = w->st;
- memset(&w->st, 0, sizeof(w->st));
- flow_free_call(w);
- flow_shape_ctx(&s->st, &s->shape);
- flow_link(env, &env->flow->states[call_idx], s);
- flow_set_pending(env->flow, s);
- return 0;
-}
-
-/* What came out for the set of @from goes to the calls that wait for the set of @to */
-static int flow_apply_exits(struct bpf_verifier_env *env, struct flow_entry *from,
- struct flow_entry *to)
-{
- struct flow_state *w, *wn;
- struct flow_exit *x;
- int err;
-
- list_for_each_entry(x, &from->exits, node) {
- list_for_each_entry_safe(w, wn, &to->calls, node) {
- err = flow_apply_exit(env, w, x);
- if (err)
- return err;
- }
- }
- return 0;
-}
-
-static int flow_do_united(struct bpf_verifier_env *env)
-{
- struct bpf_flow *flow = env->flow;
- struct flow_entry *a, *b;
- struct flow_united u;
- int err;
-
- while (flow->united_cnt) {
- u = flow->united[--flow->united_cnt];
- a = flow_entry_find(flow, u.a);
- b = flow_entry_find(flow, u.b);
- if (a == b)
- continue;
- err = flow_apply_exits(env, a, b) ?: flow_apply_exits(env, b, a);
- if (err)
- return err;
- list_splice_tail_init(&a->calls, &b->calls);
- list_splice_tail_init(&a->exits, &b->exits);
- a->up = b;
- }
- return 0;
-}
-
-/*
- * The call at @call_idx in @caller of a function that is verified on its own,
- * @entry is the state at its first insn. Both are taken.
- */
-int bpf_flow_call(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
- struct bpf_verifier_state *entry, int subprog, int call_idx)
-{
- struct bpf_flow *flow = env->flow;
- struct flow_state *w;
- int id;
-
- w = kzalloc_obj(*w, BPF_VERIFIER_GFP);
- if (!w)
- return -ENOMEM;
- w->st = *caller;
- memset(caller, 0, sizeof(*caller));
- w->st.insn_idx = call_idx;
-
- id = flow_entry_new(flow);
- if (id < 0) {
- bpf_free_verifier_state(&w->st, false);
- kfree(w);
- return id;
- }
- list_add_tail(&w->node, &flow->entries[id]->calls);
-
- entry->flow_entry = id;
- if (env->log.level & BPF_LOG_LEVEL2)
- verbose(env, "call at %d waits for func#%d\n", call_idx, subprog);
- return bpf_flow_deliver(env, entry, env->subprog_info[subprog].start, call_idx);
-}
-
-/* @st came to exit of the function that is verified on its own. It is taken. */
-int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int exit_idx)
-{
- struct bpf_flow *flow = env->flow;
- struct flow_func *f = &flow->funcs[st->frame[0]->subprogno];
- u32 id = st->flow_entry, cnt = 0;
- struct flow_exit *x, *o, *on;
- struct flow_state *w, *wn;
- struct flow_entry *entry;
- u16 joins = 0;
- int err;
-
- st->insn_idx = exit_idx;
- err = bpf_clean_verifier_state(env, st);
- if (err)
- return err;
- if (f->exit_idx < 0)
- f->exit_idx = exit_idx;
- st->insn_idx = f->exit_idx;
- entry = flow_entry_find(flow, id);
-
- list_for_each_entry_safe(o, on, &entry->exits, node) {
- if (bpf_state_subsumes(env, &o->st, st))
- return 0;
- if (flow_has_room(env, ++cnt) && flow_join_makes_up(flow, &o->st, st))
- continue;
- /* what comes out of a recursive function gets into it again */
- err = bpf_states_join(env, &flow->tmp, &o->st, st,
- env->subprog_info[st->frame[0]->subprogno].recursive ||
- o->joins > BPF_FLOW_MAX_JOINS);
- if (err < 0)
- return err;
- if (err)
- continue;
- joins = max(joins, o->joins) + 1;
- bpf_free_verifier_state(st, false);
- *st = flow->tmp;
- memset(&flow->tmp, 0, sizeof(flow->tmp));
- st->flow_entry = id;
- list_del(&o->node);
- bpf_free_verifier_state(&o->st, false);
- kfree(o);
- }
-
- x = kzalloc_obj(*x, BPF_VERIFIER_GFP);
- if (!x)
- return -ENOMEM;
- x->st = *st;
- memset(st, 0, sizeof(*st));
- x->joins = joins;
- list_add_tail(&x->node, &entry->exits);
-
- list_for_each_entry_safe(w, wn, &entry->calls, node) {
- err = flow_apply_exit(env, w, x);
- if (err)
- return err;
- }
- return 0;
-}
-
-/*
- * Make the next pending state the current one. env->cur_state has to be empty.
- * Returns -ENOENT when nothing is pending.
- */
-int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx)
-{
- struct bpf_flow *flow = env->flow;
- struct flow_state *s;
- struct rb_node *n;
- int err;
-
- do {
- err = flow_do_united(env);
- if (err)
- return err;
- } while (flow->united_cnt);
- n = rb_first_cached(&flow->pending);
- if (!n)
- return -ENOENT;
- s = rb_entry(n, struct flow_state, rb);
- rb_erase_cached(&s->rb, &flow->pending);
- s->pending = false;
-
- env->insn_idx = s->st.insn_idx;
- *prev_insn_idx = s->prev_insn_idx;
- if (test_bit(s->st.insn_idx, flow->keep)) {
- flow->from = s->seq;
- return bpf_copy_verifier_state(env->cur_state, &s->st);
- }
- flow->from = s->from;
- *env->cur_state = s->st;
- memset(&s->st, 0, sizeof(s->st));
- s->st.insn_idx = env->insn_idx;
- flow_unlink(env, s);
- flow_free_state(env, s);
- return 0;
-}
-
-/* A basic block on the way to the block that is rejected, up to the insn that leads on */
-struct flow_step {
- const struct flow_state *s; /* the state at its head, if one is kept */
- int head, last;
-};
-
-/*
- * The state at @head with that @seq. If it is gone a state that came later took its place:
- * the last one then.
- */
-static const struct flow_state *flow_find_state(const struct bpf_flow *flow, int head, u32 seq)
-{
- const struct list_head *list = &flow->states[head].list;
- const struct flow_state *s;
-
- if (!test_bit(head, flow->keep))
- return NULL;
- list_for_each_entry(s, list, node)
- if (s->seq == seq)
- return s;
- return list_empty(list) ? NULL : list_last_entry(list, struct flow_state, node);
-}
-
-/*
- * The basic block at @head, which the current state got to from insn @i, is rejected. The block
- * and the message are what was printed since @log_pos. Print the way to the block in front of it.
- *
- * The way is put together backwards from what is there. A state that is kept knows the edge that
- * it came by, after a join the edge that changed it last, and the kept state that it comes from.
- * A block whose state is not kept has one predecessor, which is its dominator.
- * A block is on the way once. Where it would come again, after a round of a loop, the way goes on
- * from its dominator: all that gets into the loop goes through it.
- */
-void bpf_flow_print_way(struct bpf_verifier_env *env, u64 log_pos, int head, int i)
-{
- struct bpf_insn *insns = env->prog->insnsi;
- const struct bpf_flow *flow = env->flow;
- const struct flow_state *s;
- struct bpf_vlog_stash stash;
- struct flow_step *way, *w;
- u32 from = flow->from, cnt = 0;
- unsigned long *seen;
-
- seen = bitmap_zalloc(flow->len, BPF_VERIFIER_GFP);
- way = kvmalloc_array(bitmap_weight(flow->point, flow->len), sizeof(*way),
- BPF_VERIFIER_GFP);
- if (!seen || !way)
- goto out;
-
- s = flow_find_state(flow, head, from);
- from = s ? s->from : from;
- while (i >= 0) {
- head = find_last_bit(flow->point, i + 1);
- s = NULL;
- if (!__test_and_set_bit(head, seen)) {
- s = flow_find_state(flow, head, from);
- way[cnt++] = (struct flow_step){ s, head, i };
- }
- from = s ? s->from : from;
- i = s ? s->prev_insn_idx : env->idoms[head];
- cond_resched();
- }
-
- if (!cnt || bpf_vlog_stash(&env->log, log_pos, &stash))
- goto out;
- env->prev_linfo = NULL;
- while (cnt-- && !bpf_must_abort()) {
- w = &way[cnt];
- if (w->s) {
- verbose(env, "from %d to %d:", w->s->prev_insn_idx, w->head);
- print_verifier_state(env, &w->s->st, w->s->st.curframe, true);
- }
- for (i = w->head; i <= w->last; i += 1 + bpf_is_ldimm64(&insns[i])) {
- verbose_linfo(env, i, "; ");
- verbose(env, "%d: ", i);
- bpf_verbose_insn(env, &insns[i]);
- verbose(env, "\n");
- }
- }
- bpf_vlog_unstash(&env->log, &stash);
-out:
- bitmap_free(seen);
- kvfree(way);
-}
diff --git a/kernel/bpf/join.c b/kernel/bpf/join.c
index 6a8efb671a8..95de080617b 100644
--- a/kernel/bpf/join.c
+++ b/kernel/bpf/join.c
@@ -1,627 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Join of verifier states.
- *
- * bpf_states_join(dst, a, b) computes the state that describes every concrete
- * state described by 'a' or by 'b'. With @widen a number that is larger in the
- * result than in 'a' gets the bound that moved put further out, so that a sequence
- * of states at a loop head ends. A pointer whose offset is larger has no join.
- *
- * Numbers and offsets of pointers are joined. A pointer and a number, pointers
- * to different objects, different references and locks are not: there is no
- * join and the states stay apart.
- *
- * The contract is checked on every call: both inputs have to be within the
- * result, see bpf_state_subsumes(). A violation is a verifier bug.
- *
- * Inputs have to be at the same insn and cleaned with
- * bpf_clean_verifier_state(): dead registers and stack slots are the same on
- * both sides then, but for dynptrs, iterators and irq flags: they stay.
- */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/cnum.h>
#include <linux/tnum.h>
#include <linux/filter.h>
-#include <linux/slab.h>
-
-struct join_idpair {
- u32 a;
- u32 b;
- bool bad;
-};
-
-struct bpf_join_state {
- struct bpf_verifier_env *env;
- struct join_idpair pairs[BPF_ID_MAP_SIZE];
- u32 npairs;
- bool pairs_overflow;
- bool widen;
- /* the states came around this loop in the frame that is joined, NULL if not */
- const struct bpf_loop *loop;
- /* the register or stack slot that is joined, as in struct bpf_loop_rungs, or -1 */
- int place;
-};
-
-static struct bpf_join_state *join_state(struct bpf_verifier_env *env)
-{
- if (!env->join) {
- env->join = kvzalloc_obj(*env->join, BPF_VERIFIER_GFP);
- if (env->join)
- env->join->env = env;
- }
- return env->join;
-}
-
-void bpf_join_free(struct bpf_verifier_env *env)
-{
- kvfree(env->join);
- env->join = NULL;
-}
-
-/*
- * An id in the result stands for the pair (id in 'a', id in 'b'). For the
- * result to be within both inputs the pairs have to form a bijection,
- * see check_ids(). The first pass over all locations records the pairs. An id
- * seen with two partners makes all pairs it takes part in bad.
- * The second pass keeps the id of 'a' for good pairs.
- */
-static void jid_mark_bad(struct bpf_join_state *js, u32 a, u32 b)
-{
- u32 i;
-
- for (i = 0; i < js->npairs; i++)
- if ((a && js->pairs[i].a == a) || (b && js->pairs[i].b == b))
- js->pairs[i].bad = true;
-}
-
-static void jid_note(struct bpf_join_state *js, u32 a, u32 b)
-{
- bool consistent, other = false;
- struct join_idpair *p;
- u32 i;
-
- if (!a || !b)
- return;
- /* a name of what the caller has, see BPF_INHERITED_ID */
- consistent = a == b || (a | b) < BPF_INHERITED_ID;
- for (i = 0; i < js->npairs; i++) {
- p = &js->pairs[i];
- if (p->a == a && p->b == b) {
- if (!consistent)
- jid_mark_bad(js, a, b);
- return;
- }
- if (p->a == a || p->b == b)
- other = true;
- }
- if (other) {
- jid_mark_bad(js, a, b);
- consistent = false;
- }
- if (js->npairs == ARRAY_SIZE(js->pairs)) {
- js->pairs_overflow = true;
- return;
- }
- p = &js->pairs[js->npairs++];
- p->a = a;
- p->b = b;
- p->bad = !consistent;
-}
-
-static bool jid_good(struct bpf_join_state *js, u32 a, u32 b)
-{
- u32 i;
-
- if (!a || !b || js->pairs_overflow)
- return false;
- for (i = 0; i < js->npairs; i++)
- if (js->pairs[i].a == a && js->pairs[i].b == b)
- return !js->pairs[i].bad;
- return false;
-}
-
-/* Ids that have to match: both zero, or a good pair */
-static bool jid_strict_ok(struct bpf_join_state *js, u32 ida, u32 idb)
-{
- if (!ida && !idb)
- return true;
- return jid_good(js, ida, idb);
-}
-
-static const struct bpf_loop_rungs *join_rungs(const struct bpf_join_state *js)
-{
- u32 i;
-
- for (i = 0; js->loop && i < js->loop->rungs_cnt; i++)
- if (js->loop->rungs[i].place == js->place)
- return &js->loop->rungs[i];
- return NULL;
-}
-
-/* Does every iteration add a constant to what is in the place? In 64 or in 32 bits, 0 if not */
-static int join_place_is_linear(const struct bpf_join_state *js)
-{
- /* the place as scalar evolution numbers it */
- u32 r = js->place < BPF_LOOP_SLOT ? js->place : js->place - BPF_LOOP_SLOT + MAX_BPF_REG;
-
- if (!js->loop || js->place < 0 || r >= BPF_SCEV_REGS_NUM)
- return 0;
- return test_bit(r, js->loop->linear_regs) ? 64 :
- test_bit(r, js->loop->linear32_regs) ? 32 : 0;
-}
-
-/*
- * Join of the range of two registers of the same type, numbers or pointers
- * with variable offset. @dst is a copy of @ra. Returns false if there is no join.
- */
-static bool join_range(struct bpf_join_state *js, struct bpf_reg_state *dst,
- const struct bpf_reg_state *ra, const struct bpf_reg_state *rb)
-{
- if (scalar_equal(scalar_of(ra), scalar_of(rb)))
- return true;
- scalar_union(scalar_of(dst), scalar_of(rb));
- if (!js->widen || scalar_equal(scalar_of(dst), scalar_of(ra)))
- return true;
- /*
- * The bound of a number that grows around a cycle is put where it may come to rest. If
- * that is far out the number gets a bound again from the next comparison. The offset of
- * a pointer doesn't: there is no access through a pointer at an offset that is not
- * known. So there is nothing to gain from that join. If the offset stops to grow
- * the next state is within the last one.
- */
- if (dst->type != SCALAR_VALUE)
- return false;
- scalar_widen(scalar_of(dst), scalar_of(ra), join_rungs(js), join_place_is_linear(js));
- return true;
-}
-
-/*
- * Join two registers, regsafe() documents what each type needs.
- * Pass 1 records id pairs, pass 2 computes @dst, which is a copy of @ra on entry.
- * Returns false if there is no join.
- */
-static bool join_reg(struct bpf_join_state *js, struct bpf_reg_state *dst,
- const struct bpf_reg_state *ra, const struct bpf_reg_state *rb, int pass)
-{
- if (ra->type == NOT_INIT || rb->type == NOT_INIT) {
- if (pass == 2 && ra->type != NOT_INIT)
- *dst = *rb;
- return true;
- }
-
- if (ra->type != rb->type)
- return false;
-
- switch (base_type(ra->type)) {
- case SCALAR_VALUE:
- if (js->env->explore_alu_limits && !reg_value_equal(ra, rb))
- return false;
- if (pass == 2)
- join_range(js, dst, ra, rb);
- return true;
- case PTR_TO_MAP_KEY:
- case PTR_TO_MAP_VALUE:
- case PTR_TO_MEM:
- case PTR_TO_BUF:
- case PTR_TO_TP_BUFFER:
- if (!reg_fixed_part_equal(ra, rb))
- return false;
- /*
- * Two places in an object are two pointers: what is at a place
- * and how much is left behind it is lost in a range of them.
- */
- if (!js->widen && tnum_is_const(ra->var_off) && tnum_is_const(rb->var_off) &&
- ra->var_off.value != rb->var_off.value)
- return false;
- break;
- case PTR_TO_PACKET_META:
- case PTR_TO_PACKET:
- if ((ra->range < 0 || rb->range < 0) && ra->range != rb->range)
- return false;
- /* how much of the packet was checked goes with numbers elsewhere in the state */
- if (!js->widen && ra->range != rb->range)
- return false;
- /*
- * Displacements between pointers that share an id are kept. No id is an id too:
- * the start of the packet plus a constant, see find_good_pkt_pointers().
- */
- if (ra->r64.base != rb->r64.base)
- return false;
- if (pass == 1) {
- jid_note(js, ra->id, rb->id);
- return true;
- }
- if (!jid_strict_ok(js, ra->id, rb->id))
- return false;
- if (rb->range != ra->range)
- dst->range = min(ra->range, rb->range);
- return join_range(js, dst, ra, rb);
- case PTR_TO_ARENA:
- /* The offset is not tracked. */
- return true;
- case PTR_TO_INSN:
- if (!reg_fixed_part_equal(ra, rb))
- return false;
- return pass == 1 || join_range(js, dst, ra, rb);
- default:
- if (!reg_value_equal(ra, rb))
- return false;
- break;
- }
-
- if (pass == 1) {
- jid_note(js, ra->id, rb->id);
- jid_note(js, ra->parent_id, rb->parent_id);
- jid_note(js, ra->map_uid, rb->map_uid);
- return true;
- }
- if (!jid_strict_ok(js, ra->id, rb->id) ||
- !jid_strict_ok(js, ra->parent_id, rb->parent_id) ||
- !jid_strict_ok(js, ra->map_uid, rb->map_uid))
- return false;
- return join_range(js, dst, ra, rb);
-}
-
-static bool slot_is_special(const struct bpf_stack_state *s)
-{
- u8 t = s->slot_type[BPF_REG_SIZE - 1];
-
- return t == STACK_DYNPTR || t == STACK_ITER || t == STACK_IRQ_FLAG;
-}
-
-static bool stype_is_invalid(u8 t)
-{
- return t == STACK_INVALID || t == STACK_POISON;
-}
-
-static bool slot_has_spill(const struct bpf_stack_state *s)
-{
- int i;
-
- for (i = 0; i < BPF_REG_SIZE; i++)
- if (s->slot_type[i] == STACK_SPILL)
- return true;
- return false;
-}
-
-static bool join_special_slot(struct bpf_join_state *js, const struct bpf_stack_state *sa,
- const struct bpf_stack_state *sb, int pass)
-{
- const struct bpf_reg_state *ra = &sa->spilled_ptr, *rb = &sb->spilled_ptr;
-
- if (memcmp(sa->slot_type, sb->slot_type, sizeof(sa->slot_type)))
- return false;
- switch (sa->slot_type[BPF_REG_SIZE - 1]) {
- case STACK_DYNPTR:
- if (ra->dynptr.type != rb->dynptr.type ||
- ra->dynptr.first_slot != rb->dynptr.first_slot)
- return false;
- break;
- case STACK_ITER:
- if (ra->type != rb->type || ra->iter.btf != rb->iter.btf ||
- ra->iter.btf_id != rb->iter.btf_id || ra->iter.state != rb->iter.state ||
- ra->iter.loops != rb->iter.loops)
- return false;
- break;
- case STACK_IRQ_FLAG:
- if (ra->irq.kfunc_class != rb->irq.kfunc_class)
- return false;
- break;
- }
- if (pass == 1) {
- jid_note(js, ra->id, rb->id);
- jid_note(js, ra->parent_id, rb->parent_id);
- return true;
- }
- return jid_strict_ok(js, ra->id, rb->id) &&
- jid_strict_ok(js, ra->parent_id, rb->parent_id);
-}
-
-/*
- * Join of one 8 byte stack slot, @ds is a copy of @sa on entry.
- * stacksafe() documents which slot is within which.
- */
-static bool join_slot(struct bpf_join_state *js, struct bpf_stack_state *ds,
- const struct bpf_stack_state *sa, const struct bpf_stack_state *sb, int pass)
-{
- static const struct bpf_stack_state invalid_slot;
- bool a_ptr, b_ptr;
- u8 res[BPF_REG_SIZE];
- int i;
-
- if (!sb)
- sb = &invalid_slot;
-
- if (slot_is_special(sa) || slot_is_special(sb))
- return join_special_slot(js, sa, sb, pass);
-
- if (!memcmp(sa->slot_type, sb->slot_type, sizeof(sa->slot_type)) &&
- sa->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL)
- return join_reg(js, &ds->spilled_ptr, &sa->spilled_ptr, &sb->spilled_ptr, pass);
-
- a_ptr = slot_has_spill(sa) && sa->spilled_ptr.type != SCALAR_VALUE;
- b_ptr = slot_has_spill(sb) && sb->spilled_ptr.type != SCALAR_VALUE;
- /* a pointer and something else */
- if (a_ptr || b_ptr)
- return false;
-
- /* the prog can't read uninitialized stack: slots of different kinds have no join */
- if (!js->env->allow_uninit_stack)
- return !memcmp(sa->slot_type, sb->slot_type, sizeof(sa->slot_type));
-
- /* scalar spills, MISC, ZERO, INVALID, POISON: no ids are kept */
- if (pass == 1)
- return true;
-
- for (i = 0; i < BPF_REG_SIZE; i++) {
- u8 x = sa->slot_type[i], y = sb->slot_type[i];
-
- if (x == STACK_SPILL)
- x = STACK_MISC;
- if (y == STACK_SPILL)
- y = STACK_MISC;
- if (stype_is_invalid(x) && stype_is_invalid(y))
- res[i] = sa->slot_type[i];
- else if (stype_is_invalid(x) || stype_is_invalid(y))
- res[i] = STACK_INVALID;
- else if (x == y)
- res[i] = x;
- else
- res[i] = STACK_MISC;
- }
- memcpy(ds->slot_type, res, sizeof(res));
- bpf_mark_reg_not_init(js->env, &ds->spilled_ptr);
- return true;
-}
-
-static bool join_loop_stack(const struct bpf_func_state *fa, const struct bpf_func_state *fb)
-{
- u32 i;
-
- if (fa->loop_stack_cnt != fb->loop_stack_cnt)
- return false;
- /* what is known about a counted loop comes from the state it was entered with */
- for (i = 0; i < fa->loop_stack_cnt; i++)
- if (fa->loop_stack[i].loop_id != fb->loop_stack[i].loop_id ||
- fa->loop_stack[i].terminates != fb->loop_stack[i].terminates ||
- fa->loop_stack[i].guarded != fb->loop_stack[i].guarded ||
- fa->loop_stack[i].counted != fb->loop_stack[i].counted ||
- fa->loop_stack[i].apart != fb->loop_stack[i].apart ||
- fa->loop_stack[i].first != fb->loop_stack[i].first ||
- (fa->loop_stack[i].apart && fa->loop_stack[i].trip != fb->loop_stack[i].trip) ||
- fa->loop_stack[i].refs != fb->loop_stack[i].refs ||
- fa->loop_stack[i].max_header_count != fb->loop_stack[i].max_header_count ||
- fa->loop_stack[i].entry_state != fb->loop_stack[i].entry_state)
- return false;
- return true;
-}
-
-static bool join_func(struct bpf_join_state *js, struct bpf_func_state *fd,
- const struct bpf_func_state *fa, const struct bpf_func_state *fb,
- u16 live_regs, int pass)
-{
- static const struct bpf_reg_state not_init = { .type = NOT_INIT };
- struct bpf_verifier_env *env = js->env;
- u32 i, nslots_b;
- bool in_a, in_b;
-
- if (pass == 1) {
- if (fa->callsite != fb->callsite || fa->frameno != fb->frameno ||
- fa->subprogno != fb->subprogno)
- return false;
- if (fa->in_callback_fn != fb->in_callback_fn ||
- fa->in_async_callback_fn != fb->in_async_callback_fn ||
- fa->in_exception_callback_fn != fb->in_exception_callback_fn ||
- fa->in_rcu_callback_fn != fb->in_rcu_callback_fn ||
- fa->async_entry_cnt != fb->async_entry_cnt ||
- memcmp(&fa->callback_ret_range, &fb->callback_ret_range,
- sizeof(fa->callback_ret_range)))
- return false;
- if (fa->entry_refs != fb->entry_refs ||
- fa->entry_rcu_locks != fb->entry_rcu_locks ||
- fa->entry_preempt_locks != fb->entry_preempt_locks)
- return false;
- if (!join_loop_stack(fa, fb))
- return false;
- jid_note(js, fa->frame_id, fb->frame_id);
- } else if (!jid_strict_ok(js, fa->frame_id, fb->frame_id)) {
- return false;
- }
-
- for (i = 0; i < MAX_BPF_REG; i++) {
- if (!(live_regs & BIT(i))) {
- if (pass == 2 && i < BPF_REG_FP)
- bpf_mark_reg_not_init(env, &fd->regs[i]);
- continue;
- }
- js->place = i;
- if (!join_reg(js, &fd->regs[i], &fa->regs[i], &fb->regs[i], pass))
- return false;
- }
-
- nslots_b = bpf_stack_nr_slots(fb);
- for (i = 0; i < bpf_stack_nr_slots(fa); i++) {
- const struct bpf_stack_state *sb = i < nslots_b ? bpf_stack_slot(fb, i) : NULL;
-
- js->place = BPF_LOOP_SLOT + i;
- if (!join_slot(js, bpf_stack_slot(fd, i), bpf_stack_slot(fa, i), sb, pass))
- return false;
- }
- js->place = -1;
- /* Slots that exist only in 'b' are STACK_INVALID in the result. */
- for (i = bpf_stack_nr_slots(fa); pass == 1 && i < nslots_b; i++) {
- const struct bpf_stack_state *sb = bpf_stack_slot(fb, i);
-
- if (slot_is_special(sb) ||
- (slot_has_spill(sb) && sb->spilled_ptr.type != SCALAR_VALUE))
- return false;
- }
-
- if (fa->out_stack_arg_cnt < fb->out_stack_arg_cnt)
- return false;
- /* incoming arguments are not joined with outgoing ones: that side has no outgoing ones */
- in_a = bpf_stack_args_are_incoming(fa) && !bpf_stack_args_are_incoming(fb);
- in_b = bpf_stack_args_are_incoming(fb) && !bpf_stack_args_are_incoming(fa);
- for (i = 0; i < fa->out_stack_arg_cnt; i++) {
- const struct bpf_reg_state *rb;
-
- if (in_a) {
- if (pass == 2)
- bpf_mark_reg_not_init(env, &fd->stack_arg_regs[i]);
- continue;
- }
- rb = i < fb->out_stack_arg_cnt && !in_b ? &fb->stack_arg_regs[i] : ¬_init;
- if (!join_reg(js, &fd->stack_arg_regs[i], &fa->stack_arg_regs[i], rb, pass))
- return false;
- }
-
- if (pass == 2) {
- fd->no_stack_arg_load = fa->no_stack_arg_load || fb->no_stack_arg_load;
- fd->callback_depth = min(fa->callback_depth, fb->callback_depth);
- for (i = 0; i < fd->loop_stack_cnt; i++)
- fd->loop_stack[i].trip = max(fd->loop_stack[i].trip, fb->loop_stack[i].trip);
- }
- return true;
-}
-
-/* Things that can't be different in two states that are joined */
-static bool join_check_state(const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- int i;
-
- if (a->curframe != b->curframe || a->insn_idx != b->insn_idx)
- return false;
- if (a->speculative != b->speculative || a->in_sleepable != b->in_sleepable ||
- a->callers_unknown != b->callers_unknown ||
- a->unwound != b->unwound || a->changed_pkt_data != b->changed_pkt_data)
- return false;
- if (a->active_locks != b->active_locks ||
- a->active_preempt_locks != b->active_preempt_locks ||
- a->active_rcu_locks != b->active_rcu_locks ||
- a->active_lock_ptr != b->active_lock_ptr ||
- !a->active_irq_id != !b->active_irq_id ||
- !a->active_lock_id != !b->active_lock_id)
- return false;
- if (a->acquired_refs != b->acquired_refs)
- return false;
- for (i = 0; i < a->acquired_refs; i++) {
- if (a->refs[i].type != b->refs[i].type)
- return false;
- switch (a->refs[i].type) {
- case REF_TYPE_PTR:
- case REF_TYPE_IRQ:
- break;
- default:
- if (a->refs[i].ptr != b->refs[i].ptr)
- return false;
- }
- }
- return true;
-}
-
-static void join_refs_note(struct bpf_join_state *js, const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- int i;
-
- jid_note(js, a->active_irq_id, b->active_irq_id);
- jid_note(js, a->active_lock_id, b->active_lock_id);
- for (i = 0; i < a->acquired_refs; i++) {
- jid_note(js, a->refs[i].id, b->refs[i].id);
- if (a->refs[i].type == REF_TYPE_PTR)
- jid_note(js, a->refs[i].parent_id, b->refs[i].parent_id);
- }
-}
-
-static bool join_refs_check(struct bpf_join_state *js, const struct bpf_verifier_state *a,
- const struct bpf_verifier_state *b)
-{
- int i;
-
- if (!jid_strict_ok(js, a->active_irq_id, b->active_irq_id) ||
- !jid_strict_ok(js, a->active_lock_id, b->active_lock_id))
- return false;
- for (i = 0; i < a->acquired_refs; i++) {
- if (!jid_strict_ok(js, a->refs[i].id, b->refs[i].id))
- return false;
- if (a->refs[i].type == REF_TYPE_PTR &&
- !jid_strict_ok(js, a->refs[i].parent_id, b->refs[i].parent_id))
- return false;
- }
- return true;
-}
-
-/* The result has to describe everything that an input describes */
-static int join_check(struct bpf_join_state *js, struct bpf_verifier_state *dst,
- const struct bpf_verifier_state *in, const char *which)
-{
- struct bpf_verifier_env *env = js->env;
-
- if (bpf_state_subsumes(env, dst, (struct bpf_verifier_state *)in))
- return 0;
- /* check_ids() has no room left: that says nothing about the join */
- if (env->idmap_scratch.cnt == BPF_ID_MAP_SIZE)
- return 1;
- bpf_log(&env->log, "join:");
- print_verifier_state(env, dst, dst->curframe, true);
- bpf_log(&env->log, "input %s:", which);
- print_verifier_state(env, in, in->curframe, true);
- verifier_bug(env, "join of states at insn %u does not subsume input '%s'",
- dst->insn_idx, which);
- return -EFAULT;
-}
-
-/*
- * Compute in @dst the state that describes everything that @a or @b describe.
- * @dst has to be a valid state (zeroed is fine) different from @a and @b, its
- * previous content is released. Bookkeeping fields are copied from @a.
- *
- * Returns 0 on success, 1 if there is no join, negative error otherwise.
- * When 1 is returned the content of @dst is undefined, but it's a valid state.
- */
-int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
- const struct bpf_verifier_state *a, const struct bpf_verifier_state *b,
- bool widen)
-{
- struct bpf_join_state *js = join_state(env);
- const struct bpf_func_state *frame;
- int pass, fr, err;
-
- if (!js)
- return -ENOMEM;
- js->widen = widen;
- js->npairs = 0;
- js->pairs_overflow = false;
-
- if (!join_check_state(a, b))
- return 1;
- err = bpf_copy_verifier_state(dst, a);
- if (err)
- return err;
-
- for (pass = 1; pass <= 2; pass++) {
- if (pass == 1)
- join_refs_note(js, a, b);
- else if (!join_refs_check(js, a, b))
- return 1;
- for (fr = 0; fr <= a->curframe; fr++) {
- u32 insn_idx = bpf_frame_insn_idx((struct bpf_verifier_state *)a, fr);
- u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
- u32 cnt;
-
- /* not the states that get to the head through a call of the function */
- frame = a->frame[fr];
- cnt = frame->loop_stack_cnt;
- js->loop = widen && fr == a->curframe && cnt &&
- frame->loop_stack[cnt - 1].loop_id == insn_idx ?
- env->insn_aux_data[insn_idx].loop : NULL;
- if (!join_func(js, dst->frame[fr], a->frame[fr], b->frame[fr],
- live_regs, pass))
- return 1;
- }
- }
-
- return join_check(js, dst, a, "a") ?: join_check(js, dst, b, "b");
-}
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index d5132e08ecd..20820098c6b 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -177,53 +177,6 @@ void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)
log->ubuf = NULL;
}
-/*
- * Take what was written since @pos out of the log. bpf_vlog_unstash() appends it to what is
- * written in between. The text is in the buffer of the user and nowhere else. What the buffer
- * doesn't hold, the start of the text if it's rolling and the end if it's fixed, is counted.
- */
-int bpf_vlog_stash(struct bpf_verifier_log *log, u64 pos, struct bpf_vlog_stash *stash)
-{
- u64 from = max(pos, log->start_pos), to = log->end_pos, i;
- u32 off, n;
-
- if (!log->ubuf)
- from = to;
- else if (log->level & BPF_LOG_FIXED)
- to = clamp(log->len_total - 1ULL, from, to);
- stash->lost_start = from - pos;
- stash->lost_end = log->end_pos - to;
- stash->len = to - from;
- /* as large as the buffer at worst: fail before the OOM killer is called */
- stash->buf = kvzalloc(stash->len, BPF_VERIFIER_GFP);
- if (!stash->buf)
- return -ENOMEM;
- /* two pieces if it wraps around */
- for (i = from; i < to && log->ubuf; i += n) {
- div_u64_rem(i, log->len_total, &off);
- n = min(to - i, log->len_total - off);
- if (copy_from_user(stash->buf + (i - from), log->ubuf + off, n))
- log->ubuf = NULL;
- }
- bpf_vlog_reset(log, pos);
- return 0;
-}
-
-void bpf_vlog_unstash(struct bpf_verifier_log *log, struct bpf_vlog_stash *stash)
-{
- u64 i, n;
-
- log->end_pos += stash->lost_start;
- for (i = 0; i < stash->len; i += n) {
- n = min(stash->len - i, sizeof(log->kbuf) - 1);
- bpf_log(log, "%.*s", (int)n, stash->buf + i);
- cond_resched();
- }
- log->end_pos += stash->lost_end;
- bpf_vlog_update_len_max(log, 1);
- kvfree(stash->buf);
-}
-
static void bpf_vlog_reverse_kbuf(char *buf, int len)
{
int i, j;
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index a37b0434739..378dd07dd21 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -35,56 +35,6 @@ static bool range_within(const struct bpf_reg_state *old,
(cur->base - old->base) % old->step == 0;
}
-/* If in the old state two registers had the same id, then they need to have
- * the same id in the new state as well. But that id could be different from
- * the old state, so we need to track the mapping from old to new ids.
- * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
- * regs with old id 5 must also have new id 9 for the new state to be safe. But
- * regs with a different old id could still have new id 9, we don't care about
- * that.
- * So we look through our idmap to see if this old id has been seen before. If
- * so, we require the new id to match; otherwise, we add the id pair to the map.
- */
-static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
-{
- struct bpf_id_pair *map = idmap->map;
- unsigned int i;
-
- /* either both IDs should be set or both should be zero */
- if (!!old_id != !!cur_id)
- return false;
-
- if (old_id == 0) /* cur_id == 0 as well */
- return true;
-
- /* a name of what the caller has, see BPF_INHERITED_ID */
- if ((old_id | cur_id) >= BPF_INHERITED_ID)
- return old_id == cur_id;
-
- for (i = 0; i < idmap->cnt; i++) {
- if (map[i].old == old_id)
- return map[i].cur == cur_id;
- if (map[i].cur == cur_id)
- return false;
- }
-
- /* Reached the end of known mappings; haven't seen this id before */
- if (idmap->cnt < BPF_ID_MAP_SIZE) {
- map[idmap->cnt].old = old_id;
- map[idmap->cnt].cur = cur_id;
- idmap->cnt++;
- return true;
- }
-
- /*
- * idmap slots are bounded by the number of registers and stack slots.
- * Since referenced dynptrs acquire intermediate references that do
- * not live in either, so the map can be exhausted. Since it is unlikely,
- * fail the verification by treating the states as not equivalent.
- */
- return false;
-}
-
static void __clean_func_state(struct bpf_verifier_env *env,
struct bpf_func_state *st,
u16 live_regs, int frame)
@@ -195,107 +145,6 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
return 0;
}
-static bool regs_exact(const struct bpf_reg_state *rold,
- const struct bpf_reg_state *rcur,
- struct bpf_idmap *idmap)
-{
- return reg_value_equal(rold, rcur) &&
- check_ids(rold->id, rcur->id, idmap) &&
- check_ids(rold->parent_id, rcur->parent_id, idmap) &&
- check_ids(rold->map_uid, rcur->map_uid, idmap);
-}
-
-/* Returns true if (rold safe implies rcur safe) */
-static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
- struct bpf_reg_state *rcur, struct bpf_idmap *idmap)
-{
- if (rold->type == NOT_INIT)
- /* nothing can be done with it */
- return true;
-
- /* Enforce that register types have to match exactly, including their
- * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general
- * rule.
- *
- * One can make a point that using a pointer register as unbounded
- * SCALAR would be technically acceptable, but this could lead to
- * pointer leaks because scalars are allowed to leak while pointers
- * are not. We could make this safe in special cases if root is
- * calling us, but it's probably not worth the hassle.
- *
- * Also, register types that are *not* MAYBE_NULL could technically be
- * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE
- * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point
- * to the same map).
- * However, if the old MAYBE_NULL register then got NULL checked,
- * doing so could have affected others with the same id, and we can't
- * check for that because we lost the id when we converted to
- * a non-MAYBE_NULL variant.
- * So, as a general rule we don't allow mixing MAYBE_NULL and
- * non-MAYBE_NULL registers as well.
- */
- if (rold->type != rcur->type)
- return false;
-
- switch (base_type(rold->type)) {
- case SCALAR_VALUE:
- if (env->explore_alu_limits) {
- /* explore_alu_limits disables tnum_in() and range_within()
- * logic and requires everything to be strict
- */
- return reg_value_equal(rold, rcur);
- }
- return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off);
- case PTR_TO_MAP_KEY:
- case PTR_TO_MAP_VALUE:
- case PTR_TO_MEM:
- case PTR_TO_BUF:
- case PTR_TO_TP_BUFFER:
- case PTR_TO_STACK:
- case PTR_TO_BTF_ID:
- /* If the new min/max/var_off satisfy the old ones and
- * everything else matches, we are OK.
- */
- return reg_fixed_part_equal(rold, rcur) &&
- range_within(rold, rcur) &&
- tnum_in(rold->var_off, rcur->var_off) &&
- check_ids(rold->id, rcur->id, idmap) &&
- check_ids(rold->parent_id, rcur->parent_id, idmap) &&
- check_ids(rold->map_uid, rcur->map_uid, idmap);
- case PTR_TO_PACKET_META:
- case PTR_TO_PACKET:
- /* We must have at least as much range as the old ptr
- * did, so that any accesses which were safe before are
- * still safe. This is true even if old range < old off,
- * since someone could have accessed through (ptr - k), or
- * even done ptr -= k in a register, to get a safe access.
- */
- if (rold->range < 0 || rcur->range < 0) {
- /* special case for [BEYOND|AT]_PKT_END */
- if (rold->range != rcur->range)
- return false;
- } else if (rold->range > rcur->range) {
- return false;
- }
- /* id relations must be preserved */
- if (!check_ids(rold->id, rcur->id, idmap))
- return false;
- /* Preserve displacements between pointers sharing an ID. */
- if (rold->id && rold->r64.base != rcur->r64.base)
- return false;
- /* new val must satisfy old val knowledge */
- return range_within(rold, rcur) &&
- tnum_in(rold->var_off, rcur->var_off);
- case PTR_TO_ARENA:
- return true;
- case PTR_TO_INSN:
- return reg_fixed_part_equal(rold, rcur) &&
- range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off);
- default:
- return regs_exact(rold, rcur, idmap);
- }
-}
-
static struct bpf_reg_state unbound_reg;
static __init int unbound_reg_init(void)
@@ -466,129 +315,6 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
return true;
}
-/*
- * Compare stack arg slots between old and current states.
- * Outgoing stack args are a part of the state and must agree.
- */
-static bool stack_arg_safe(struct bpf_verifier_env *env, struct bpf_func_state *old,
- struct bpf_func_state *cur, struct bpf_idmap *idmap)
-{
- int i, nslots, cur_cnt = cur->out_stack_arg_cnt;
-
- /* incoming arguments are not compared with outgoing ones, the other way is ruled out */
- if (bpf_stack_args_are_incoming(cur) && !bpf_stack_args_are_incoming(old))
- cur_cnt = 0;
- nslots = max(old->out_stack_arg_cnt, cur_cnt);
- for (i = 0; i < nslots; i++) {
- struct bpf_reg_state *old_arg, *cur_arg;
- struct bpf_reg_state not_init = { .type = NOT_INIT };
-
- old_arg = i < old->out_stack_arg_cnt ?
- &old->stack_arg_regs[i] : ¬_init;
- cur_arg = i < cur_cnt ?
- &cur->stack_arg_regs[i] : ¬_init;
- if (!regsafe(env, old_arg, cur_arg, idmap))
- return false;
- }
-
- return true;
-}
-
-static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur,
- struct bpf_idmap *idmap)
-{
- int i;
-
- if (old->acquired_refs != cur->acquired_refs)
- return false;
-
- if (old->active_locks != cur->active_locks)
- return false;
-
- if (old->active_preempt_locks != cur->active_preempt_locks)
- return false;
-
- if (old->active_rcu_locks != cur->active_rcu_locks)
- return false;
-
- if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
- return false;
-
- if (!check_ids(old->active_lock_id, cur->active_lock_id, idmap) ||
- old->active_lock_ptr != cur->active_lock_ptr)
- return false;
-
- for (i = 0; i < old->acquired_refs; i++) {
- if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap) ||
- old->refs[i].type != cur->refs[i].type)
- return false;
- switch (old->refs[i].type) {
- case REF_TYPE_PTR:
- if (!check_ids(old->refs[i].parent_id, cur->refs[i].parent_id, idmap))
- return false;
- break;
- case REF_TYPE_IRQ:
- break;
- case REF_TYPE_LOCK:
- case REF_TYPE_RES_LOCK:
- case REF_TYPE_RES_LOCK_IRQ:
- if (old->refs[i].ptr != cur->refs[i].ptr)
- return false;
- break;
- default:
- WARN_ONCE(1, "Unhandled enum type for reference state: %d\n", old->refs[i].type);
- return false;
- }
- }
-
- return true;
-}
-
-/*
- * In a loop that is walked without a count a state of an earlier iteration stands for a state of
- * a later one, see flow_cycle_closed(). Never the other way round.
- *
- * In a loop that is counted and walked the state that gets to iteration max_header_count is
- * dropped: there is no such iteration at run time. The number in a state is the number of
- * iterations that the program has done, and only states with the same number are compared. They
- * differ in what is counted anyway.
- *
- * The walk of a loop with two entries is meant to get to the end of the loop, like today's walk of
- * all paths. A state that is within the state of the iteration before it is on its way there:
- * for (i = 7; i >= 0; i--) { if (i >= n) continue; ... }
- * i is in 0..6, then in 0..5. So here too only states with the same number are compared.
- *
- * Where the iterations are not kept apart a state stands for many of them, and the number says
- * nothing about what it describes.
- */
-static bool loop_stack_safe(struct bpf_verifier_env *env, struct bpf_func_state *old,
- struct bpf_func_state *cur)
-{
- struct loop_stack_entry *old_stack = old->loop_stack;
- struct loop_stack_entry *cur_stack = cur->loop_stack;
- u32 i;
-
- if (old->loop_stack_cnt != cur->loop_stack_cnt)
- return false;
-
- for (i = 0; i < old->loop_stack_cnt; i++) {
- if (old_stack[i].loop_id == cur_stack[i].loop_id &&
- old_stack[i].max_header_count == cur_stack[i].max_header_count &&
- old_stack[i].terminates == cur_stack[i].terminates &&
- old_stack[i].guarded == cur_stack[i].guarded &&
- old_stack[i].counted == cur_stack[i].counted &&
- old_stack[i].apart == cur_stack[i].apart &&
- old_stack[i].first == cur_stack[i].first &&
- (!old_stack[i].apart ||
- (old_stack[i].probe ? old_stack[i].trip <= cur_stack[i].trip :
- old_stack[i].trip == cur_stack[i].trip)))
- continue;
- return false;
- }
-
- return true;
-}
-
/* compare two verifier states
*
* The state that gets to an insn doesn't have to be looked at when there is
@@ -648,13 +374,6 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
return true;
}
-static void reset_idmap_scratch(struct bpf_verifier_env *env)
-{
- struct bpf_idmap *idmap = &env->idmap_scratch;
-
- idmap->cnt = 0;
-}
-
/* Everything that @cur describes is described by @old. */
bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
struct bpf_verifier_state *cur)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 48b8a790d6e..a4eca416dc2 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -8816,13 +8816,6 @@ static bool arg_type_is_mem_size(enum bpf_arg_type type)
return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO;
}
-/* A number that check_func_arg() takes only when it is a known constant */
-static bool arg_type_is_const(enum bpf_arg_type type)
-{
- return type == ARG_CONST_SCALAR || type == ARG_CONST_MEM_SIZE ||
- type == ARG_CONST_ALLOC_SIZE_OR_ZERO;
-}
-
static bool arg_type_is_raw_mem(enum bpf_arg_type type)
{
/*
@@ -12241,15 +12234,7 @@ static bool is_rbtree_lock_required_kfunc(u32 btf_id);
static void account_processed_insn(struct bpf_verifier_env *env)
{
- struct bpf_verifier_state *st = env->cur_state;
struct bpf_func_state *frame = cur_func(env);
- u32 fr, i;
-
- /* every walk on patience that the state is in pays for the insn */
- for (fr = 0; fr <= st->curframe; fr++)
- for (i = 0; i < st->frame[fr]->loop_stack_cnt; i++)
- if (bpf_loop_walk_may_give_up(&st->frame[fr]->loop_stack[i]))
- env->walk_insns[st->frame[fr]->loop_stack[i].walk_id]++;
env->insn_processed++;
frame->insns_subtotal++;
@@ -14009,13 +13994,6 @@ static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta)
return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]);
}
-/* void * in the prototype. What it returns points to an object of the kernel. */
-bool bpf_is_cast_kfunc(const struct bpf_call_arg_meta *meta)
-{
- return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) ||
- is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast]);
-}
-
static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta)
{
return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]);
@@ -15292,54 +15270,6 @@ bool bpf_insn_is_iter_next(struct bpf_verifier_env *env, struct bpf_insn *insn)
bpf_is_iter_next_kfunc(&meta);
}
-/*
- * check_func_arg() goes by the types of the arguments in the proto of a helper or a kfunc.
- * So does what has to be known ahead of the call: the registers with arguments of a kind.
- */
-static u16 call_arg_regs(struct bpf_verifier_env *env, struct bpf_insn *insn,
- bool (*is)(enum bpf_arg_type type))
-{
- const struct btf_func_model *fm = NULL;
- const struct bpf_func_proto *fn;
- struct bpf_kfunc_desc *desc;
- u32 i, slot = 0;
- u16 mask = 0;
-
- if (bpf_pseudo_kfunc_call(insn)) {
- desc = find_kfunc_desc(env, insn->imm, insn->off);
- if (!desc)
- return 0;
- fn = &desc->proto;
- fm = &desc->func_model;
- } else if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) {
- return 0;
- }
- for (i = 0; i < ARRAY_SIZE(fn->arg_type) && fn->arg_type[i] != ARG_UNUSED &&
- slot < MAX_BPF_FUNC_REG_ARGS; i++) {
- if (is(fn->arg_type[i]))
- mask |= BIT(BPF_REG_1 + slot);
- slot += fm ? btf_func_model_arg_slots(fm, i) : 1;
- }
- return mask;
-}
-
-/* The registers in which the helper or the kfunc takes nothing but a known constant */
-u16 bpf_call_const_arg_regs(struct bpf_verifier_env *env, struct bpf_insn *insn)
-{
- return call_arg_regs(env, insn, arg_type_is_const);
-}
-
-/* The registers in which it takes a number that has to be in a range */
-u16 bpf_call_checked_arg_regs(struct bpf_verifier_env *env, struct bpf_insn *insn)
-{
- if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_set_retval)
- return BIT(BPF_REG_1);
- /* without an exception callback the prog returns what bpf_throw() takes */
- if (bpf_is_throw_kfunc(insn) && !env->exception_callback_subprog)
- return BIT(BPF_REG_1);
- return call_arg_regs(env, insn, arg_type_is_mem_size);
-}
-
/*
* Describe a kfunc's stack access through argument slot @arg (0-based).
* As for helpers, must_write describes destruction of prior verifier state,
@@ -19318,25 +19248,6 @@ static void refine_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier
}
}
-/* It's known which way the jump at env->insn_idx goes */
-static void mark_loop_exit_decided(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
-{
- struct bpf_func_state *frame = st->frame[st->curframe];
- bool flag = bpf_flow_flag_regs(env, env->insn_idx);
- struct loop_stack_entry *entry;
- u32 i;
-
- for (i = 0; i < frame->loop_stack_cnt; i++) {
- entry = &frame->loop_stack[i];
- /* only because the states are kept apart. That holds for the first such jump. */
- if (entry->flag_apart && flag)
- entry->flag_apart = false;
- else if (entry->apart && entry->probe &&
- bpf_jump_leaves_loop(env, entry->loop_id, env->insn_idx))
- entry->decided = true;
- }
-}
-
/*
* One branch of the jump @insn is dead, the current state takes the other one. @reg1 and @reg2 are
* the operands as that branch refines them. They hold every value that takes it, whether the other
@@ -19454,8 +19365,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
env->true_reg1 = *dst_reg;
env->true_reg2 = *src_reg;
pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32);
- if (pred >= 0)
- mark_loop_exit_decided(env, this_branch);
if (pred == 1) {
/* Only follow the goto, ignore fall-through. If needed, push
* the fall-through branch for simulation under speculative
@@ -20888,92 +20797,6 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
return -EFAULT;
}
-static int flow_try_walk(struct bpf_verifier_env *env, struct loop_stack_entry *entry);
-
-/*
- * Push the loop entered at insn_idx onto the stack. Usually this adds a single
- * entry on top of its enclosing loop. However, an edge may enter an inner loop
- * directly, bypassing the header(s) of its enclosing loop(s), e.g.:
- *
- * 1: for (...): // enclosing loop, header at 1
- * 2: for (...): // inner loop, header at 2
- * ...
- * 3: if ...:
- * goto 2b; // enters loop 2 without going through header 1
- *
- * Such a bypass makes the enclosing loop irreducible, so its header is missing
- * from the stack and both headers (1) and (2) need to be pushed onto stack.
- * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds;
- * the bypassed ancestors are irreducible: walked, see flow_try_walk(), or pushed as
- * non-terminating.
- *
- * Assumes loop_stack_pop() has already truncated the stack to the common ancestor
- * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack.
- */
-static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_func_state *frame = cur_func(env);
- struct loop_stack_entry *loop_stack = frame->loop_stack;
- u32 missing_headers[LOOP_STACK_SIZE];
- u32 cnt = frame->loop_stack_cnt;
- u32 num_missing = 0;
- int h, err;
-
- *pushed = false;
-
- for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) {
- if (cnt && loop_stack[cnt - 1].loop_id == h)
- break;
- if (num_missing == LOOP_STACK_SIZE)
- goto e2big;
- missing_headers[num_missing] = h;
- }
-
- if (num_missing == 0)
- return 0;
- if (cnt + num_missing > LOOP_STACK_SIZE)
- goto e2big;
- frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing,
- sizeof(*frame->loop_stack));
- if (!frame->loop_stack)
- return -ENOMEM;
- loop_stack = frame->loop_stack;
-
- for (; num_missing; num_missing--, cnt++) {
- h = missing_headers[num_missing - 1];
- /* wherever it is entered: a loop can have more than one entry */
- if (!aux[h].loop->exits_cnt) {
- verbose(env, "loop at insn %d never exits\n", h);
- return -EINVAL;
- }
- loop_stack[cnt] = (struct loop_stack_entry){
- .loop_id = h,
- .refs = env->cur_state->acquired_refs,
- .entry_id = ++env->loop_entry_cnt,
- };
- /* there is no count for a loop that is not entered at its head */
- if (h != env->insn_idx && !aux[h].loop->rt_bounded) {
- frame->loop_stack_cnt = cnt;
- err = flow_try_walk(env, &loop_stack[cnt]);
- if (err < 0)
- return err;
- if (!err)
- bpf_flow_loop_gets_guards(env, h);
- }
- loop_stack[cnt].guarded = aux[h].loop->guarded;
- if (env->log.level & BPF_LOG_LEVEL2)
- verbose(env, "entering loop %d\n", h);
- *pushed = true;
- }
- frame->loop_stack_cnt = cnt;
- return 0;
-
-e2big:
- verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx);
- return -E2BIG;
-}
-
/*
* An exit from a loop can cross several nested loops, e.g.:
*
@@ -21030,319 +20853,6 @@ static int maybe_clamp_scev_regs(struct bpf_verifier_env *env)
return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry, false);
}
-/*
- * The current state came back to the head of a loop. If it has a reference more than the states
- * that are there it is neither within one of them nor joined with one. It goes around again:
- * bpf_for(i, 0, n) { if (!p) p = bpf_ringbuf_reserve(...); ... }
- * That ends when the loop stops taking references, or when there is no place left to keep them.
- * A reference that no live register and no live stack slot holds cannot be released any more.
- * There is nothing to wait for.
- *
- * Only what was taken in the loop is looked at. New references are added at the end. What callers
- * hold is not in the state of a function that is verified on its own.
- */
-static int check_refs_taken_in_loop(struct bpf_verifier_env *env,
- const struct loop_stack_entry *entry)
-{
- u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR) | (1 << STACK_ITER);
- struct bpf_verifier_state *st = env->cur_state;
- struct bpf_stack_state *stack;
- struct bpf_func_state *frame;
- struct bpf_reg_state *reg;
- bool held;
- int i, j, id;
-
- for (i = entry->refs; i < st->acquired_refs; i++) {
- if (st->refs[i].type != REF_TYPE_PTR)
- continue;
- id = st->refs[i].id;
- held = false;
- for (j = 0; j < st->acquired_refs; j++)
- if (st->refs[j].type == REF_TYPE_PTR && st->refs[j].parent_id == id)
- held = true;
- bpf_for_each_reg_in_vstate_mask(st, frame, reg, stack, mask, ({
- if (reg->id == id || reg->parent_id == id)
- held = true;
- }));
- if (held)
- continue;
- verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", id, st->refs[i].insn_idx);
- verbose(env, "nothing holds it in the next iteration of the loop at insn %d\n",
- entry->loop_id);
- return -EINVAL;
- }
- return 0;
-}
-
-/* How many insns it may take to go through the iterations of a loop one by one. */
-#define BPF_FLOW_MAX_WALK 100000
-/* The same when it is not known that the walk gets to the end of the loop. */
-#define BPF_FLOW_MAX_TRY 4096
-
-/* How many times the current insn is looked at because of the loops around it that are walked. */
-static u64 flow_walk_factor(const struct bpf_verifier_state *st)
-{
- const struct loop_stack_entry *e;
- u64 factor = 1;
- u32 fr, i;
-
- for (fr = 0; fr <= st->curframe; fr++) {
- for (i = 0; i < st->frame[fr]->loop_stack_cnt; i++) {
- e = &st->frame[fr]->loop_stack[i];
- /* the walk of a loop with two entries pays as it goes, see env->walk_insns */
- if (e->counted || e->probe)
- factor = min_t(u64, factor * max(e->max_header_count, 1U), U32_MAX);
- }
- }
- return factor;
-}
-
-/*
- * The current state enters the loop of @entry, for which there is no count. It's found out on the
- * way whether the loop has an end: the walk goes on for as long as the jumps that leave the loop
- * are decided. Returns 1 if the iterations are kept apart from now on, 0 if that takes too long.
- */
-static int flow_try_walk(struct bpf_verifier_env *env, struct loop_stack_entry *entry)
-{
- struct bpf_loop *loop = env->insn_aux_data[entry->loop_id].loop;
- u64 factor = flow_walk_factor(env->cur_state);
- u32 patience, *walk_insns;
-
- patience = env->allow_uninit_stack && !loop->rt_bounded &&
- (!loop->has_calls || loop->has_guards) ? BPF_FLOW_MAX_TRY : 0;
- if (loop->walk_failed)
- patience = 0;
- /* Two trips for a loop of any length. Many a loop is left in the second one. */
- if (patience && factor == 1)
- patience = max_t(u64, patience, 2ULL * loop->nr_insns);
- if (factor * loop->nr_insns > patience)
- return 0;
- entry->max_header_count = max(patience / (u32)(factor * loop->nr_insns), 1U);
- /*
- * A compiler makes a flag of the counter of
- * for (i = 0; i < 2; i++) off += f(i);
- * and there is nothing to count. In the second trip the flag decides the jump.
- * A loop with calls gets these two trips, whatever they take, and no more: its size says
- * nothing about the functions that it calls. It gets may_goto at once, so that nothing is
- * done again when the second trip comes back to the head.
- */
- if (loop->has_calls) {
- entry->max_header_count = min(entry->max_header_count, 2U);
- patience = U32_MAX;
- bpf_flow_loop_gets_guards(env, entry->loop_id);
- entry->guarded = true;
- }
- entry->terminates = true;
- /* scalar evolution has nothing to say about a loop with two entries */
- entry->probe = !loop->irreducible;
- entry->walk_id = env->walk_cnt + 1;
- if (is_power_of_2(entry->walk_id)) {
- walk_insns = kvrealloc(env->walk_insns, 2 * entry->walk_id * sizeof(u32),
- BPF_VERIFIER_GFP);
- if (!walk_insns)
- return -ENOMEM;
- env->walk_insns = walk_insns;
- }
- env->walk_cnt++;
- env->walk_insns[entry->walk_id] = 0;
- entry->walk_patience = patience;
- entry->apart = true;
- return 1;
-}
-
-/*
- * The current state gets to env->insn_idx. Leave the loops that the insn is not in.
- * @enters: the insn is the head of a loop that the state is not in yet.
- */
-int bpf_flow_loop_leave(struct bpf_verifier_env *env, bool *enters)
-{
- struct bpf_func_state *frame = cur_func(env);
- int err;
-
- err = loop_stack_pop(env);
- if (err)
- return err;
- *enters = env->insn_aux_data[env->insn_idx].loop &&
- (!frame->loop_stack_cnt ||
- frame->loop_stack[frame->loop_stack_cnt - 1].loop_id != env->insn_idx);
- return 0;
-}
-
-/*
- * Enter the loops that env->insn_idx is in.
- *
- * When the number of iterations follows from the state that enters a loop at its head
- * induction variables get the ranges that they have in that many iterations, and the loop
- * needs no may_goto.
- * The rest of the state is left as it is. What comes back to the head is joined with it.
- * @ends_walk: the state came back to the head and is the last one that is kept apart from
- * the states of other iterations.
- */
-int bpf_flow_loop_enter(struct bpf_verifier_env *env, bool *backedge, bool *drop, bool *ends_walk)
-{
- struct bpf_verifier_state *cur = env->cur_state, *entry_state;
- const struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];
- struct bpf_func_state *frame = cur_func(env);
- struct loop_stack_entry *entry;
- struct bpf_loop_iters iters;
- struct bpf_loop *loop;
- bool pushed, decided;
- u32 trip;
- u64 per_trip;
- int err;
-
- err = loop_stack_push(env, &pushed);
- if (err)
- return err;
-
- if (frame->loop_stack_cnt == 0)
- return 0;
-
- entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
- if (env->insn_idx != entry->loop_id) {
- /* the jump that counts didn't leave the loop */
- if (entry->entry_state && !entry->apart && cur->last_insn_idx == entry->count_insn)
- return bpf_clamp_scev_regs(env, frame, entry->loop_id, entry, true);
- return 0;
- }
-
- /*
- * No edge goes from a call to itself: the call goes on with one function of a set, see
- * split_func_set(), in the same iteration.
- */
- if (!pushed && cur->last_insn_idx == env->insn_idx &&
- BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL)
- return 0;
-
- /* The callback came back to the helper that calls it, in the same iteration. */
- if (frame->callback_depth)
- return 0;
-
- /* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */
- if (!pushed) {
- if (!entry->apart) {
- entry->trip++;
- *backedge = true;
- return check_refs_taken_in_loop(env, entry) ?: maybe_clamp_scev_regs(env);
- }
- /*
- * The first iteration is over. What the loop makes anew in every iteration is
- * in this state. The rest is what the count says, like at the entry.
- */
- if (entry->first) {
- entry->first = false;
- entry->apart = false;
- entry->trip++;
- err = bpf_compute_loop_iters(env, entry->entry_state, &iters);
- if (err < 0)
- return err;
- if (verifier_bug_if(!err, env, "loop at %d is not counted any more",
- env->insn_idx))
- return -EFAULT;
- err = maybe_clamp_scev_regs(env);
- if (err)
- return err;
- err = bpf_widen_scev_regs(env, cur, entry->entry_state, &iters, true);
- return err < 0 ? err : 0;
- }
- loop = env->insn_aux_data[env->insn_idx].loop;
- /* the next iteration is another kind of state, there is no cycle */
- trip = entry->trip;
- decided = !entry->probe || entry->decided || !trip;
- entry->decided = false;
- entry->flag_apart = false;
- if (decided && ++entry->trip < entry->max_header_count &&
- (entry->counted ||
- env->walk_insns[entry->walk_id] <= entry->walk_patience))
- return 0;
- /* there is no such iteration at run time */
- if (entry->counted) {
- *drop = true;
- return 0;
- }
- /*
- * The walk doesn't get to the end of the loop. From now on what comes back to
- * the head is joined with what is there. The loop gets may_goto.
- */
- if (!loop->rt_bounded && !loop->has_guards) {
- verbose(env, "loop at insn %d never exits\n", env->insn_idx);
- return -EINVAL;
- }
- if (!loop->rt_bounded)
- bpf_flow_loop_gets_guards(env, env->insn_idx);
- /* the jumps were decided, it's the loop that is too long. Or it had its two trips */
- if (decided || loop->has_calls)
- loop->walk_failed = true;
- /* the iteration that it went through: its state is at the head */
- entry->trip = trip;
- *ends_walk = true;
- return 0;
- }
-
- loop = env->insn_aux_data[env->insn_idx].loop;
- err = bpf_compute_loop_iters(env, cur, &iters);
- if (err < 0)
- return err;
-
- /*
- * There is no summary of what the loop does to something that is live.
- * Go through the iterations one by one if that is not too long.
- * @per_trip: the insns of one iteration, times the walks around the loop.
- */
- per_trip = flow_walk_factor(cur) * loop->nr_insns;
- if (err == 2 && per_trip <= BPF_FLOW_MAX_WALK &&
- iters.max_header_count <= BPF_FLOW_MAX_WALK / (u32)per_trip) {
- entry->max_header_count = iters.max_header_count;
- entry->terminates = true;
- entry->counted = true;
- entry->apart = true;
- return 0;
- }
- if (err == 2 && iters.spill_base)
- err = 0;
- /* The walk has to get to the end of the loop, so it is known to have one. Or not: */
- if (!err) {
- err = flow_try_walk(env, entry);
- if (err)
- return err < 0 ? err : 0;
- if (!loop->rt_bounded && !loop->has_guards) {
- verbose(env, "loop at insn %d never exits\n", env->insn_idx);
- return -EINVAL;
- }
- if (!loop->rt_bounded) {
- bpf_flow_loop_gets_guards(env, env->insn_idx);
- entry->guarded = true;
- }
- return 0;
- }
-
- /* What the ranges are derived from for as long as the loop is looked at. */
- entry_state = bpf_flow_keep_state(env, cur);
- if (!entry_state)
- return -ENOMEM;
-
- entry->min_header_count = iters.min_header_count;
- entry->max_header_count = iters.max_header_count;
- entry->iv_reg = iters.iv_reg;
- entry->iv_range = iters.iv_range;
- entry->count_insn = iters.count_insn;
- entry->entry_state = entry_state;
- entry->terminates = true;
- /*
- * idx = 0; for (i = 0; i < 100000; i++) idx = table[idx];
- * What the loop makes of idx is not known before its body is looked at, and what idx is
- * at the entry goes with i == 0 only. So the state goes through the first iteration as
- * it is. The state that comes back to the head has idx in 0..255.
- */
- if (!bitmap_empty(iters.fresh_regs, BPF_SCEV_REGS_NUM)) {
- entry->first = true;
- entry->apart = true;
- return 0;
- }
- err = bpf_widen_scev_regs(env, cur, entry_state, &iters, false);
- return err < 0 ? err : 0;
-}
-
/* Can the main program return 0? */
static bool zero_is_valid_retval(struct bpf_verifier_env *env)
{
@@ -21728,35 +21238,6 @@ static bool round_is_dropped(struct bpf_verifier_env *env)
return (env->restart && bpf_flow_loops_lost_bound(env)) || env->guards_late;
}
-static int do_check(struct bpf_verifier_env *env)
-{
- bool level1 = (env->log.level & BPF_LOG_LEVEL) == BPF_LOG_LEVEL1;
- u64 log_pos = env->log.end_pos;
- int prev_insn_idx = -1, insn_idx;
- int err;
-
- err = bpf_flow_deliver(env, env->cur_state, env->insn_idx, -1);
- while (!err) {
- err = bpf_flow_next(env, &prev_insn_idx);
- if (err == -ENOENT)
- return 0;
- if (err)
- break;
- /* What is known about how the state came to be is not kept with it. */
- bpf_diag_event_log_restore(env, 0);
- /* Neither is the text: the level 1 log has one basic block at a time. */
- if (level1) {
- bpf_vlog_reset(&env->log, log_pos);
- env->prev_linfo = NULL;
- }
- insn_idx = env->insn_idx;
- err = do_check_block(env, prev_insn_idx);
- if (err && level1 && !round_is_dropped(env))
- bpf_flow_print_way(env, log_pos, insn_idx, prev_insn_idx);
- }
- return err;
-}
-
static int find_btf_percpu_datasec(struct btf *btf)
{
const struct btf_type *t;
@@ -23106,44 +22587,6 @@ static int do_check_main(struct bpf_verifier_env *env)
return ret;
}
-/*
- * What stops a loop is assumed before the main pass and checked in it, see flow_is_bound(). The
- * main prog and global functions may get to the same loop. It has may_goto for all of them or
- * for none, so all are done again when the assumption doesn't hold, whatever came out.
- * The same when a loop gets may_goto, or its may_goto gets another way out, and what went through
- * the loop before is not there any more, see bpf_flow_loop_gets_guards().
- *
- * A round that is dropped counts as paths that never run. What it found out about the insns, the
- * subprogs and the prog stays. Each of these is made for more than one state that gets to an insn:
- * it grows to the safe side, or another value is refused, or the last one that gets there sets it.
- * Where the round got to is dropped with its verdict: the passes that follow take an insn that was
- * seen for one that passed its checks, and a function that is called has to be safe.
- */
-static int do_check_all(struct bpf_verifier_env *env)
-{
- struct bpf_func_info_aux *sub_aux;
- u64 log_pos = env->log.end_pos;
- int i, ret;
-
- for (;;) {
- bpf_flow_new_round(env);
- ret = do_check_main(env);
- ret = ret ?: do_check_subprogs(env);
- if (!round_is_dropped(env))
- return ret;
- /* the log is dropped when a function is found to be safe */
- bpf_vlog_reset(&env->log, min(log_pos, env->log.end_pos));
- for (i = 0; i < env->prog->len; i++)
- env->insn_aux_data[i].seen = 0;
- for (i = 1; env->prog->aux->func_info && i < env->subprog_cnt; i++) {
- sub_aux = subprog_aux(env, i);
- memset(sub_aux->called, 0, sizeof(sub_aux->called));
- memset(sub_aux->verified, 0, sizeof(sub_aux->verified));
- sub_aux->called_with_pkt_ptrs = false;
- }
- }
-}
-
/*
* What it took to verify a static subprog on its own is not in insns_total of its callers.
* Add it, once per caller. The main prog and global subprogs have all of it already.
@@ -25175,10 +24618,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
- ret = bpf_flow_compute_kinds(env);
- if (ret < 0)
- goto skip_full_check;
-
if (env->log.level & BPF_LOG_LEVEL2)
log_program(env);
@@ -25338,9 +24777,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
kvfree(env->kfunc_index.slots);
kvfree(env->insn_array_maps);
bpf_free_scev(env);
- kvfree(env->walk_insns);
bpf_flow_free(env);
- bpf_join_free(env);
bpf_stack_liveness_free(env);
kvfree(env->cfg.postorder_nums);
kvfree(env->cfg.insn_postorder);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 03/49] bpf: Delete the old main pass of the verifier
2026-11-20 7:41 ` [PATCH bpf-next 03/49] bpf: Delete the old main pass of the verifier Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 16707 bytes --]
> bpf: Delete the old main pass of the verifier
TL;DR: Deleting the old verifier pass leaves callers behind in verifier.c,
states.c, flow.c and scalar_widen(), so with CONFIG_BPF_SYSCALL=y the kernel
may not build or link, possibly through the whole series, breaking bisection.
> The patch only deletes. Later patches add the new code as whole
> functions. The kernel builds again with "bpf: Verify in passes".
No commit with the subject "bpf: Verify in passes" exists in
120103063a0c..a50a112fb866, and no commit in that range defines the deleted
functions again. Is the series meant to leave 18 commits that cannot build
with CONFIG_BPF_SYSCALL=y? Documentation/process/submitting-patches.rst
asks that the kernel builds and runs properly after each patch, and git
bisect can stop on any of them.
Could the deletion be folded into the patch that adds the replacement
pass, or could the old functions be kept until then? The individual
breakages are below.
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
[ ... ]
> -/* Where may_goto that the verifier adds goes, see check_loop_guard() */
> -enum bpf_guard_edge {
> - BPF_NO_GUARD_EDGE,
> - BPF_GUARD_LEAVES_SUBPROG,
> - BPF_GUARD_LEAVES_LOOP,
> -};
> -
> -struct loop_stack_entry {
> - struct bpf_verifier_state *entry_state;
> - u32 min_header_count;
> - u32 max_header_count;
> - u32 loop_id:23;
> - u32 terminates:1;
> - /* the loop had may_goto when the state entered it, see check_loop_guard() */
> - u32 guarded:1;
[ ... ]
> -/* The state is in a walk on patience, which may not get to the end of the loop */
> -static inline bool bpf_loop_walk_may_give_up(const struct loop_stack_entry *entry)
> -{
> - return entry->apart && entry->terminates && !entry->counted && !entry->first;
> -}
> -
Does the code that is left in verifier.c still use these? Both
struct bpf_verifier_stack_elem and check_loop_guard() still do:
enum bpf_guard_edge guard_edge;
if (!entry->guarded && !bpf_loop_walk_may_give_up(entry))
if (aux->guard_edge_dropped && loop->guarded)
env->head->guard_edge = BPF_GUARD_LEAVES_SUBPROG;
so the enum is an incomplete type, and the helper, the aux bit, the
bpf_loop field and the enum values are all gone. copy_func_state() also
uses sizeof(*src->loop_stack) on the deleted struct loop_stack_entry, and
print_verifier_state() in log.c reads state->loop_stack[i].apart. Later
commits (98b796be1fd1 "bpf: Peel the first trips of a loop and join the
other trips" and 885b6e36bec3 "bpf: Delete scalar evolution") fix some of
this, but the loop_stack_entry that 98b796be1fd1 adds back has no
terminates, guarded or loop_id member, so check_loop_guard() still does not
compile at a50a112fb866.
[ ... ]
> -/* The places that the table of a loop knows: registers, then stack slots, fp-8 first */
> -#define BPF_LOOP_SLOT 64
> -#define BPF_LOOP_PLACES 128
> -
> -/*
> - * Where the bounds of the number in @place may come to rest at the head of a loop: the lowest and
> - * the highest upper bound, the highest and the lowest lower bound that the jumps of the loop
> - * let come back to the head, see flow_compute_rungs().
> - */
> -struct bpf_loop_rungs {
> - s64 max[2];
> - s64 min[2];
> - bool has_max;
> - bool has_min;
> - u8 place;
> -};
> -
[ ... ]
> @@ -1621,44 +1505,19 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
> bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
> struct bpf_verifier_state *cur);
>
> -/* join.c */
> -int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
> - const struct bpf_verifier_state *a, const struct bpf_verifier_state *b,
> - bool widen);
> -void bpf_join_free(struct bpf_verifier_env *env);
> -
> /* flow.c */
> int bpf_flow_prepare(struct bpf_verifier_env *env);
> void bpf_flow_reset(struct bpf_verifier_env *env);
> void bpf_flow_free(struct bpf_verifier_env *env);
> -bool bpf_flow_is_point(struct bpf_verifier_env *env, int insn_idx);
> -struct bpf_verifier_state *bpf_flow_keep_state(struct bpf_verifier_env *env,
> - const struct bpf_verifier_state *st);
> -int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
> - int insn_idx, int prev_insn_idx);
> -int bpf_flow_guard_edge(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
> - int insn_idx, int prev_insn_idx, enum bpf_guard_edge edge);
Is it intended to keep the scalar_widen() prototype that names
struct bpf_loop_rungs? With the struct definition gone, the struct is first
named inside the prototype's parameter list:
void scalar_widen(struct scalar *s, const struct scalar *old,
const struct bpf_loop_rungs *rungs, int linear);
so every file that includes bpf_verifier.h gets a "declared inside parameter
list" warning (an error with CONFIG_WERROR). The definition in verifier.c
conflicts with the prototype and dereferences an incomplete type:
if (rungs && rungs->has_max && max <= rungs->max[1])
stop = min(stop, max <= rungs->max[0] ? rungs->max[0] : rungs->max[1]);
The only caller was join_range() in join.c, which this commit deletes, and
no later commit up to a50a112fb866 touches scalar_widen() or
struct bpf_loop_rungs. The commit message lists "bounds for widening
flow_compute_rungs()" and "the join" as deleted, so this is old code that a
later patch cannot simply add to. Should scalar_widen() and its prototype
be deleted here, or struct bpf_loop_rungs kept?
The same removal leaves two other join-only helpers with no callers through
the end of the series: bpf_state_subsumes() in states.c (used only by
join_check()) and the static inline bpf_stack_args_are_incoming() in
bpf_verifier.h (used only by join_func()). They compile, but could be
deleted together with scalar_widen() if they are not meant for the new join.
Also, the prototypes of bpf_flow_prepare(), bpf_flow_reset(),
bpf_flow_free(), bpf_flow_call() and bpf_flow_exit() stay, while the
definitions are removed from flow.c. verifier.c still calls
bpf_flow_guard_edge(), bpf_flow_deliver() and bpf_flow_is_point() from
flow_drain() and do_check_block(), whose prototypes are removed here, so
those are implicit declarations. The other five fail at link time.
[ ... ]
> bool restart; /* see bpf_flow_next_is_no_bound() */
> bool guards_late; /* see bpf_flow_loop_gets_guards() */
This isn't a bug, but bpf_flow_loop_gets_guards() and bpf_flow_new_round()
are deleted from flow.c, so this comment now points at a function that does
not exist. The comment above check_loop_guard() in verifier.c has the same
problem:
* Every state that goes through the loop does, see bpf_flow_loop_gets_guards().
Also, bpf_flow_new_round() was the only code that cleared env->guards_late
and env->restart, and loop->guarded is deleted from struct bpf_loop.
check_loop_guard() still writes env->guards_late, but its only reader,
round_is_dropped(), has no caller after do_check_all() goes away, and the
same is true of bpf_flow_loops_lost_bound(). Both fields are write-only
now. Neither comment is touched by any commit up to a50a112fb866.
> diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
> --- a/kernel/bpf/flow.c
> +++ b/kernel/bpf/flow.c
[ ... ]
> @@ -32,157 +8,11 @@
>
> #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
>
> -/*
> - * States at an insn that have no join: kinds of values, references, chains of calls.
> - * A state that arrives is compared with those that it may be related to, see flow_may_relate().
> - * At worst that is all of them, and the work is the square of it.
> - */
> -#define BPF_FLOW_MAX_STATES 1024
[ ... ]
Does flow_compute_order() still compile? This hunk removes the definition
of struct bpf_flow, but the function is kept and still does:
static int flow_compute_order(struct bpf_verifier_env *env, struct bpf_flow *flow)
...
flow->pos[n] = pos++;
struct bpf_flow is only forward-declared through the
"struct bpf_flow *flow;" member of struct bpf_verifier_env, so this
dereferences an incomplete type. Its only caller, bpf_flow_prepare(), is
deleted below, so the static function is also unused (an error with
CONFIG_WERROR). struct flow_kept and the jhash.h and rbtree.h includes
have no users left either, which does not match "the code that builds the
log from kept states" being deleted.
[ ... ]
> @@ -517,2895 +305,3 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
> kvfree(seen);
> return ret;
> }
> -
> -/* The slot that @insn spills to or fills from, if it's one that struct flow_need has a bit for */
> -static int flow_fp_slot(const struct bpf_insn *insn, u8 base)
> -{
[ ... ]
This hunk deletes the definitions of bpf_flow_prepare(), bpf_flow_reset(),
bpf_flow_free(), bpf_flow_is_point(), bpf_flow_guard_edge(),
bpf_flow_deliver(), bpf_flow_call(), bpf_flow_exit() and
bpf_flow_is_needed(). Their callers are still in the tree:
bpf_check(): ret = bpf_flow_prepare(env);
bpf_check(): bpf_flow_free(env);
free_states(): bpf_flow_reset(env);
flow_drain(): err = bpf_flow_guard_edge(env, &elem->st, ...
flow_drain(): err = bpf_flow_deliver(env, &elem->st, ...
do_check_block(): bpf_flow_is_point(env, env->insn_idx)
call_func_on_its_own(): err = bpf_flow_call(env, state, &entry, subprog, insn_idx);
exit_func_on_its_own(): err = bpf_flow_exit(env, state, env->insn_idx);
scev.c:bpf_compute_loop_iters(): if (!call && !bpf_flow_is_needed(env, insn_idx, r))
Does flow.o link with CONFIG_BPF_SYSCALL=y? The five functions that keep
their prototypes are undefined references, and the others are implicit
declarations. Only the scev.c reference goes away, in 885b6e36bec3
("bpf: Delete scalar evolution"). At a50a112fb866, verifier.c still calls
all of the others and none is defined.
> diff --git a/kernel/bpf/join.c b/kernel/bpf/join.c
> --- a/kernel/bpf/join.c
> +++ b/kernel/bpf/join.c
> @@ -1,627 +1,6 @@
[ ... ]
Is the join deletion complete? join_range() was the only caller of
scalar_widen(), and this commit also deletes struct bpf_loop_rungs, but
scalar_widen() stays in verifier.c:
void scalar_widen(struct scalar *s, const struct scalar *old,
const struct bpf_loop_rungs *rungs, int linear)
{
...
((max > cnum64_smax(old->r64) && !(rungs && rungs->has_max && max <= rungs->max[1])) ||
(min < cnum64_smin(old->r64) && !(rungs && rungs->has_min && min >= rungs->min[1])))) {
...
if (rungs && rungs->has_max && max <= rungs->max[1])
stop = min(stop, max <= rungs->max[0] ? rungs->max[0] : rungs->max[1]);
Nothing defines the struct, so the prototype declares it at parameter scope
and verifier.c dereferences an incomplete type, which is a hard error
whenever CONFIG_BPF_SYSCALL=y. No later commit in the range touches
scalar_widen() or bpf_loop_rungs. Should scalar_widen(), its prototype and
the @rungs/@linear kernel-doc go together with join_range(), or should
struct bpf_loop_rungs be kept if the new pass is meant to call it?
> diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
> --- a/kernel/bpf/states.c
> +++ b/kernel/bpf/states.c
[ ... ]
> @@ -648,13 +374,6 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
> return true;
> }
>
> -static void reset_idmap_scratch(struct bpf_verifier_env *env)
> -{
> - struct bpf_idmap *idmap = &env->idmap_scratch;
> -
> - idmap->cnt = 0;
> -}
> -
> /* Everything that @cur describes is described by @old. */
> bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
> struct bpf_verifier_state *cur)
Can states.c compile after this? The commit deletes check_ids(),
regs_exact(), regsafe(), stack_arg_safe(), refsafe(), loop_stack_safe() and
reset_idmap_scratch(), but keeps their callers:
stacksafe(): regsafe(), check_ids()
func_states_equal(): check_ids(), regsafe(),
stack_arg_safe(env, old, cur, &env->idmap_scratch),
loop_stack_safe(env, old, cur)
bpf_state_subsumes(): reset_idmap_scratch(env),
refsafe(old, cur, &env->idmap_scratch)
None of these names is defined anywhere after this commit, and
scripts/Makefile.warn sets -Werror=implicit-function-declaration, so
states.o fails to build. range_within() also loses its only caller
(regsafe()), which gives -Wunused-function until b63f08629edd ("bpf: Add
equations and their arithmetic") adds scalar_subsumes().
At a50a112fb866 ("bpf: Delete the equations that say nothing") stacksafe(),
func_states_equal() and bpf_state_subsumes() still call
check_ids(), regsafe(), stack_arg_safe(), loop_stack_safe(),
reset_idmap_scratch() and refsafe(), and none of them is defined.
The break looks avoidable: once flow.c and join.c lose their callers,
bpf_state_subsumes() has no callers left, and the chain
bpf_state_subsumes() -> func_states_equal() -> stacksafe() is unreachable
and is the only user of the deleted helpers. The commit message says it
deletes "the compare of registers and references". Could that chain be
deleted together with its helpers, or the helpers kept until the patch that
rewires bpf_state_subsumes()?
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -23104,44 +22585,6 @@ static int do_check_main(struct bpf_verifier_env *env)
> return ret;
> }
>
> -/*
> - * What stops a loop is assumed before the main pass and checked in it, see flow_is_bound(). The
> [ ... ]
> -static int do_check_all(struct bpf_verifier_env *env)
> -{
> - struct bpf_func_info_aux *sub_aux;
> - u64 log_pos = env->log.end_pos;
> - int i, ret;
> -
> - for (;;) {
> - bpf_flow_new_round(env);
> - ret = do_check_main(env);
> - ret = ret ?: do_check_subprogs(env);
> - if (!round_is_dropped(env))
> - return ret;
[ ... ]
> -}
> -
> /*
> * What it took to verify a static subprog on its own is not in insns_total of its callers.
> * Add it, once per caller. The main prog and global subprogs have all of it already.
Does bpf_check() still compile? do_check() and do_check_all() are deleted,
but their callers remain:
do_check_common(): ret = do_check(env);
bpf_check(): ret = bpf_flow_prepare(env);
bpf_check(): ret = do_check_all(env);
bpf_check(): bpf_flow_free(env);
No header declares do_check() or do_check_all(), so verifier.o fails with
an implicit declaration error, and bpf_flow_prepare()/bpf_flow_free() are
undefined at link time. The same goes for bpf_flow_deliver() in
do_check_block(), and for bpf_flow_reset(), bpf_flow_call() and
bpf_flow_exit() in free_states(), call_func_on_its_own() and
exit_func_on_its_own(). With do_check_all() gone, do_check_main(),
do_check_subprogs(), round_is_dropped() and do_check_block() have no
callers left, and the comment above round_is_dropped() ("see
do_check_all()") points at a function that no longer exists. At
a50a112fb866 verifier.c still calls do_check(), do_check_all(),
bpf_flow_prepare() and bpf_flow_free(), and none is defined.
[ ... ]
Is check_loop_guard() meant to keep working? This commit removes the walk
accounting from account_processed_insn(), but check_loop_guard() is left
as it was and still calls bpf_loop_walk_may_give_up(entry) and reads
entry->terminates, entry->guarded and entry->loop_id. Those are gone with
struct loop_stack_entry, so the function is undeclared and the type is
incomplete. iter_stops_loops(), loop_stack_pop() and
maybe_clamp_scev_regs() also still use struct loop_stack_entry members,
and loop_stack_pop() and maybe_clamp_scev_regs() lose their only callers
(bpf_flow_loop_leave() and bpf_flow_loop_enter()). At a50a112fb866 the
check `if (!entry->guarded && !bpf_loop_walk_may_give_up(entry))` is still
in check_loop_guard() and the function is not defined anywhere; the
struct loop_stack_entry restored at the tip has no loop_id, terminates or
guarded member. A smaller split would be to delete the old pass in the
commit that adds its replacement, or to stub the remaining call sites
(check_loop_guard(), do_check(), do_check_all()).
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 04/49] bpf: Delete scalar evolution
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (2 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 03/49] bpf: Delete the old main pass of the verifier Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:03 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load Alexei Starovoitov
` (44 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
scev.c computes an expression for each register at each insn, before
the main pass. The expression is the same for all paths. It has three
users:
what code
linked registers at a jump refine_linked_regs()
ranges in a loop bpf_widen_scev_regs(), the clamps
trip count bpf_compute_loop_iters()
The trip count is a proof with a case for each form of a loop:
compute_backedge_taken_count() <, <=, >, >=, != and how each wraps
bound_of_zext32() a 64-bit jump on a 32-bit counter
bpf_compute_leave_conds() a condition is stored, a later
jump tests it and leaves
fresh_regs, mask_regs, ... values that are not linear
Nothing checks the count. If it is too small, the verifier does not
check a trip that runs.
Later patches replace all of that with one mechanism: equations in the
state. A join guesses them, and a prune checks them. Delete scev.c, its
callers, and the helpers that only it uses.
What is not a plain deletion:
- Only scev.c needs all exits and all latches of a loop. The other
users need the first one. exit_to replaces exits[], latch replaces
latches[]. MAX_LOOP_EXITS and MAX_LATCHES go away.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 131 +-
include/linux/cnum.h | 3 -
include/linux/tnum.h | 2 -
kernel/bpf/Makefile | 2 +-
kernel/bpf/cnum.c | 37 -
kernel/bpf/const_fold.c | 265 +-
kernel/bpf/fixups.c | 1 -
kernel/bpf/flow.c | 11 +-
kernel/bpf/liveness.c | 18 -
kernel/bpf/log.c | 18 +-
kernel/bpf/loops.c | 267 +-
kernel/bpf/scev.c | 5504 ----------------------------------
kernel/bpf/tnum.c | 11 -
kernel/bpf/verifier.c | 324 +-
14 files changed, 59 insertions(+), 6535 deletions(-)
delete mode 100644 kernel/bpf/scev.c
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b0f520c168d..894b8890f85 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -113,8 +113,6 @@ struct bpf_reg_state {
* the dynptr to determine proper reference freeing.
* Similarly to dynptrs, we use ID to track "belonging" of a reference
* to a specific instance of bpf_iter.
- * A scalar has no id. The scalars that it is linked to are static: computed before the
- * main pass, see bpf_scev_next_linked_reg().
*/
u32 id;
union {
@@ -384,33 +382,6 @@ struct bpf_retval_range {
bool return_32bit;
};
-/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */
-#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS)
-
-struct bpf_loop_iters {
- u32 min_header_count; /* min number of times header is executed */
- u32 max_header_count; /* max number of times header is executed */
- /* live, changed by the loop, no summary of how: left to the join at the head */
- unsigned long forget_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- /*
- * live, made anew in every iteration: the first one tells what it is in the rest.
- * Or not set at the entry, and also in forget_regs: the first one sets it.
- */
- unsigned long fresh_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- /* [0, max] at the entry, and the loop only clears bits */
- unsigned long mask_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- /*
- * The variable that the latches of all back edges test, or -1, and what
- * it can be at the loop header: what it is at the entry or what one of
- * the latches lets through. A 32-bit latch bounds its low 32 bits only.
- */
- int iv_reg;
- struct scalar iv_range;
- /* a jump that counts: it leaves the loop in the last iteration, if not before. Or -1 */
- int count_insn;
- /* a pointer to the stack that stores go through moves in the loop */
- bool spill_base;
-};
#define LOOP_STACK_SIZE 16
@@ -733,68 +704,25 @@ void bpf_call_graph_free(struct bpf_call_graph *cg);
___idx++)
#define MAX_BACKEDGES 16
-#define MAX_LOOP_EXITS 256
-
-/* Number of exit conditions dominating a backedge. */
-#define MAX_LATCHES 4
struct bpf_backedge {
int from;
- /*
- * Conditional jumps within the loop that dominate 'from' and have
- * a successor outside the loop, nearest first. latches[0] is the
- * latch; the remaining entries are additional exit conditions.
- * After scalar evolution: those that test a counter, see bpf_scev_jump_counts().
- * One of them may leave the loop by way of a flag, see bpf_edge_leaves_loop().
- * Each usable condition tells iterations in which the back edge is
- * not taken, so keeping only a subset is sound.
- */
- int latches[MAX_LATCHES];
- u8 latches_cnt;
- /* bit k: the loop is left on the edge of latches[k] that is taken, not on the other one */
- u8 taken_leaves;
-};
-
-struct bpf_loop_exit {
- int from; /* instruction inside the loop */
- int to; /* instruction outside the loop */
+ /* the nearest conditional jump of the loop that dominates 'from' and leaves it, or -1 */
+ int latch;
};
struct bpf_loop {
struct bpf_backedge backedges[MAX_BACKEDGES];
- /* edges exiting from this loop, includes edges from nested loops */
- struct bpf_loop_exit *exits;
+ int exit_to; /* where the first edge that leaves the loop goes, or -1 */
int backedges_cnt;
- int exits_cnt;
bool irreducible;
bool backedges_overflow;
- /* @exits is cut at MAX_LOOP_EXITS. Only a loop around this one needs all of them. */
- bool exits_overflow;
- /* @exits of a loop in this one is cut. Then @exits of this loop can lack exits too. */
- bool nested_exits_overflow;
/* every trip goes through an insn that stops the loop, see flow_is_bound() */
bool rt_bounded;
/* insns in the loop, nested loops included */
u32 nr_insns;
/* not rt_bounded: there are places for may_goto, see guard_to in bpf_insn_aux_data */
bool has_guards;
- /*
- * Loop-entry registers used to compute stack addresses for loads,
- * stores and calls requiring fixed stack offsets, in this and nested
- * loops.
- */
- unsigned long store_base_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- /*
- * Loop-entry registers used to compute the operands of conditional jumps, in this and
- * nested loops.
- */
- unsigned long tested_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- /*
- * what gets a constant added in every iteration, in 64 and in 32 bits. A sum that is
- * not among tested_regs is taken for that too: nothing stops it either
- */
- unsigned long linear_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
- unsigned long linear32_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)];
};
struct bpf_insn_aux_data {
@@ -853,9 +781,6 @@ struct bpf_insn_aux_data {
u64 guard_at_header:1; /* there is no better place than the head of the loop */
u64 guard_no_unwind:1; /* it cannot unwind from here, at least sometimes */
u64 guard_unset_reg:1; /* where it goes a register is read that is not set here */
- u64 need_scev:1;
- /* a jump in a loop: the edge that is not taken, bit 0, or the other one leaves the loop */
- u64 edge_leaves:2;
u64 alu_state:5; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill
* pattern for a bpf_fastcall call.
@@ -891,14 +816,12 @@ struct bpf_insn_aux_data {
/*
* Bitmask of R0-R9 that hold known values at this instruction.
* const_reg_mask: scalar constants that fit in 32 bits.
- * const_reg_wide_mask: scalar constants that don't, there is no val.
* const_reg_map_mask: map pointers, val is map_index into used_maps[].
* const_reg_subprog_mask: subprog pointers, val is subprog number.
* const_reg_vals[i] holds the 32-bit value for register i.
* Populated by compute_const_regs() pre-pass.
*/
u16 const_reg_mask;
- u16 const_reg_wide_mask;
u16 const_reg_map_mask;
u16 const_reg_subprog_mask;
u32 const_reg_vals[10];
@@ -1112,7 +1035,6 @@ struct bpf_idmap {
};
struct bpf_liveness;
-struct scev;
struct bpf_fd_array {
union {
@@ -1340,9 +1262,6 @@ struct bpf_verifier_env {
struct bpf_iarray *succ;
struct bpf_iarray *gotox_tmp_buf;
int *idoms;
- /* per insn, while bpf_compute_leave_conds() runs */
- struct bpf_leave_cond *leave_conds;
- struct scev *scev;
struct bpf_flow *flow;
};
@@ -1407,8 +1326,6 @@ int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual);
__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
u32 insn_off,
const char *prefix_fmt, ...);
-void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num);
-void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num);
#define verifier_bug_if(cond, env, fmt, args...) \
({ \
@@ -1520,10 +1437,6 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
int bpf_insert_loop_guards(struct bpf_verifier_env *env);
bool bpf_insn_is_iter_next(struct bpf_verifier_env *env, struct bpf_insn *insn);
bool bpf_insn_leaves_loop(struct bpf_verifier_env *env, int h, int n);
-int bpf_compute_leave_conds(struct bpf_verifier_env *env);
-bool bpf_edge_leaves_loop(struct bpf_verifier_env *env, int h, int n, int t);
-int bpf_find_latches(struct bpf_verifier_env *env, bool flags);
-bool bpf_jump_leaves_loop(struct bpf_verifier_env *env, int h, int n);
int bpf_insert_recursion_checks(struct bpf_verifier_env *env);
int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
@@ -1861,7 +1774,6 @@ u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t);
int bpf_find_subprog(struct bpf_verifier_env *env, int off);
bool bpf_is_throw_kfunc(struct bpf_insn *insn);
-bool bpf_is_dynptr_slice_rdwr_kfunc(struct bpf_insn *insn);
int bpf_compute_const_regs(struct bpf_verifier_env *env);
int bpf_prune_dead_branches(struct bpf_verifier_env *env);
int bpf_check_cfg(struct bpf_verifier_env *env);
@@ -2026,26 +1938,15 @@ void scalar_set_unknown(struct scalar *s);
void scalar_set_const(struct scalar *s, u64 imm);
void scalar_add(struct scalar *dst, const struct scalar *a, const struct scalar *b);
void scalar_alu(struct scalar *dst, const struct scalar *src, const struct bpf_insn *insn);
-void scalar_zext32(struct scalar *s);
-void scalar_ext(struct scalar *s, u32 bits, bool sign);
bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits);
void scalar_union(struct scalar *acc, const struct scalar *src);
-int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
- const struct scalar *range);
-bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b);
void scalar_widen(struct scalar *s, const struct scalar *old,
const struct bpf_loop_rungs *rungs, int linear);
-int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc,
- const struct bpf_reg_state *src);
-
int bpf_stack_liveness_init(struct bpf_verifier_env *env);
void bpf_stack_liveness_free(struct bpf_verifier_env *env);
int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
-const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx);
-bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx);
bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);
bool bpf_stack_slot_filled(struct bpf_verifier_env *env, u32 frameno, u32 spi);
-bool bpf_half_slot_live_before(struct bpf_verifier_env *env, u32 insn_idx, u32 half_spi);
int bpf_compute_live_registers(struct bpf_verifier_env *env);
#define BPF_MAP_KEY_POISON (1ULL << 63)
@@ -2182,31 +2083,5 @@ int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
-int bpf_init_scev(struct bpf_verifier_env *env);
-void bpf_free_scev(struct bpf_verifier_env *env);
-int bpf_compute_scev(struct bpf_verifier_env *env);
-int bpf_scev_jump_counts(struct bpf_verifier_env *env, int header, int n, bool taken_leaves);
-/* What is known about a scalar or about the offset of a pointer. */
-struct scev_value {
- struct scalar;
- int ptr_reg; /* SCEV register identifying the pointer origin, or -1. */
- u8 bits; /* The value is known modulo 2^bits. 0: nothing is known. */
- bool refined; /* It depends on a register that the jump refined. */
-};
-bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, struct bpf_func_state *st, int insn_idx,
- int next_idx, u16 refined_regs, u32 *pos, u32 *linked_regno,
- u32 *fill_off, struct scev_value *val);
-bool bpf_scev_mem_addr(struct bpf_verifier_env *env, int insn_idx, u32 regno, u32 *base_regno,
- s64 *off);
-bool bpf_is_widenable_reg_type(const struct bpf_reg_state *reg);
-bool bpf_scev_in_slice_slots(struct bpf_verifier_env *env, int off, u32 size, bool fixed_off);
-
-int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- struct bpf_loop_iters *iters);
-int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters,
- bool back);
-int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 insn_idx,
- const struct loop_stack_entry *entry, bool behind);
#endif /* _LINUX_BPF_VERIFIER_H */
diff --git a/include/linux/cnum.h b/include/linux/cnum.h
index e00148d352e..d31e5fcb2cb 100644
--- a/include/linux/cnum.h
+++ b/include/linux/cnum.h
@@ -84,9 +84,6 @@ bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner);
struct cnum32 cnum32_from_cnum64(struct cnum64 cnum);
struct cnum64 cnum64_zext32(struct cnum32 cnum);
-struct cnum64 cnum64_sext32(struct cnum32 cnum);
-struct cnum64 cnum64_lshift32(struct cnum32 cnum);
-struct cnum32 cnum32_rshift32(struct cnum64 cnum);
struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b);
struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step);
diff --git a/include/linux/tnum.h b/include/linux/tnum.h
index 16682248f96..ca2cfec8de0 100644
--- a/include/linux/tnum.h
+++ b/include/linux/tnum.h
@@ -134,6 +134,4 @@ static inline bool tnum_subreg_is_const(struct tnum a)
/* Returns the smallest member of t larger than z */
u64 tnum_step(struct tnum t, u64 z);
-u32 tnum_alignment(struct tnum a);
-
#endif /* _LINUX_TNUM_H */
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index d26c62326e5..4e5bb513a8a 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse
endif
CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy)
-obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o scev.o
+obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c
index e64c09a37cc..a298753158d 100644
--- a/kernel/bpf/cnum.c
+++ b/kernel/bpf/cnum.c
@@ -31,43 +31,6 @@ struct cnum64 cnum64_zext32(struct cnum32 cnum)
return cnum64_from_urange(cnum32_umin(cnum), cnum32_umax(cnum));
}
-/*
- * The sign extension of each value of 'cnum'.
- * An arc that crosses S32_MAX/S32_MIN gives [S32_MIN, S32_MAX].
- */
-struct cnum64 cnum64_sext32(struct cnum32 cnum)
-{
- if (cnum32_is_empty(cnum))
- return CNUM64_EMPTY;
- return cnum64_from_srange(cnum32_smin(cnum), cnum32_smax(cnum));
-}
-
-/*
- * The smallest arc with { x << 32 : x in 'cnum' }. The circle of 2^32 numbers is laid on the circle
- * of 2^64 numbers: sums, the order of signed and of unsigned numbers and the places where they wrap
- * stay.
- */
-struct cnum64 cnum64_lshift32(struct cnum32 cnum)
-{
- if (cnum32_is_empty(cnum))
- return CNUM64_EMPTY;
- return (struct cnum64){ .base = (u64)cnum.base << 32, .size = (u64)cnum.size << 32 };
-}
-
-/* { x >> 32 : x in 'cnum' } */
-struct cnum32 cnum32_rshift32(struct cnum64 cnum)
-{
- u32 base = cnum.base >> 32;
- u32 size = ((cnum.base + cnum.size) >> 32) - base;
-
- if (cnum64_is_empty(cnum))
- return CNUM32_EMPTY;
- /* less than 1 << 32 numbers are missing, or the arc ends one before its base */
- if (cnum.size >> 32 == U32_MAX || size == U32_MAX)
- return CNUM32_UNBOUNDED;
- return (struct cnum32){ .base = base, .size = size };
-}
-
/*
* Suppose 'a' and 'b' are laid out as follows:
*
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index 1fd97f624b9..bf81bd54232 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -8,7 +8,7 @@
* instruction. Tracks 64-bit constant values in R0-R9 through the program,
* using a fixed-point iteration in reverse postorder. Records which registers
* hold known constants and their values in
- * env->insn_aux_data[].{const_reg_mask, const_reg_wide_mask, const_reg_vals}.
+ * env->insn_aux_data[].{const_reg_mask, const_reg_vals}.
*/
enum const_arg_state {
@@ -301,7 +301,7 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
/* Save computed constants into insn_aux[]: the value if it fits into 32-bit */
for (i = 0; i < insn_cnt; i++) {
- u16 mask = 0, wide_mask = 0, map_mask = 0, subprog_mask = 0;
+ u16 mask = 0, map_mask = 0, subprog_mask = 0;
struct bpf_insn_aux_data *aux = &insn_aux[i];
struct const_arg_info *ci = ci_in[i];
@@ -312,10 +312,8 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
case CONST_ARG_CONST: {
u64 val = c->val;
- if (val != (u32)val) {
- wide_mask |= BIT(r);
+ if (val != (u32)val)
break;
- }
mask |= BIT(r);
aux->const_reg_vals[r] = val;
break;
@@ -333,7 +331,6 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
}
}
aux->const_reg_mask = mask;
- aux->const_reg_wide_mask = wide_mask;
aux->const_reg_map_mask = map_mask;
aux->const_reg_subprog_mask = subprog_mask;
}
@@ -360,262 +357,6 @@ static int eval_const_branch(u8 opcode, u64 dst_val, u64 src_val)
}
}
-/* 1: the registers @ci take the conditional jump @insn, 0: they don't, -1: not known */
-static int const_branch_taken(const struct bpf_insn *insn, const struct const_arg_info *ci)
-{
- u64 dst = ci[insn->dst_reg].val, src = insn->imm;
-
- if (!ci_is_const(&ci[insn->dst_reg]))
- return -1;
- if (BPF_SRC(insn->code) == BPF_X) {
- if (!ci_is_const(&ci[insn->src_reg]))
- return -1;
- src = ci[insn->src_reg].val;
- }
- /* see bpf_prune_dead_branches() */
- if (BPF_CLASS(insn->code) == BPF_JMP32) {
- dst = (s32)dst;
- src = (s32)src;
- }
- return eval_const_branch(insn->code, dst, src);
-}
-
-/*
- * When is the program that stands in front of an insn of a loop sure to leave the loop?
- * Always, or when one register holds a number of a set, whatever the other registers hold.
- * A set has one bit per range of numbers of 32 bits. A flag is a small number: each of 0 - 31
- * is a range. An error code or a size is not: 32 - 63, 64 - 127, ... are the other ranges.
- */
-struct bpf_leave_cond {
- u64 vals[MAX_BPF_REG];
- bool always;
-};
-
-#define LEAVE_SMALL 32
-#define LEAVE_RANGES (LEAVE_SMALL + 32 - ilog2(LEAVE_SMALL))
-
-static const struct bpf_leave_cond leave_never;
-static const struct bpf_leave_cond leave_always = {
- .vals = { [0 ... MAX_BPF_REG - 1] = U64_MAX },
- .always = true,
-};
-
-static u64 leave_val_bit(u64 val)
-{
- if (val > U32_MAX)
- return 0;
- return BIT_ULL(val < LEAVE_SMALL ? val : ilog2(val) + LEAVE_SMALL - ilog2(LEAVE_SMALL));
-}
-
-/* The condition for the loop at @h in front of @t, where an insn of that loop goes on */
-static const struct bpf_leave_cond *leave_cond_at(struct bpf_verifier_env *env, int h, int t)
-{
- int l = bpf_loop_at_index(env, t);
-
- if (l != h)
- return bpf_is_nested_loop(env, l, h) ? &leave_never : &leave_always;
- return t == h || !env->leave_conds ? &leave_never : &env->leave_conds[t];
-}
-
-/* The same when the jump at @n goes on at @t: always, if the constants at @n meet it */
-static const struct bpf_leave_cond *leave_cond_edge(struct bpf_verifier_env *env, int h, int n,
- int t)
-{
- const struct bpf_insn_aux_data *aux = &env->insn_aux_data[n];
- const struct bpf_leave_cond *p = leave_cond_at(env, h, t);
- int r;
-
- for (r = 0; r < ARRAY_SIZE(aux->const_reg_vals); r++)
- if ((aux->const_reg_mask & BIT(r)) &&
- (p->vals[r] & leave_val_bit(aux->const_reg_vals[r])))
- return &leave_always;
- return p;
-}
-
-/* Is what the insn at @idx makes of @reg == @val in the set @vals? @ci: nothing is known. */
-static bool insn_makes(struct bpf_verifier_env *env, struct const_arg_info *ci, int idx,
- int reg, u64 val, u64 vals)
-{
- struct const_arg_info unknown = { .state = CONST_ARG_UNKNOWN, .val = 0 };
- struct bpf_insn *insns = env->prog->insnsi;
- struct const_arg_info *dst = &ci[insns[idx].dst_reg];
- bool ret;
-
- if (reg >= 0) {
- ci[reg].state = CONST_ARG_CONST;
- ci[reg].val = val;
- }
- const_reg_xfer(env, ci, &insns[idx], insns, idx);
- ret = ci_is_const(dst) && (vals & leave_val_bit(dst->val));
- *dst = unknown;
- if (reg >= 0)
- ci[reg] = unknown;
- return ret;
-}
-
-/* @p holds behind the insn at @idx. Make it what holds in front of it. */
-static void leave_cond_insn(struct bpf_verifier_env *env, int idx, struct bpf_leave_cond *p)
-{
- struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[idx];
- int r, v, dst = insn->dst_reg, src = insn->src_reg;
- struct const_arg_info ci[MAX_BPF_REG] = {};
- u8 class = BPF_CLASS(insn->code);
- u64 vals;
-
- /* Nothing is known about a register that is no constant although all were constants */
- for (r = 0; r < MAX_BPF_REG; r++)
- ci[r].state = CONST_ARG_CONST;
- const_reg_xfer(env, ci, insn, insns, idx);
- for (r = 0; r < MAX_BPF_REG; r++) {
- if (!ci_is_const(&ci[r]))
- p->vals[r] = 0;
- ci[r].state = CONST_ARG_UNKNOWN;
- }
- if ((class != BPF_ALU && class != BPF_ALU64 && class != BPF_LD) ||
- dst >= MAX_BPF_REG || !p->vals[dst])
- return;
- vals = p->vals[dst];
- p->vals[dst] = 0;
- if (insn_makes(env, ci, idx, -1, 0, vals)) {
- *p = leave_always;
- return;
- }
- for (v = 0; v < LEAVE_SMALL && class != BPF_LD; v++) {
- if (insn_makes(env, ci, idx, dst, v, vals))
- p->vals[dst] |= BIT_ULL(v);
- if (BPF_SRC(insn->code) == BPF_X && src != dst && src < MAX_BPF_REG &&
- insn_makes(env, ci, idx, src, v, vals))
- p->vals[src] |= BIT_ULL(v);
- }
-}
-
-/*
- * const_branch_taken() for all of @lo - @hi in the register that @insn compares with a number.
- * The range has numbers of 32 bits of one sign. So '<' and the like treat all of them alike
- * if they treat both ends alike. '==' and '!=' single out the number that they compare with.
- * '&': @lo is the bit that all of them have, @hi has the bits that any of them has.
- */
-static int range_branch_taken(const struct bpf_insn *insn, struct const_arg_info *ci, u64 lo,
- u64 hi)
-{
- u64 vals[] = { lo, hi, clamp_t(u64, (u32)insn->imm, lo, hi) };
- int i, taken[ARRAY_SIZE(vals)];
-
- for (i = 0; i < ARRAY_SIZE(vals); i++) {
- ci[insn->dst_reg].val = vals[i];
- taken[i] = const_branch_taken(insn, ci);
- }
- return taken[0] == taken[1] && taken[0] == taken[2] ? taken[0] : -1;
-}
-
-/*
- * @p is what is enough on both edges of the jump at @idx. If the jump compares a register
- * with a number then a number in the register goes one way: enough on that edge is enough.
- */
-static void leave_cond_jump(struct bpf_verifier_env *env, int h, int idx,
- struct bpf_leave_cond *p)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- struct const_arg_info ci[MAX_BPF_REG] = {};
- u64 t, f, lo, go = 0, stay = 0;
- int i, taken;
-
- if (BPF_SRC(insn->code) != BPF_K)
- return;
- t = leave_cond_edge(env, h, idx, idx + insn->off + 1)->vals[insn->dst_reg];
- f = leave_cond_edge(env, h, idx, idx + 1)->vals[insn->dst_reg];
- if (t == f)
- return;
- ci[insn->dst_reg].state = CONST_ARG_CONST;
- for (i = 0; i < LEAVE_RANGES; i++) {
- lo = i < LEAVE_SMALL ? i : BIT_ULL(i - LEAVE_SMALL + ilog2(LEAVE_SMALL));
- taken = range_branch_taken(insn, ci, lo, i < LEAVE_SMALL ? lo : 2 * lo - 1);
- if (taken > 0)
- go |= BIT_ULL(i);
- else if (!taken)
- stay |= BIT_ULL(i);
- }
- p->vals[insn->dst_reg] |= (t & go) | (f & stay);
-}
-
-/*
- * A compiler makes
- * w1 = 0; if w7 > 255 goto l; ...; w1 = 1; l: ...; w1 &= 1; if w1 != 0 goto head;
- * of 'do { ...; more = ++i < 256 && ...; ...; } while (more);'. The jump that leaves the loop
- * looks at 1 or 0. The jump that counts stays in the loop on both edges.
- *
- * So find for every insn of a loop when the loop is left from there on. Go against the edges:
- * if w1 != 0 goto head w1 is 0 the numbers that don't go to the head
- * w1 &= 1 w1 is even the numbers that the insn turns into 0
- * l: ... w1 is even nothing writes w1
- * w1 = 1 never 1 is not even
- * w1 is 0 at the jump that counts and 0 is even: the edge to l leaves the loop.
- *
- * An insn that is not in the loop: always. The head: never. More than one successor: what is
- * enough for each of them. Every insn has one turn, in post order. A cycle without the head is
- * a nested loop, so the successors in the loop itself had their turn. One that didn't: never.
- * The jumps on the way know constants too: 'r7 = 0; if w1 != 0 goto a; ...; a: if r7 == 2 goto
- * head' leaves the loop when w1 is not 0.
- *
- * Every edge that the program has at run time counts, the edges of may_goto that the verifier
- * adds too. May_goto of the loop and of the loops around it leaves the loop. A nested loop can
- * get one in front of a jump that the numbers decide the other way. So a nested loop: never.
- *
- * What is kept is the answer for the two edges of each jump: insn_aux_data->edge_leaves.
- */
-int bpf_compute_leave_conds(struct bpf_verifier_env *env)
-{
- const struct bpf_leave_cond *q;
- struct bpf_leave_cond *p;
- struct bpf_iarray *succ;
- int i, r, s, h, idx;
- bool jump;
- u64 any;
-
- env->leave_conds = kvcalloc(env->prog->len, sizeof(*p), BPF_VERIFIER_GFP);
- if (!env->leave_conds)
- return -ENOMEM;
- for (i = 0; i < env->cfg.cur_postorder; i++) {
- idx = env->cfg.insn_postorder[i];
- h = bpf_loop_at_index(env, idx);
- if (h < 0)
- continue;
- p = &env->leave_conds[idx];
- *p = leave_always;
- jump = bpf_insn_is_cond_jump(env->prog->insnsi[idx].code);
- succ = bpf_insn_successors(env, idx);
- iarray_for_each(s, succ) {
- q = jump ? leave_cond_edge(env, h, idx, s) : leave_cond_at(env, h, s);
- for (r = 0; r < MAX_BPF_REG; r++)
- p->vals[r] &= q->vals[r];
- p->always &= q->always;
- if (jump && q->always)
- env->insn_aux_data[idx].edge_leaves |= BIT(s != idx + 1);
- }
- if (jump) {
- leave_cond_jump(env, h, idx, p);
- continue;
- }
- for (r = 0, any = 0; r < MAX_BPF_REG; r++)
- any |= p->vals[r];
- if (any && !p->always)
- leave_cond_insn(env, idx, p);
- }
- kvfree(env->leave_conds);
- env->leave_conds = NULL;
- return 0;
-}
-
-/*
- * Does the program leave the loop at @h when the conditional jump at @n goes on at @t?
- * @h is the innermost loop around @n.
- */
-bool bpf_edge_leaves_loop(struct bpf_verifier_env *env, int h, int n, int t)
-{
- return leave_cond_at(env, h, t)->always ||
- (env->insn_aux_data[n].edge_leaves & BIT(t != n + 1));
-}
-
/*
* Rewrite conditional branches with constant outcomes into unconditional
* jumps using register values resolved by bpf_compute_const_regs() pass.
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index aca3c7a53b6..bd47a38c7bc 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -754,7 +754,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
for (i = start; i < end; i++) {
if (aux_data[i].loop) {
- kvfree(aux_data[i].loop->exits);
kvfree(aux_data[i].loop);
aux_data[i].loop = NULL;
}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 79d1be515f2..ac5e7c26c2a 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -153,8 +153,7 @@ static void flow_place_guards(struct bpf_verifier_env *env, int h, u32 *seen)
*/
if (loop->irreducible) {
for (i = 0; i < loop->backedges_cnt; i++)
- if (!loop->backedges[i].latches_cnt ||
- loop->backedges[i].latches[0] != loop->backedges[i].from)
+ if (loop->backedges[i].latch != loop->backedges[i].from)
goto header;
for (i = 0; i < loop->backedges_cnt; i++)
flow_set_guard(env, h, loop->backedges[i].from);
@@ -177,10 +176,10 @@ static void flow_place_guards(struct bpf_verifier_env *env, int h, u32 *seen)
}
for (i = 0; i < loop->backedges_cnt; i++)
- if (!loop->backedges[i].latches_cnt)
+ if (loop->backedges[i].latch < 0)
goto header;
for (i = 0; i < loop->backedges_cnt; i++)
- flow_set_guard(env, h, loop->backedges[i].latches[0]);
+ flow_set_guard(env, h, loop->backedges[i].latch);
return;
header:
@@ -197,9 +196,9 @@ static void flow_place_guards(struct bpf_verifier_env *env, int h, u32 *seen)
for (o = h; aux[o].loop_header >= 0; o = aux[o].loop_header)
;
/* a loop that has no exit can only be stopped together with the program */
- if (!aux[o].loop->exits_cnt)
+ t = aux[o].loop->exit_to;
+ if (t < 0)
return;
- t = aux[o].loop->exits[0].to;
aux[h].guard_place = true;
aux[h].guard_at_header = true;
aux[h].guard_to = t;
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index bd186d3c201..57c37ebae00 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -795,24 +795,6 @@ static void compute_may_write_masks(struct bpf_verifier_env *env)
}
}
-/* live_stack_before of the insn at @insn_idx for one half-slot */
-bool bpf_half_slot_live_before(struct bpf_verifier_env *env, u32 insn_idx, u32 half_spi)
-{
- struct func_instance *instance;
- int bkt;
-
- hash_for_each(env->liveness->func_instances, bkt, instance, hl_node)
- if (insn_idx - instance->subprog_start < instance->insn_cnt &&
- is_live_before(instance, insn_idx, instance->depth, half_spi, FM_LIVE_BEFORE))
- return true;
- return false;
-}
-
-const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx)
-{
- return env->insn_aux_data[insn_idx].may_write_mask;
-}
-
/* print use/def slots for all instances ordered by callsite first, then by depth */
static int print_instances(struct bpf_verifier_env *env)
{
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 20820098c6b..e58202955f1 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -527,7 +527,7 @@ static void verbose_unum(struct bpf_verifier_env *env, u64 num)
verbose(env, "%#llx", num);
}
-void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num)
+static void verbose_snum(struct bpf_verifier_env *env, s64 num)
{
if (is_snum_decimal(num))
verbose(env, "%lld", num);
@@ -535,14 +535,6 @@ void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num)
verbose(env, "%#llx", num);
}
-void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num)
-{
- if (is_snum_decimal(num))
- verbose(env, "%d", num);
- else
- verbose(env, "%#x", (u32)num);
-}
-
int tnum_strn(char *str, size_t size, struct tnum a)
{
/* print as a constant, if tnum is fully known */
@@ -617,7 +609,7 @@ static void print_scalar_ranges(struct bpf_verifier_env *env,
}
if (m1->name[0] == 's')
- bpf_verbose_snum(env, m1->val);
+ verbose_snum(env, m1->val);
else
verbose_unum(env, m1->val);
}
@@ -638,7 +630,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
t = reg->type;
if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) {
- bpf_verbose_snum(env, reg->var_off.value);
+ verbose_snum(env, reg->var_off.value);
return;
}
@@ -649,7 +641,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
if (state->frameno != reg->frameno)
verbose(env, "[%d]", reg->frameno);
if (tnum_is_const(reg->var_off)) {
- bpf_verbose_snum(env, reg->var_off.value);
+ verbose_snum(env, reg->var_off.value);
return;
}
}
@@ -687,7 +679,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
/* a pointer register with fixed offset */
if (reg->var_off.value) {
verbose_a("imm=");
- bpf_verbose_snum(env, reg->var_off.value);
+ verbose_snum(env, reg->var_off.value);
}
} else {
print_scalar_ranges(env, reg, &sep);
diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
index 11019768505..8a63bb1799a 100644
--- a/kernel/bpf/loops.c
+++ b/kernel/bpf/loops.c
@@ -237,32 +237,9 @@ static void add_backedge(struct bpf_verifier_env *env, int from, int h)
return;
}
loop->backedges[cnt].from = from;
- loop->backedges[cnt].latches_cnt = 0;
loop->backedges_cnt++;
}
-static int add_exit(struct bpf_loop *loop, int from, int to)
-{
- if (loop->exits_overflow)
- return 0;
- if (loop->exits_cnt == MAX_LOOP_EXITS) {
- loop->exits_overflow = true;
- return 0;
- }
- if (!loop->exits) {
- loop->exits = kvzalloc_objs(*loop->exits, MAX_LOOP_EXITS,
- BPF_VERIFIER_GFP);
- if (!loop->exits)
- return -ENOMEM;
- }
- loop->exits[loop->exits_cnt] = (struct bpf_loop_exit) {
- .from = from,
- .to = to,
- };
- loop->exits_cnt++;
- return 0;
-}
-
static int mark_as_header(struct bpf_verifier_env *env, int h)
{
struct bpf_insn_aux_data *aux = env->insn_aux_data;
@@ -306,32 +283,6 @@ static int assign_header(struct bpf_verifier_env *env, struct loops_dfs *dfs, in
return 0;
}
-static bool is_cond_jmp_insn(struct bpf_insn *insn)
-{
- u8 class = BPF_CLASS(insn->code);
- u8 opcode = BPF_OP(insn->code);
-
- if (class != BPF_JMP && class != BPF_JMP32)
- return false;
-
- switch (opcode) {
- case BPF_JEQ:
- case BPF_JGE:
- case BPF_JGT:
- case BPF_JLE:
- case BPF_JLT:
- case BPF_JNE:
- case BPF_JSET:
- case BPF_JSGE:
- case BPF_JSGT:
- case BPF_JSLE:
- case BPF_JSLT:
- return true;
- default:
- return false;
- }
-}
-
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx)
{
struct bpf_insn_aux_data *aux = env->insn_aux_data;
@@ -340,100 +291,23 @@ int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx)
}
/*
- * Collect conditional jumps within n's loop that dominate 'n' and leave
- * the loop on one edge, see bpf_edge_leaves_loop(), nearest first.
- * A jump that cannot count the iterations takes no place, see bpf_scev_jump_counts().
+ * The backedge->from and its latch can reside in an inner loop of @h, e.g.:
+ *
+ * 1: r7 += 1;
+ * 2: r6 += 1;
+ * if r7 == 2 goto 1b;
+ * if r6 < 2 goto 2b;
+ *
+ * For now, let's assume that latches are unknown for such cases.
+ * (GCC/LLVM handle this by inserting artificial cfg nodes).
*/
-static int find_dominating_conditions(struct bpf_verifier_env *env, int n,
- struct bpf_backedge *backedge)
+static int find_latch(struct bpf_verifier_env *env, int h, int n)
{
- struct bpf_insn *insns = env->prog->insnsi;
- bool t_exits_loop, f_exits_loop;
- int *idoms = env->idoms;
- int n_loop, counts;
-
- n_loop = bpf_loop_at_index(env, n);
- if (idoms_intersect(env, n, n_loop) != n_loop)
- return 0;
- while (n >= 0) {
- if (is_cond_jmp_insn(&insns[n]) && bpf_loop_at_index(env, n) == n_loop) {
- t_exits_loop = bpf_edge_leaves_loop(env, n_loop, n, n + insns[n].off + 1);
- f_exits_loop = bpf_edge_leaves_loop(env, n_loop, n, n + 1);
- /* latches[0] is the latch: it has to leave the loop itself */
- if ((t_exits_loop || f_exits_loop) &&
- (backedge->latches_cnt || bpf_jump_leaves_loop(env, n_loop, n))) {
- counts = !backedge->latches_cnt ? 1 :
- bpf_scev_jump_counts(env, n_loop, n, t_exits_loop);
- if (counts < 0)
- return counts;
- if (!counts)
- goto next;
- if (backedge->latches_cnt == MAX_LATCHES)
- return 0;
- if (t_exits_loop)
- backedge->taken_leaves |= BIT(backedge->latches_cnt);
- backedge->latches[backedge->latches_cnt++] = n;
- }
- }
-next:
- if (n == n_loop)
- break;
- n = idoms[n];
- }
- return 0;
-}
-
-/*
- * Without @flags: the jumps that leave a loop themselves. That tells where the verifier may add
- * may_goto. It takes the constants that hold with these edges to follow a flag.
- * Once more after scalar evolution, which knows the jumps that count. The flags that were found
- * are remembered.
- */
-int bpf_find_latches(struct bpf_verifier_env *env, bool flags)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_backedge *backedge;
- struct bpf_loop *loop;
- int i, j, k, err;
-
- if (flags) {
- err = bpf_compute_leave_conds(env);
- if (err)
- return err;
- }
- for (i = 0; i < env->prog->len; i++) {
- loop = aux[i].loop;
- if (!loop)
- continue;
- for (j = 0; j < loop->backedges_cnt; j++) {
- backedge = &loop->backedges[j];
- backedge->latches_cnt = 0;
- backedge->taken_leaves = 0;
- /*
- * In theory, the backedge->from and it's latch can reside in
- * an inner loop of `i`, e.g.:
- *
- * 1: r7 += 1;
- * 2: r6 += 1;
- * if r7 == 2 goto 1b;
- * if r6 < 2 goto 2b;
- *
- * For now, let's assume that latches are unknown for such cases.
- * (GCC/LLVM handle this by inserting artificial cfg nodes).
- */
- if (bpf_loop_at_index(env, backedge->from) != i)
- continue;
- err = find_dominating_conditions(env, backedge->from, backedge);
- if (err)
- return err;
- for (k = 1; flags && k < backedge->latches_cnt; k++)
- if ((env->log.level & BPF_LOG_LEVEL2) &&
- !bpf_jump_leaves_loop(env, i, backedge->latches[k]))
- bpf_log(&env->log, "loop at %d, exit by way of a flag at %d\n",
- i, backedge->latches[k]);
- }
- }
- return 0;
+ if (bpf_loop_at_index(env, n) != h || idoms_intersect(env, n, h) != h)
+ return -1;
+ for (; n >= 0 && !bpf_insn_leaves_loop(env, h, n); n = n == h ? -1 : env->idoms[n])
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
+ return n;
}
/*
@@ -547,13 +421,15 @@ bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int oute
return false;
}
-/* Whether the conditional jump at @n, in the loop @h or in a loop nested in it, leaves @h */
-bool bpf_jump_leaves_loop(struct bpf_verifier_env *env, int h, int n)
+/* Whether @n is a conditional jump in the loop @h itself that leaves @h */
+bool bpf_insn_leaves_loop(struct bpf_verifier_env *env, int h, int n)
{
struct bpf_insn *insn = &env->prog->insnsi[n];
u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
int t, f;
+ if (bpf_loop_at_index(env, n) != h)
+ return false;
if ((class != BPF_JMP && class != BPF_JMP32) || op == BPF_JA || op == BPF_CALL ||
op == BPF_EXIT || op == BPF_JCOND)
return false;
@@ -563,17 +439,9 @@ bool bpf_jump_leaves_loop(struct bpf_verifier_env *env, int h, int n)
(f != h && !bpf_is_nested_loop(env, f, h));
}
-/* The same for an insn that is in the loop @h itself */
-bool bpf_insn_leaves_loop(struct bpf_verifier_env *env, int h, int n)
-{
- if (bpf_loop_at_index(env, n) != h)
- return false;
- return bpf_jump_leaves_loop(env, h, n);
-}
-
int bpf_compute_loops(struct bpf_verifier_env *env)
{
- int i, j, k, s, t, iloop, sloop, err = 0, len = env->prog->len;
+ int i, j, s, t, iloop, sloop, err = 0, len = env->prog->len;
struct bpf_insn_aux_data *aux = env->insn_aux_data;
struct bpf_verifier_log *log = &env->log;
struct bpf_backedge *backedge;
@@ -595,9 +463,15 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
if (err)
goto out;
}
- err = bpf_find_latches(env, false);
- if (err)
- goto out;
+ for (i = 0; i < len; i++) {
+ loop = aux[i].loop;
+ if (!loop)
+ continue;
+ loop->exit_to = -1;
+ for (j = 0; j < loop->backedges_cnt; j++) {
+ loop->backedges[j].latch = find_latch(env, i, loop->backedges[j].from);
+ }
+ }
/* find exits */
for (i = 0; i < len; i++) {
iloop = aux[i].loop ? i : aux[i].loop_header;
@@ -605,68 +479,15 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
continue;
succ = bpf_insn_successors(env, i);
iarray_for_each(s, succ) {
- /*
- * Nothing left to record once the innermost loop of 'i'
- * overflowed: the walk below would stop at it right away.
- */
- if (aux[iloop].loop->exits_overflow)
- break;
- sloop = aux[s].loop ? s : aux[s].loop_header;
- if (iloop == sloop)
- continue;
- if (bpf_is_nested_loop(env, sloop, iloop))
- continue;
- /*
- * At this point 'sloop' is either -1, an outer loop,
- * or a loop in another branch of the loop hierarchy.
- */
- for (t = iloop; t >= 0 && t != sloop; t = aux[t].loop_header) {
- /*
- * Account for the following configuration:
- *
- * tloop {
- * iloop {
- * ... i: goto s;
- * }
- * sloop {
- * s:
- * ...
- * }
- * }
- */
- if (bpf_is_nested_loop(env, sloop, t))
- break;
- /*
- * Record edges i -> s as exits from tloop when:
- *
- * sloop {
- * tloop {
- * iloop {
- * ... i: goto s;
- * }
- * }
- * s: ...
- * }
- */
- /* Enclosing loops get nested_exits_overflow, see below */
- if (aux[t].loop->exits_overflow)
- break;
- err = add_exit(aux[t].loop, i, s);
- if (err)
- goto out;
- }
+ sloop = bpf_loop_at_index(env, s);
+ for (t = iloop; t >= 0 && t != sloop && !bpf_is_nested_loop(env, sloop, t);
+ t = aux[t].loop_header)
+ if (aux[t].loop->exit_to < 0)
+ aux[t].loop->exit_to = s;
}
if (bpf_is_ldimm64(env->prog->insnsi + i))
i++;
}
- /* A nested loop with a truncated exit list can't be abstracted by SCEV */
- for (i = 0; i < len; i++) {
- loop = aux[i].loop;
- if (!loop || !loop->exits_overflow)
- continue;
- for (t = aux[i].loop_header; t >= 0; t = aux[t].loop_header)
- aux[t].loop->nested_exits_overflow = true;
- }
if (env->log.level & BPF_LOG_LEVEL2) {
for (i = 0; i < len; i++) {
@@ -678,28 +499,16 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
bpf_log(log, ", nested in %d", aux[i].loop_header);
if (loop->irreducible)
bpf_log(log, ", irreducible");
- if (loop->exits_overflow)
- bpf_log(log, ", too many exits");
- if (loop->nested_exits_overflow)
- bpf_log(log, ", too many exits of a nested loop");
bpf_log(log, "\n");
for (j = 0; j < loop->backedges_cnt; j++) {
backedge = &loop->backedges[j];
bpf_log(log, " backedge from %d", backedge->from);
- for (k = 0; k < backedge->latches_cnt; k++) {
- if (k == 0)
- bpf_log(log, ", latch at ");
- if (k == 1)
- bpf_log(log, ", dominating exits at ");
- if (k > 1)
- bpf_log(log, ", ");
- bpf_log(log, "%d", backedge->latches[k]);
- }
+ if (backedge->latch >= 0)
+ bpf_log(log, ", latch at %d", backedge->latch);
bpf_log(log, "\n");
}
- for (j = 0; j < loop->exits_cnt; j++)
- bpf_log(log, " exit from %d to %d\n",
- loop->exits[j].from, loop->exits[j].to);
+ if (loop->exit_to >= 0)
+ bpf_log(log, " first exit to %d\n", loop->exit_to);
}
}
diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c
deleted file mode 100644
index 5218ce1cafc..00000000000
--- a/kernel/bpf/scev.c
+++ /dev/null
@@ -1,5504 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
-
-#include "linux/cnum.h"
-#include <linux/bpf_verifier.h>
-#include <linux/gcd.h>
-#include <linux/jhash.h>
-#include <linux/log2.h>
-#include <linux/math64.h>
-#include <linux/min_heap.h>
-#include <linux/bug.h>
-#include <linux/tnum.h>
-#include <linux/overflow.h>
-
-#define REGS_NUM BPF_SCEV_REGS_NUM
-/* The SCEV registers of an env: r0 - r10, the stack slots and memory locations, see struct env */
-#define MEM_REGS_NUM 16
-#define ENV_REGS_NUM (REGS_NUM + MEM_REGS_NUM)
-#define UNKNOWN_EXPR_ID 0
-#define OPAQUE_EXPR_ID 1
-/* (REG r). UNKNOWN, OPAQUE and (REG r) have fixed ids and are not in the hash table. */
-#define REG_EXPR_ID(r) (2 + (r))
-
-/*
- * BPF instructions use 'code', 'src_reg', 'off' and 'imm' fields for instruction encoding.
- * For scalar evolution purpose we want to reuse most of the opcode definitions,
- * but also add a few custom operations (REG and IMM).
- * Use the enum below to uniformly represent all operations relevant for SCEV.
- */
-enum expr_op {
- /* leave range 0..255 for standard bpf opcodes */
- UNKNOWN = 256, /* start custom opcodes from the second byte */
- REG,
- IMM,
- SDIV, SMOD,
- SEXT8, SEXT16, SEXT32,
- ZEXT8, ZEXT16, ZEXT32,
- BSWAP16, BSWAP32, BSWAP64,
- /* during the loop body execution the value can be either of param[0] or param[1] */
- ANY,
- /* some value that verifier is not going to track precisely */
- OPAQUE,
- /* '*(u8/16/32 *)(r10 + X) = Y' writes define slot contents only partially */
- SPILL8, SPILL16, SPILL32,
- /*
- * SCEV expression corresponding to linear equation 'param[0] + param[1] * k',
- * where k is a loop iteration number. Loop here refers to innermost loop
- * containing instruction associated with this expression, as returned by
- * bpf_loop_at_index().
- */
- LINEAR_SCEV,
- /*
- * (add rB rC) is the SCEV of rA: rA == rB + rC at the header in every iteration. rB and rC
- * are the header phis here, not the entry values as below LINEAR_SCEV.
- */
- ADD_SCEV,
-};
-
-/*
- * An SSA name for a value that no expression describes: the def of the SCEV register 'reg'
- * - by the insn at 'idx'. An insn that may write defines too (may_def in liveness.c). A nested loop
- * defines at its header what it leaves.
- * - by a phi before the insn at 'idx': a join point, a loop header.
- * Without its backedges and with nested loops collapsed the body of a loop is acyclic: an insn
- * executes at most once per iteration, so a def is one value per iteration. A name is unique within
- * the env of one loop.
- * The operands of a phi are a set, 8 at most; a phi(a, b) among them is expanded. They have no
- * defs, see strip_defs(): (+ r6 %r2.4) and (+ r6 %r2.7) are one operand, (+ r6 _). So they don't
- * tell one phi from another: phis are value numbered by their bases, see mk_phi().
- * In the env of loop H a phi at H merges the backedge values only, the header phi itself is
- * (REG r). In the env of the enclosing loop a phi at H merges the entering edges.
- * Op and params of a def by an insn are what that insn computes, see alu64_with_def():
- * set_expr_base() looks through such an add, eval_linked_exprs() evaluates it, split_const_offset()
- * and eval_linked_exprs() check prevent_zext of the insn. Those of a phi are the shape that its
- * operands share, no insn computes them: nothing looks through a phi.
- */
-struct expr_def {
- u32 set:1; /* 0: the expression has no def */
- u32 phi:1;
- u32 reg:9;
- u32 idx:21;
-};
-/* def.reg of the phi of two bases, see mk_phi() */
-#define BASE_REG ENV_REGS_NUM
-static_assert(BASE_REG < 1 << 9 && BPF_COMPLEXITY_LIMIT_INSNS <= 1 << 21);
-
-static const struct expr_def no_def;
-
-struct expr {
- u16 op:15;
- /* A value number: all SCEV registers of an env that have this id hold the same value */
- u16 is_vn:1;
- /* Where find_linked_expr() looks for this id */
- u16 linked_expr;
- /*
- * UNKNOWN, OPAQUE and ANY don't describe a value. With a def the id stands for one value,
- * see struct expr_def. Without a def every use of the id can be another value.
- */
- struct expr_def def;
- union {
- u32 params[2];
- s64 imm;
- };
- /* expr == base + off, see set_expr_base() */
- u32 base;
- /*
- * id of the first add on the way to 'base' whose insn may still become ALU32, see
- * prevent_zext. 0: none
- */
- u32 arena_add;
- s64 off;
-};
-
-struct env {
- bool empty;
- u32 reg2expr[ENV_REGS_NUM];
- u32 reg2scev[REGS_NUM];
- /*
- * Register promotion of memory that is not behind r10. The SCEV register REGS_NUM + i is
- * the memory at the address mem_addr[i], a value number of this env. 0: not in use.
- * - A store through a pointer defines it like a spill defines a stack slot.
- * - A load defines its register as before, and the memory gets that value number: it holds
- * what a store of the register would leave. No expression of a register depends on
- * memory, which is private only if it is stack. That is not known here.
- * - A join point has a phi. An insn that may write is a may-def, see forget_mem_regs().
- * - A store through r10 is not: the main pass takes the stack of a caller only.
- * A value number of a loop means nothing in the enclosing loop: no memory is tracked at
- * a header, a nested loop is a may-def.
- */
- u32 mem_addr[MEM_REGS_NUM];
-};
-
-/*
- * A SCEV register and the id of its expression before a conditional jump. 'addr' is mem_addr[] of
- * a memory location.
- */
-struct scev_linked_reg {
- u32 reg;
- u32 id;
- u32 addr;
- /* The jump refines it. Otherwise it is a leader, or it holds the address of memory. */
- bool refine;
- /* At a visit: it is the linked expression 'expr' + a constant. NO_IDX: not evaluated. */
- u16 expr;
-};
-
-#define NO_IDX U16_MAX
-#define LEADERS_NUM 3
-
-/*
- * The expressions of the linked registers of a jump are a DAG. Its nodes are bases, see
- * set_expr_base(): (+ x 4) and (+ x 8) are one node, x. A visit of the jump lists them in
- * post-order: every base is there once, after its operands. The list is not kept.
- */
-struct scev_linked_expr {
- u32 id;
- /* The bases of the operands: positions among the linked expressions of the jump */
- u16 params[2];
- /*
- * GVN's leader table. The SCEV registers that hold 'id' + a constant (mod 2^N): positions
- * among the linked registers of the jump. First the tested registers: they have news in any
- * number of bits. The last one is the widest of the others, in 'leader_bits' bits.
- */
- u16 leaders[LEADERS_NUM];
- u8 leader_bits;
- /* It can be evaluated: it has a leader, or its operands can be evaluated */
- u8 has_value:1;
- /* find_linked_regs(): a register that the jump refines is made of it */
- u8 needed:1;
- /* By its innermost base it can have a constant difference to a tested register */
- u8 tested_base:1;
- /*
- * LLVM: SCEVExprContains(). By its bases a tested register can be among the
- * operands. Once the leaders are known: the value depends on a tested register.
- */
- u8 has_tested:1;
-};
-
-struct insn_envs {
- u32 cnt;
- /* The linked registers of a jump, see find_linked_regs(): linked_cnt of scev->linked[] */
- u32 linked_first;
- u32 linked_cnt;
- struct {
- int loop_header;
- struct env *env;
- } entries[];
-};
-
-#define EXPR_STACK_DEPTH 8
-
-struct expr_stack_elt {
- u32 id:28;
- u32 pre:1;
- u32 next_param:2;
-};
-
-DEFINE_MIN_HEAP(int, scev_worklist);
-
-struct scev {
- /* Read-only representatives for stack slots without a saved register. */
- struct bpf_reg_state zero_reg;
- struct bpf_reg_state unknown_reg;
- /* r -> (REG r): before the first insn of a loop every SCEV register is its header phi */
- struct env identity_env;
- /*
- * Expressions are identified by id, exprs_ht ensures that
- * each expression exists as a unique instance.
- * This allows for fast equivalence check: id1 === id2.
- * exprs_ht[hash] is the first id of a chain, exprs_next[id] is the next one, 0 ends it.
- * Both have exprs_cap elements, like exprs[].
- */
- u32 *exprs_ht;
- u32 *exprs_next;
- struct scev_worklist worklist;
- struct insn_envs **envs;
- struct expr *exprs;
- struct scev_linked_reg *linked;
- /* The linked expressions of the jump at hand and their values, see eval_linked_exprs() */
- struct scev_linked_expr *linked_exprs;
- struct scev_value *vals;
- /* The stack slots that a store through a dynptr slice may alias, see expr_is_vn() */
- DECLARE_BITMAP(slice_slots, MAX_BPF_STACK_SLOTS);
- /*
- * Loops are analyzed one by one, this array keeps track if a particular
- * instruction was visited on a current pass.
- */
- u32 *discovered;
- int exprs_cnt;
- int exprs_cap;
- u32 linked_cnt;
- u32 linked_cap;
- u32 linked_exprs_cnt;
- u32 linked_exprs_cap;
- u32 vals_cap;
- int envs_cnt;
- int stack_sz;
- int simplify_depth;
- struct expr_stack_elt expr_stack[EXPR_STACK_DEPTH];
- u32 ids_buf[EXPR_STACK_DEPTH];
-};
-
-static u32 expr_hash(struct expr *e)
-{
- /* def.phi is not hashed: the def by an insn and the phi before it share a chain */
- return jhash_3words(e->op, e->params[0], e->params[1],
- e->def.idx * REGS_NUM + e->def.reg);
-}
-
-static bool same_def(struct expr_def a, struct expr_def b)
-{
- return a.set == b.set && a.phi == b.phi && a.reg == b.reg && a.idx == b.idx;
-}
-
-/* Is @e the expression @id? The base of a phi is a part of its key. 0: the phi is its own base. */
-static int expr_eq(struct scev *scev, struct expr *e, u32 id)
-{
- struct expr *x = &scev->exprs[id];
-
- if (e->op != x->op || e->imm != x->imm || !same_def(e->def, x->def))
- return false;
- if (!e->def.phi)
- return true;
- return e->base ? e->base == x->base && e->off == x->off : x->base == id;
-}
-
-static void exprs_ht_add(struct scev *scev, u32 id)
-{
- u32 *bucket = &scev->exprs_ht[expr_hash(&scev->exprs[id]) & (scev->exprs_cap - 1)];
-
- scev->exprs_next[id] = *bucket;
- *bucket = id;
-}
-
-/* @cap is a power of two. No warning for a size that kvmalloc() rejects. */
-static int grow_exprs(struct scev *scev, u32 cap)
-{
- gfp_t gfp = BPF_VERIFIER_GFP;
- void *tmp;
- u32 id;
-
- tmp = kvrealloc(scev->exprs, array_size(cap, sizeof(*scev->exprs)), gfp);
- if (!tmp)
- return -ENOMEM;
- scev->exprs = tmp;
- tmp = kvrealloc(scev->exprs_next, array_size(cap, sizeof(u32)), gfp);
- if (!tmp)
- return -ENOMEM;
- scev->exprs_next = tmp;
- tmp = kvrealloc(scev->exprs_ht, array_size(cap, sizeof(u32)), gfp);
- if (!tmp)
- return -ENOMEM;
- scev->exprs_ht = tmp;
- scev->exprs_cap = cap;
- memset(scev->exprs_ht, 0, array_size(cap, sizeof(u32)));
- for (id = REG_EXPR_ID(REGS_NUM); id < scev->exprs_cnt; id++)
- exprs_ht_add(scev, id);
- return 0;
-}
-
-static u32 op_params_num(u32 op);
-
-static bool expr_is_vn(struct scev *scev, struct expr *e)
-{
- u32 i, reg = e->def.reg;
-
- /*
- * bpf_dynptr_slice_rdwr() can return the buffer that it got. A store through that slice may
- * alias the slots of the buffer and is in no bpf_may_write_mask(): it is no def, so a def
- * of such a slot is no value number. A spill through r10 has the expression of the
- * register, and invalidate_stack_slices() invalidates the slices there. The next slice
- * comes from a call that may write to the slot.
- */
- if (e->def.set)
- return reg < MAX_BPF_REG || reg >= REGS_NUM ||
- !test_bit(reg - MAX_BPF_REG, scev->slice_slots);
- switch (e->op) {
- case UNKNOWN:
- case OPAQUE:
- case ANY:
- /* a partial spill: the other bits of the slot are not known */
- case SPILL8:
- case SPILL16:
- case SPILL32:
- /* a function of the iteration number */
- case LINEAR_SCEV:
- /* a relation between header phis */
- case ADD_SCEV:
- return false;
- }
- for (i = 0; i < op_params_num(e->op); i++)
- if (!scev->exprs[e->params[i]].is_vn)
- return false;
- return true;
-}
-
-/*
- * @e gets the id @id. (+ x imm) has the base of x, which was constructed earlier. mk_phi() sets the
- * base of a phi that it value numbers. Every other expression is its own base, an extension and a
- * partial spill too, see next_base(). So split_const_offset() does not traverse the expression.
- */
-static void set_expr_base(struct scev *scev, struct expr *e, u32 id)
-{
- struct expr *x;
-
- e->arena_add = 0;
- /* mk_phi() found the base */
- if (e->def.phi && e->base)
- return;
- e->base = id;
- e->off = 0;
- /* nothing looks through a phi, see struct expr_def */
- if (e->def.phi || e->op != BPF_ADD || scev->exprs[e->params[1]].op != IMM)
- return;
- x = &scev->exprs[e->params[0]];
- e->base = x->base;
- e->off = (u64)x->off + scev->exprs[e->params[1]].imm;
- e->arena_add = e->def.set ? id : x->arena_add;
-}
-
-static int add_expr(struct scev *scev, struct expr e)
-{
- u32 id;
-
- if (e.op == REG)
- return REG_EXPR_ID(e.params[0]);
- if (!e.def.set && !e.imm && (e.op == UNKNOWN || e.op == OPAQUE))
- return e.op == UNKNOWN ? UNKNOWN_EXPR_ID : OPAQUE_EXPR_ID;
-
- id = scev->exprs_ht[expr_hash(&e) & (scev->exprs_cap - 1)];
- for (; id; id = scev->exprs_next[id])
- if (expr_eq(scev, &e, id))
- return id;
-
- if (scev->exprs_cnt == scev->exprs_cap && grow_exprs(scev, scev->exprs_cap * 2))
- return -ENOMEM;
-
- id = scev->exprs_cnt++;
- e.is_vn = expr_is_vn(scev, &e);
- set_expr_base(scev, &e, id);
- scev->exprs[id] = e;
- exprs_ht_add(scev, id);
- return id;
-}
-
-static int simplify(struct scev *scev, u32 id);
-
-/*
- * An expression is simplified at construction. Its params are simplified already, so simplify()
- * rewrites the root only, whatever the depth of the expression.
- * The rules construct expressions too, and one rule can enable the next: the depth of (zext32
- * (spill32 (+ (spill32 (+ ..) 1)) 1)) is up to the prog. Beyond depth EXPR_STACK_DEPTH an
- * expression is correct, but not simplified.
- */
-static int expr2(struct scev *scev, u32 op, int a, int b)
-{
- int id;
-
- if (a < 0)
- return a;
- if (b < 0)
- return b;
- id = add_expr(scev, (struct expr){ .op = op, .params = {a, b} });
- if (id < 0 || scev->simplify_depth == EXPR_STACK_DEPTH)
- return id;
- scev->simplify_depth++;
- id = simplify(scev, id);
- scev->simplify_depth--;
- return id;
-}
-
-static int expr1(struct scev *scev, u32 op, int a)
-{
- if (a < 0)
- return a;
- return expr2(scev, op, a, 0);
-}
-
-static int imm_expr(struct scev *scev, s64 value)
-{
- return add_expr(scev, (struct expr){ .op = IMM, .imm = value });
-}
-
-/*
- * The insn at @idx defines @reg. Without its backedges and with nested loops collapsed the body is
- * acyclic: an insn executes at most once per iteration, so a def is one value per iteration. A
- * subprog is the outermost loop, which runs once.
- */
-static struct expr_def insn_def(int idx, u32 reg)
-{
- return (struct expr_def){ .set = 1, .reg = reg, .idx = idx };
-}
-
-/* The phi of @reg before the insn at @idx. One value per iteration for the same reason. */
-static struct expr_def phi_def(int idx, u32 reg)
-{
- return (struct expr_def){ .set = 1, .phi = 1, .reg = reg, .idx = idx };
-}
-
-/* @id with another def */
-static int expr_with_def(struct scev *scev, int id, struct expr_def def)
-{
- struct expr e;
-
- if (id < 0)
- return id;
- e = scev->exprs[id];
- e.def = def;
- e.base = 0;
- return add_expr(scev, e);
-}
-
-static bool same_exprs(struct scev *scev, int id_a, int id_b)
-{
- return id_a == id_b;
-}
-
-static bool is_imm(struct scev *scev, int id, s64 *imm)
-{
- if (scev->exprs[id].op != IMM)
- return false;
- *imm = scev->exprs[id].imm;
- return true;
-}
-
-static bool is_op(struct scev *scev, int id, u32 op)
-{
- if (scev->exprs[id].op != op)
- return false;
- return true;
-}
-
-static bool is_unop(struct scev *scev, enum expr_op op, int id, u32 *p0)
-{
- if (scev->exprs[id].op != op)
- return false;
- *p0 = scev->exprs[id].params[0];
- return true;
-}
-
-static bool is_binop(struct scev *scev, enum expr_op op, int id, u32 *p0, u32 *p1)
-{
- if (scev->exprs[id].op != op)
- return false;
- *p0 = scev->exprs[id].params[0];
- *p1 = scev->exprs[id].params[1];
- return true;
-}
-
-static bool is_reg(struct scev *scev, int id, u32 *reg)
-{
- return is_unop(scev, REG, id, reg);
-}
-
-static bool is_add(struct scev *scev, int id, u32 *left, u32 *right)
-{
- return is_binop(scev, BPF_ADD, id, left, right);
-}
-
-static bool is_zext32(struct scev *scev, int id, u32 *left)
-{
- return is_unop(scev, ZEXT32, id, left);
-}
-
-/* (zextN x), (sextN x) and (spillN x) depend on the low N bits of x only */
-static u32 op_low_bits(u32 op)
-{
- switch (op) {
- case ZEXT8:
- case SEXT8:
- case SPILL8:
- return 8;
- case ZEXT16:
- case SEXT16:
- case SPILL16:
- return 16;
- case ZEXT32:
- case SEXT32:
- case SPILL32:
- return 32;
- }
- return 0;
-}
-
-static bool is_zext(u32 op)
-{
- return op == ZEXT8 || op == ZEXT16 || op == ZEXT32;
-}
-
-/* The low @bits bits of the slot are those of @left. The other bits are not known. */
-static bool is_spill(struct scev *scev, int id, u32 *bits, u32 *left)
-{
- u32 op = scev->exprs[id].op;
-
-#ifdef __BIG_ENDIAN
- /* '*(u32 *)(r10 - 8) = rX' writes the high half, see mk_fill() */
- return false;
-#endif
- if (op != SPILL8 && op != SPILL16 && op != SPILL32)
- return false;
- *bits = op_low_bits(op);
- *left = scev->exprs[id].params[0];
- return true;
-}
-
-static bool is_spill32(struct scev *scev, int id, u32 *left)
-{
- u32 bits;
-
- return is_spill(scev, id, &bits, left) && bits == 32;
-}
-
-/* (zextN x), (sextN x) or (spillN x) with N >= @bits: the low @bits bits are those of @left */
-static bool is_low_bits(struct scev *scev, int id, u32 bits, u32 *left)
-{
- u32 op = scev->exprs[id].op;
-
-#ifdef __BIG_ENDIAN
- /* see is_spill() */
- if (op == SPILL8 || op == SPILL16 || op == SPILL32)
- return false;
-#endif
- /* nothing looks through a phi, see struct expr_def */
- if (op_low_bits(op) < bits || scev->exprs[id].def.phi)
- return false;
- *left = scev->exprs[id].params[0];
- return true;
-}
-
-/*
- * A value is *@base + *@off (mod 2^*@bits). An extension and a partial spill are values of their
- * own, and they have the low N bits of their operand: go on to the base of the operand, modulo 2^N.
- * Only to fewer bits: three steps at most. N is 32 or less: it does not matter if an add on the way
- * becomes ALU32, see split_const_offset().
- */
-static bool next_base(struct scev *scev, u32 *base, s64 *off, u32 *bits)
-{
- struct expr *x, *e = &scev->exprs[*base];
- u32 low = op_low_bits(e->op);
-
- /* nothing looks through a phi, see struct expr_def */
- if (!low || low >= *bits || e->def.set)
- return false;
- x = &scev->exprs[e->params[0]];
- *base = x->base;
- *off = (u64)*off + x->off;
- *bits = low;
- return true;
-}
-
-/* Two expressions that have a constant difference in some bits have the same one, or two REGs */
-static u32 innermost_base(struct scev *scev, u32 id)
-{
- u32 base = scev->exprs[id].base, bits = 64;
- s64 off = 0;
-
- while (next_base(scev, &base, &off, &bits))
- ;
- return base;
-}
-
-static bool is_any(struct scev *scev, int id, u32 *left, u32 *right)
-{
- return is_binop(scev, ANY, id, left, right);
-}
-
-static bool is_linear(struct scev *scev, int id, u32 *base, u32 *slope)
-{
- return is_binop(scev, LINEAR_SCEV, id, base, slope);
-}
-
-static bool is_linear_or_linear32(struct scev *scev, int id, u32 *base, u32 *slope, bool *zext32)
-{
- u32 t;
-
- *zext32 = is_zext32(scev, id, &t);
- return is_linear(scev, *zext32 ? t : id, base, slope);
-}
-
-/* What a register is at the entry of the loop, or the low half of that. */
-static bool is_reg_or_low32(struct scev *scev, int id, u32 *reg)
-{
- u32 t;
-
- return is_reg(scev, id, reg) || (is_zext32(scev, id, &t) && is_reg(scev, t, reg));
-}
-
-static bool is_opaque(struct scev *scev, int id)
-{
- return is_op(scev, id, OPAQUE);
-}
-
-static void log_reg(struct bpf_verifier_env *env, u32 reg)
-{
- if (reg < MAX_BPF_REG)
- bpf_log(&env->log, "r%d", reg);
- else if (reg < REGS_NUM)
- bpf_log(&env->log, "*fp%d", (MAX_BPF_REG - reg - 1) * 8);
- else
- bpf_log(&env->log, "mem%d", reg - REGS_NUM);
-}
-
-static const char *op_str(u32 op)
-{
- switch (op) {
- case BPF_ADD: return "+";
- case BPF_SUB: return "-";
- case BPF_MUL: return "*";
- case BPF_DIV: return "/";
- case SDIV: return "s/";
- case BPF_OR: return "|";
- case BPF_AND: return "&";
- case BPF_LSH: return "<<";
- case BPF_RSH: return ">>";
- case BPF_NEG: return "-";
- case BPF_MOD: return "%";
- case SMOD: return "s%";
- case BPF_XOR: return "^";
- case BPF_ARSH: return "s>>";
- case SEXT8: return "sext8";
- case SEXT16: return "sext16";
- case SEXT32: return "sext32";
- case ZEXT8: return "zext8";
- case ZEXT16: return "zext16";
- case ZEXT32: return "zext32";
- case BSWAP16: return "bswap16";
- case BSWAP32: return "bswap32";
- case BSWAP64: return "bswap64";
- case SPILL8: return "spill8";
- case SPILL16: return "spill16";
- case SPILL32: return "spill32";
- case ANY: return "any";
- case LINEAR_SCEV: return "linear";
- case ADD_SCEV: return "add";
- }
- return NULL;
-}
-
-static u32 op_params_num(u32 op)
-{
- switch (op) {
- case BPF_ADD:
- case BPF_SUB:
- case BPF_MUL:
- case BPF_DIV:
- case BPF_MOD:
- case BPF_OR:
- case BPF_XOR:
- case BPF_AND:
- case BPF_LSH:
- case BPF_RSH:
- case BPF_ARSH:
- case SDIV:
- case SMOD:
- case ANY:
- case LINEAR_SCEV:
- case ADD_SCEV:
- return 2;
- case BPF_NEG:
- case SEXT8:
- case SEXT16:
- case SEXT32:
- case ZEXT8:
- case ZEXT16:
- case ZEXT32:
- case BSWAP16:
- case BSWAP32:
- case BSWAP64:
- case SPILL8:
- case SPILL16:
- case SPILL32:
- return 1;
- case REG:
- case IMM:
- case OPAQUE:
- return 0;
- default:
- return 0;
- }
-}
-
-static bool expr_stack_push(struct scev *scev, u32 id)
-{
- if (scev->stack_sz >= EXPR_STACK_DEPTH)
- return false;
- scev->expr_stack[scev->stack_sz].id = id;
- scev->expr_stack[scev->stack_sz].pre = true;
- scev->expr_stack[scev->stack_sz].next_param = 0;
- scev->stack_sz++;
- return true;
-}
-
-enum {
- PRE = BIT(1), POST = BIT(2), DEPTH_LIMIT = BIT(3)
-};
-
-static bool expr_next(struct scev *scev, u32 *id, u32 *order)
-{
- struct expr_stack_elt *elt;
- struct expr *expr;
- u32 num_params;
-
- if (scev->stack_sz == 0)
- return false;
-
- elt = &scev->expr_stack[scev->stack_sz - 1];
- *id = elt->id;
- *order = 0;
- expr = &scev->exprs[elt->id];
- num_params = op_params_num(expr->op);
- if (elt->pre) {
- elt->pre = false;
- *order = PRE;
- return true;
- }
- if (elt->next_param == num_params) {
- *order = POST;
- scev->stack_sz--;
- return true;
- }
- if (scev->stack_sz == EXPR_STACK_DEPTH) {
- *order = POST | DEPTH_LIMIT;
- scev->stack_sz--;
- return true;
- }
- expr_stack_push(scev, expr->params[elt->next_param]);
- elt->next_param++;
- return expr_next(scev, id, order);
-}
-
-/*
- * An SSA name is printed as '%', the SCEV register, '.', the insn. A def is printed in full at
- * every use: (+ %r8.7=phi(1, 2) 4).
- * %r0.6 insn 6 defines r0. The value is OPAQUE, '_' without a def
- * %fp-8.8? insn 8 may write fp-8. The value is UNKNOWN, '?' without a def
- * %r8.7=phi(1, 2) the phi of r8 before insn 7
- * %r0.7=phi(_) a phi of distinct OPAQUE values
- * %r6.9=phi(?) a phi of distinct UNKNOWN values, or of more than 8 values
- * %r2.12=(+ x 4) insn 12 defines r2, see alu64_with_def()
- * %r2.25=phi((+ x 4)) two adds of one shape, each may have been converted to ALU32:
- * the phi is not (+ x 4)
- * A name is unique within the env of one loop. In the env of loop H a phi at H merges the backedge
- * values only, the header phi itself is (REG r): in 'scev at header 1: r6=%r6.1=phi(r6, (+ r6 1))'
- * one backedge leaves r6 as it is, the other one adds 1.
- * '%' in front of r or fp is a name, followed by a blank it is BPF_MOD.
- */
-static void log_def(struct bpf_verifier_env *env, struct expr_def def)
-{
- if (!def.set)
- return;
- if (def.reg < MAX_BPF_REG)
- bpf_log(&env->log, "%%r%d.%d", def.reg, def.idx);
- else if (def.reg < REGS_NUM)
- bpf_log(&env->log, "%%fp%d.%d", (MAX_BPF_REG - def.reg - 1) * 8, def.idx);
- else
- bpf_log(&env->log, "%%mem%d.%d", def.reg - REGS_NUM, def.idx);
-}
-
-static void log_expr(struct bpf_verifier_env *env, u32 id)
-{
- struct bpf_verifier_log *log = &env->log;
- bool operand, bare, first = false;
- struct scev *scev = env->scev;
- u32 order, d, bare_any = 0;
- struct expr_def def;
- struct expr *expr;
- const char *str;
-
- scev->stack_sz = 0;
- expr_stack_push(scev, id);
- while (expr_next(scev, &id, &order)) {
- expr = &scev->exprs[id];
- def = expr->def;
- /* the depth of @id in expr_stack[], expr_next() pops it before POST */
- d = order & PRE ? scev->stack_sz - 1 : scev->stack_sz;
- operand = d && (bare_any & BIT(d - 1));
- /* No '(any' for the operands of phi(a, b). (any a b) among them is expanded. */
- if (order & PRE) {
- bare_any &= ~BIT(d);
- if (expr->op == ANY && (def.set ? def.phi : operand))
- bare_any |= BIT(d);
- }
- bare = bare_any & BIT(d);
- if ((order & PRE) && (def.set || !bare)) {
- if (d)
- bpf_log(log, "%s", !operand ? " " : first ? "" : ", ");
- log_def(env, def);
- if (def.phi)
- bpf_log(log, "=phi(");
- first = bare;
- }
- switch (expr->op) {
- case UNKNOWN:
- if (order & PRE)
- bpf_log(log, "?");
- break;
- case OPAQUE:
- if ((order & PRE) && (!def.set || def.phi))
- bpf_log(log, "_");
- break;
- case REG:
- if (order & PRE)
- log_reg(env, expr->params[0]);
- break;
- case IMM:
- if (order & PRE)
- bpf_log(log, "%lld", expr->imm);
- break;
- default:
- if ((order & PRE) && !bare) {
- str = op_str(expr->op);
- bpf_log(log, def.set && !def.phi ? "=(" : "(");
- if (str)
- bpf_log(log, "%s", str);
- else
- bpf_log(log, "bad-expr-op %x", expr->op);
- }
- if (order & DEPTH_LIMIT) {
- bpf_log(log, bare && !def.set && !first ? ", ..." : "...");
- first = false;
- }
- if ((order & POST) && !bare)
- bpf_log(log, ")");
- }
- if ((order & POST) && def.phi)
- bpf_log(log, ")");
- }
-}
-
-static bool reg_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- u16 live_regs = aux[insn_idx].live_regs_before;
-
- /* memory */
- if (reg >= REGS_NUM)
- return true;
- return reg < MAX_BPF_REG
- ? (BIT(reg) & live_regs)
- : test_bit(reg - MAX_BPF_REG, aux[insn_idx].live_stack_before);
-}
-
-/* The first bytes of a stack slot are its high bits */
-static bool is_big_endian_slot(u32 reg)
-{
- return IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && reg >= MAX_BPF_REG;
-}
-
-/* Are the upper 32 bits of @reg live before the insn at @insn_idx? */
-static bool hi32_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx)
-{
- return reg < MAX_BPF_REG || is_big_endian_slot(reg) ||
- bpf_half_slot_live_before(env, insn_idx, (reg - MAX_BPF_REG) * 2);
-}
-
-static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx)
-{
- struct bpf_verifier_log *log = &env->log;
- struct scev *scev = env->scev;
- bool printed_some = false;
- bool is_self_scev;
- bool is_self_reg;
- int i, r;
-
- if (e->empty) {
- bpf_log(log, " <empty>\n");
- return;
- }
-
- for (i = 0; i < REGS_NUM; i++) {
- if (!reg_alive_at(env, i, insn_idx))
- continue;
- is_self_reg = is_reg(scev, e->reg2expr[i], &r) && i == r;
- is_self_scev = is_reg(scev, e->reg2scev[i], &r) && i == r;
- if (is_self_reg && is_self_scev)
- continue;
- printed_some = true;
- bpf_log(log, " ");
- log_reg(env, i);
- bpf_log(log, "=");
- log_expr(env, e->reg2expr[i]);
- if (!is_op(scev, e->reg2expr[i], UNKNOWN)) {
- bpf_log(log, " / ");
- log_expr(env, e->reg2scev[i]);
- }
- bpf_log(log, "\n");
- }
- if (!printed_some)
- bpf_log(log, " <all regs unchanged>\n");
-}
-static struct env *find_loop_env(struct scev *scev, int loop_header, int insn_idx)
-{
- struct insn_envs *envs = scev->envs[insn_idx];
- u32 i;
-
- if (!envs)
- return NULL;
- for (i = 0; i < envs->cnt; i++)
- if (envs->entries[i].loop_header == loop_header)
- return envs->entries[i].env;
- return NULL;
-}
-
-static struct env *find_header_env(struct scev *scev, int header)
-{
- return find_loop_env(scev, header, header);
-}
-
-static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx)
-{
- struct insn_envs *envs = scev->envs[insn_idx];
- u32 cnt = envs ? envs->cnt : 0;
- struct insn_envs *tmp;
- struct env *e;
-
- e = find_loop_env(scev, loop_header, insn_idx);
- if (e)
- return e;
-
- e = kzalloc_obj(*e, BPF_VERIFIER_GFP);
- if (!e)
- return NULL;
-
- tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), BPF_VERIFIER_GFP | __GFP_ZERO);
- if (!tmp) {
- kfree(e);
- return NULL;
- }
-
- e->empty = true;
- tmp->entries[cnt].loop_header = loop_header;
- tmp->entries[cnt].env = e;
- tmp->cnt = cnt + 1;
- scev->envs[insn_idx] = tmp;
- return e;
-}
-
-/* The env has no users any more */
-static void free_loop_env(struct scev *scev, int loop_header, int insn_idx)
-{
- struct insn_envs *envs = scev->envs[insn_idx];
- u32 i;
-
- for (i = 0; i < envs->cnt; i++) {
- if (envs->entries[i].loop_header != loop_header)
- continue;
- kfree(envs->entries[i].env);
- envs->entries[i] = envs->entries[--envs->cnt];
- break;
- }
- if (envs->cnt || envs->linked_cnt)
- return;
- kfree(envs);
- scev->envs[insn_idx] = NULL;
-}
-
-static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id)
-{
- if (id < 0)
- return id;
- e->reg2expr[reg] = id;
- return 0;
-}
-
-/* The insn at @idx defines @reg. The value is not tracked: @id is UNKNOWN or OPAQUE. */
-static int forget_reg(struct scev *scev, struct env *e, u32 reg, int id, int idx)
-{
- return replace_reg(scev, e, reg, expr_with_def(scev, id, insn_def(idx, reg)));
-}
-
-static void setup_initial_loop_env(struct env *e)
-{
- int i;
-
- /*
- * Initial assignments are of form rX -> rX are needed as seeds for subsequent
- * work queue pop / transfer() calls. Because e->empty is true, the first backedge
- * join() will overwrite the them. compute_header_scevs() accounts for register
- * states at loop entry.
- */
- for (i = 0; i < REGS_NUM; i++)
- e->reg2expr[i] = REG_EXPR_ID(i);
-}
-
-/* A subprog can return a pair in r0 and r2: a call defines r1 - r5 too if they are live after it */
-static int forget_call_regs(struct bpf_verifier_env *env, struct env *e, int idx)
-{
- int i, err;
-
- for (i = BPF_REG_0; i <= BPF_REG_5; i++) {
- e->reg2expr[i] = UNKNOWN_EXPR_ID;
- if (idx + 1 >= env->prog->len || !reg_alive_at(env, i, idx + 1))
- continue;
- err = forget_reg(env->scev, e, i, UNKNOWN_EXPR_ID, idx);
- if (err)
- return err;
- }
- return 0;
-}
-
-static u32 spill_spi(struct bpf_insn *insn)
-{
- return (-insn->off - 1) / BPF_REG_SIZE;
-}
-
-static u32 off_to_reg(struct bpf_insn *insn)
-{
- return spill_spi(insn) + MAX_BPF_REG;
-}
-
-static bool is_spill_off(int off)
-{
- return off % BPF_REG_SIZE == 0 &&
- off <= -BPF_REG_SIZE &&
- off >= -MAX_BPF_STACK_JIT;
-}
-
-static int mark_opaque(struct scev *scev, struct env *e, int idx, int off, int size)
-{
- int b, spi, err;
-
- for (b = off; b < off + size; b++) {
- if (b >= 0 || b < -MAX_BPF_STACK_JIT)
- continue;
- spi = (-b - 1) / BPF_REG_SIZE;
- err = forget_reg(scev, e, MAX_BPF_REG + spi, OPAQUE_EXPR_ID, idx);
- if (err)
- return err;
- }
- return 0;
-}
-
-static int mk_spill(struct scev *scev, u8 size, int id)
-{
- switch (size) {
- case BPF_B: return expr1(scev, SPILL8, id);
- case BPF_H: return expr1(scev, SPILL16, id);
- case BPF_W: return expr1(scev, SPILL32, id);
- case BPF_DW: return id;
- }
- return UNKNOWN_EXPR_ID;
-}
-
-/* The size of the spill in a slot with the expression @id. It holds the low bytes of *@x. */
-static u32 expr_spill_size(struct scev *scev, u32 id, u32 *x)
-{
- struct expr *e = &scev->exprs[id];
- /* nothing looks through a phi, see struct expr_def */
- bool partial = (e->op == SPILL8 || e->op == SPILL16 || e->op == SPILL32) && !e->def.phi;
-
- *x = partial ? e->params[0] : id;
- return partial ? op_low_bits(e->op) / BITS_PER_BYTE : BPF_REG_SIZE;
-}
-
-/* -ENOENT: the value of the fill is not known */
-static int mk_fill(struct scev *scev, int id, struct bpf_insn *insn)
-{
- u32 x, spill_size = expr_spill_size(scev, id, &x);
- u32 size = bpf_size_to_bytes(BPF_SIZE(insn->code));
- bool sx = BPF_MODE(insn->code) == BPF_MEMSX;
-
- /* The fill reads the low bytes of the spill, as in check_stack_read_fixed_off() */
- if (size <= spill_size && bpf_stack_narrow_access_ok(insn->off, size, spill_size))
- id = x;
- else if (!is_spill_off(insn->off))
- return -ENOENT;
-#ifdef __BIG_ENDIAN
- /* No register becomes (spillN x): its low bits would not be those of x */
- else if (spill_size < BPF_REG_SIZE)
- return -ENOENT;
- /* A narrow fill reads the high bits of the slot on big-endian. */
- else
- return expr2(scev, sx ? BPF_ARSH : BPF_RSH, id,
- imm_expr(scev, BITS_PER_BYTE * (BPF_REG_SIZE - size)));
-#endif
-
- switch (BPF_SIZE(insn->code)) {
- case BPF_B: return expr1(scev, sx ? SEXT8 : ZEXT8, id);
- case BPF_H: return expr1(scev, sx ? SEXT16 : ZEXT16, id);
- case BPF_W: return expr1(scev, sx ? SEXT32 : ZEXT32, id);
- case BPF_DW: return sx ? UNKNOWN_EXPR_ID : id; /* DW sign-extended load is invalid */
- }
- return UNKNOWN_EXPR_ID;
-}
-
-static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int idx, int id)
-{
- u8 size = BPF_SIZE(insn->code);
-
- if (insn->dst_reg != BPF_REG_FP)
- return 0;
- if (is_spill_off(insn->off))
- return replace_reg(scev, e, off_to_reg(insn), mk_spill(scev, size, id));
- return mark_opaque(scev, e, idx, insn->off, bpf_size_to_bytes(size));
-}
-
-static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int idx)
-{
- int id = -ENOENT;
-
- if (insn->src_reg == BPF_REG_FP && insn->off < 0 && insn->off >= -MAX_BPF_STACK_JIT)
- id = e->reg2expr[off_to_reg(insn)];
- /*
- * The def of a slot that a slice may alias is no value number, see expr_is_vn(): two fills
- * can read distinct values. One fill reads one value: the insn defines the register.
- */
- if (id >= 0 && scev->exprs[id].def.set && !scev->exprs[id].is_vn)
- id = -ENOENT;
- if (id >= 0)
- id = mk_fill(scev, id, insn);
- if (id == -ENOENT)
- return forget_reg(scev, e, insn->dst_reg, OPAQUE_EXPR_ID, idx);
- return replace_reg(scev, e, insn->dst_reg, id);
-}
-
-static bool const_diff(struct bpf_verifier_env *env, int h, u32 id_a, u32 id_b, s64 *delta,
- u32 *bits);
-
-/* The memory location of @e at the address @addr, 0: one that is not in use. MEM_REGS_NUM: none. */
-static u32 find_mem_reg(const struct env *e, u32 addr)
-{
- u32 i;
-
- for (i = 0; i < MEM_REGS_NUM && e->mem_addr[i] != addr; i++)
- ;
- return i;
-}
-
-static void forget_mem_reg(struct env *e, u32 i)
-{
- e->mem_addr[i] = 0;
- e->reg2expr[REGS_NUM + i] = UNKNOWN_EXPR_ID;
-}
-
-/*
- * The insn may write @size bytes at the address @addr, 0: anywhere. A may-def of all memory
- * locations but those that have the same address base and don't overlap. Two bases can be two names
- * of one memory, 'f(&c, &c)'.
- */
-static void forget_mem_regs(struct bpf_verifier_env *env, struct env *e, u32 addr, u32 size)
-{
- u32 i, x, bits;
- s64 delta;
-
- for (i = 0; i < MEM_REGS_NUM; i++) {
- /* the caller defines it */
- if (e->mem_addr[i] == addr)
- continue;
- if (addr && const_diff(env, -1, addr, e->mem_addr[i], &delta, &bits) &&
- bits == 64 && (delta >= size ||
- delta + expr_spill_size(env->scev, e->reg2expr[REGS_NUM + i], &x) <= 0))
- continue;
- forget_mem_reg(e, i);
- }
-}
-
-/*
- * A jump tests a register, and a load makes a new value number. So once no register that is live
- * after the insn at @idx has the base of its value no jump links a memory location by a constant
- * difference: it goes. One that holds (* n 8) goes too, although a jump on n would evaluate it.
- */
-static void forget_dead_mem_regs(struct bpf_verifier_env *env, struct env *e, int idx)
-{
- struct scev *scev = env->scev;
- u32 i, r, base;
-
- for (i = 0; i < MEM_REGS_NUM; i++) {
- base = innermost_base(scev, e->reg2expr[REGS_NUM + i]);
- for (r = 0; r < REGS_NUM; r++)
- if (reg_alive_at(env, r, idx + 1) &&
- innermost_base(scev, e->reg2expr[r]) == base)
- break;
- if (r == REGS_NUM)
- forget_mem_reg(e, i);
- }
-}
-
-/*
- * The address of the load or store at @idx through @reg, 0: the memory is not tracked. r10 has the
- * stack slots, r11 and r12 are no SCEV registers. The main pass takes nothing but stack.
- */
-static int mk_mem_addr(struct bpf_verifier_env *env, struct env *e, int idx, u32 reg)
-{
- struct scev *scev = env->scev;
-
- if (reg >= BPF_REG_FP || !(env->insn_aux_data[idx].stack_ptrs & BIT(reg)) ||
- !scev->exprs[e->reg2expr[reg]].is_vn)
- return 0;
- return expr2(scev, BPF_ADD, e->reg2expr[reg], imm_expr(scev, env->prog->insnsi[idx].off));
-}
-
-/*
- * The insn at @idx stored the low bytes of @id at the address @addr, or it loaded them from there.
- * The location takes a SCEV register that is not in use, or that of a dead one.
- */
-static int replace_mem_reg(struct bpf_verifier_env *env, struct env *e, int idx, int addr, int id)
-{
- u32 i;
-
- if (addr <= 0)
- return addr;
- i = find_mem_reg(e, addr);
- if (i == MEM_REGS_NUM)
- i = find_mem_reg(e, 0);
- if (i == MEM_REGS_NUM) {
- forget_dead_mem_regs(env, e, idx);
- i = find_mem_reg(e, 0);
- }
- if (i == MEM_REGS_NUM)
- return 0;
- e->mem_addr[i] = addr;
- return replace_reg(env->scev, e, REGS_NUM + i,
- mk_spill(env->scev, BPF_SIZE(env->prog->insnsi[idx].code), id));
-}
-
-static int load_mem(struct bpf_verifier_env *env, struct env *e, int idx)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- /* the insn defines dst_reg, which can be src_reg */
- int addr = mk_mem_addr(env, e, idx, insn->src_reg);
-
- return maybe_load_fp(env->scev, e, insn, idx) ?:
- replace_mem_reg(env, e, idx, addr, e->reg2expr[insn->dst_reg]);
-}
-
-static int store_mem(struct bpf_verifier_env *env, struct env *e, int idx, int id)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- int addr;
-
- if (insn->dst_reg == BPF_REG_FP)
- return maybe_store_fp(env->scev, e, insn, idx, id);
- addr = mk_mem_addr(env, e, idx, insn->dst_reg);
- if (addr < 0)
- return addr;
- forget_mem_regs(env, e, addr, bpf_size_to_bytes(BPF_SIZE(insn->code)));
- return replace_mem_reg(env, e, idx, addr, id);
-}
-
-/*
- * The low 32 bits of a + b depend on the low 32 bits of a and b only: 'w1 += w2' is
- * (zext32 (+ a b)). That doesn't hold for a right shift and for a division. Their operands are zero
- * or sign extended and a shift amount is masked, so that every expression is a 64-bit operation.
- */
-static void alu32_operands(struct scev *scev, u32 op, int *a, int *b)
-{
- s64 imm = 0;
- bool imm_b = *b >= 0 && is_imm(scev, *b, &imm);
-
- switch (op) {
- case BPF_RSH:
- case BPF_ARSH:
- *a = expr1(scev, op == BPF_RSH ? ZEXT32 : SEXT32, *a);
- fallthrough;
- case BPF_LSH:
- *b = imm_b ? imm_expr(scev, imm & 31) :
- expr2(scev, BPF_AND, *b, imm_expr(scev, 31));
- break;
- case BPF_DIV:
- case BPF_MOD:
- *a = expr1(scev, ZEXT32, *a);
- *b = imm_b ? imm_expr(scev, (u32)imm) : expr1(scev, ZEXT32, *b);
- break;
- case SDIV:
- case SMOD:
- *a = expr1(scev, SEXT32, *a);
- *b = imm_b ? imm_expr(scev, (s32)imm) : expr1(scev, SEXT32, *b);
- break;
- }
-}
-
-/*
- * In a prog with an arena an ALU64 insn is converted to ALU32 at fixup if an operand is a pointer
- * to the arena, see adjust_reg_min_max_vals(). Types are not known here. So @id, what the insn at
- * @idx computes from @a, gets a def: the value of @dst after the insn, converted or not. See
- * split_const_offset() for when the op can be trusted. Constant operands are scalars in every
- * state.
- */
-static int alu64_with_def(struct scev *scev, int id, int a, int idx, u32 dst)
-{
- s64 imm;
-
- if (id < 0 || is_imm(scev, id, &imm))
- return id;
- /* (+ x 0) is x only if the insn stays ALU64 */
- return expr_with_def(scev, id == a ? UNKNOWN_EXPR_ID : id, insn_def(idx, dst));
-}
-
-static int transfer(struct bpf_verifier_env *env, struct env *e, int idx)
-{
- const bool little_endian = htons(0x3412) == 0x1234;
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- bool arena = env->prog->aux->arena;
- struct scev *scev = env->scev;
- u32 *reg2expr = e->reg2expr;
- u8 class = BPF_CLASS(insn->code);
- u8 x_or_k = BPF_SRC(insn->code);
- u8 opcode = BPF_OP(insn->code);
- u8 mode = BPF_MODE(insn->code);
- u32 dst = insn->dst_reg;
- u32 src = insn->src_reg;
- u32 op, sext;
- bool need_bswap;
- int i, id, a;
- s64 imm;
-
- /* reg2expr[12] is a stack slot, not r12 */
- if (is_arena_sp_adj(insn) || is_arena_sp_st(insn))
- return 0;
- if (is_arena_sp_mov(insn) || is_arena_sp_ldx(insn))
- goto mark_dst_unknown;
-
- switch (class) {
- case BPF_ALU:
- case BPF_ALU64:
- switch (opcode) {
- case BPF_MOV:
- switch (insn->off) {
- case 0: sext = 0; break;
- case 8: sext = SEXT8; break;
- case 16: sext = SEXT16; break;
- case 32: sext = SEXT32; break;
- default:
- goto mark_dst_unknown;
- }
-
- if (x_or_k == BPF_X && insn->imm == 0)
- id = reg2expr[src];
- else if (x_or_k == BPF_K && src == 0 && insn->off == 0)
- id = imm_expr(scev, insn->imm);
- else
- goto mark_dst_unknown;
-
- if (sext)
- id = expr1(scev, sext, id);
- if (class == BPF_ALU)
- id = expr1(scev, ZEXT32, id);
-
- return replace_reg(scev, e, dst, id);
-
- case BPF_ADD:
- case BPF_SUB:
- case BPF_MUL:
- case BPF_DIV:
- case BPF_MOD:
- case BPF_OR:
- case BPF_XOR:
- case BPF_AND:
- case BPF_LSH:
- case BPF_RSH:
- case BPF_ARSH:
- if (opcode == BPF_DIV && insn->off == 1)
- op = SDIV;
- else if (opcode == BPF_MOD && insn->off == 1)
- op = SMOD;
- else if (insn->off == 0)
- op = opcode;
- else
- goto mark_dst_unknown;
-
- if (x_or_k == BPF_X && insn->imm == 0)
- id = reg2expr[src];
- else if (x_or_k == BPF_K && src == 0)
- id = imm_expr(scev, insn->imm);
- else
- goto mark_dst_unknown;
-
- a = reg2expr[dst];
- if (class == BPF_ALU)
- alu32_operands(scev, op, &a, &id);
- id = expr2(scev, op, a, id);
-
- if (class == BPF_ALU)
- id = expr1(scev, ZEXT32, id);
- else if (arena)
- id = alu64_with_def(scev, id, a, idx, dst);
-
- return replace_reg(scev, e, dst, id);
-
- case BPF_NEG:
- if (src == 0 && insn->off == 0 && insn->imm == 0)
- id = expr1(scev, BPF_NEG, reg2expr[dst]);
- else
- goto mark_dst_unknown;
-
- if (class == BPF_ALU)
- id = expr1(scev, ZEXT32, id);
-
- return replace_reg(scev, e, dst, id);
-
- case BPF_END:
- if (src != 0 || insn->off != 0 || (class == BPF_ALU64 && x_or_k != 0))
- goto mark_dst_unknown;
-
- need_bswap = class == BPF_ALU64 || ((x_or_k == BPF_TO_LE) != little_endian);
- switch (insn->imm) {
- case 16: op = need_bswap ? BSWAP16 : ZEXT16; break;
- case 32: op = need_bswap ? BSWAP32 : ZEXT32; break;
- case 64: op = need_bswap ? BSWAP64 : 0; break;
- default:
- goto mark_dst_unknown;
- }
-
- id = reg2expr[dst];
- if (op)
- id = expr1(scev, op, reg2expr[dst]);
-
- return replace_reg(scev, e, dst, id);
- default:
- goto mark_dst_unknown;
- }
- break;
- case BPF_LDX:
- switch (mode) {
- case BPF_MEM:
- case BPF_MEMSX:
- /*
- * for (i = 0; i < nr_cpus; i++) with "const volatile int nr_cpus":
- * bpf_compute_const_regs() knows what is loaded from a read-only map.
- */
- if (insn->src_reg != BPF_REG_FP && idx + 1 < env->prog->len &&
- (env->insn_aux_data[idx + 1].const_reg_mask & BIT(dst))) {
- imm = env->insn_aux_data[idx + 1].const_reg_vals[dst];
- return replace_reg(scev, e, dst, imm_expr(scev, imm));
- }
- return load_mem(env, e, idx);
- default:
- goto mark_dst_unknown;
- }
- case BPF_STX:
- switch (mode) {
- case BPF_MEM:
- return store_mem(env, e, idx, reg2expr[src]);
- case BPF_ATOMIC:
- if (insn->imm == BPF_LOAD_ACQ)
- return maybe_load_fp(scev, e, insn, idx);
- if (insn->imm == BPF_STORE_REL) {
- return store_mem(env, e, idx, reg2expr[src]);
- }
- /*
- * verifier does not track other atomic ops precisely,
- * hence mark the results as opaque.
- */
- if (insn->dst_reg == BPF_REG_FP) {
- id = mark_opaque(scev, e, idx, insn->off,
- bpf_size_to_bytes(BPF_SIZE(insn->code)));
- if (id < 0)
- return id;
- } else {
- forget_mem_regs(env, e, 0, 0);
- }
- if (insn->imm == BPF_CMPXCHG)
- return forget_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID, idx);
- if (insn->imm & BPF_FETCH)
- return forget_reg(scev, e, src, OPAQUE_EXPR_ID, idx);
- break;
- }
- break;
- case BPF_ST:
- id = imm_expr(scev, insn->imm);
- if (id < 0)
- return id;
- return store_mem(env, e, idx, id);
- case BPF_JMP:
- case BPF_JMP32:
- if (opcode == BPF_CALL) {
- forget_mem_regs(env, e, 0, 0);
- id = forget_call_regs(env, e, idx);
- if (id < 0)
- return id;
- /* made anew, like what is loaded. The parameter tells the two apart. */
- return forget_reg(scev, e, BPF_REG_0, expr1(scev, OPAQUE, 1), idx);
- }
- /* for non-CALL there are no changes in register states */
- break;
- case BPF_LD:
- switch (mode) {
- case BPF_IMM:
- /* rX = imm ll */
- if (BPF_SIZE(insn->code) == BPF_DW && insn->src_reg == 0) {
- imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
- return replace_reg(scev, e, dst, imm_expr(scev, imm));
- }
- /* map, map value, BTF id, function */
- if (BPF_SIZE(insn->code) == BPF_DW)
- return forget_reg(scev, e, dst, OPAQUE_EXPR_ID, idx);
- goto mark_dst_unknown;
- case BPF_ABS:
- case BPF_IND:
- return forget_call_regs(env, e, idx);
- default:
- goto mark_dst_unknown;
- }
- break;
- default:
- /* unknown instruction, nuke state */
- for (i = 0; i < REGS_NUM; i++)
- reg2expr[i] = UNKNOWN_EXPR_ID;
- forget_mem_regs(env, e, 0, 0);
- break;
- }
- return 0;
-
-mark_dst_unknown:
- return forget_reg(scev, e, dst, UNKNOWN_EXPR_ID, idx);
-}
-
-/*
- * if (c) sum += a[i]; else sum += b[i];
- * @id without defs: (+ r6 %r2.4) -> (+ r6 _). A def tells one load from another. The code that
- * analyzes loops asks how a value is computed, and that is the same on both paths.
- * Beyond depth EXPR_STACK_DEPTH an expression keeps its defs: no user looks there.
- */
-static int strip_defs(struct scev *scev, u32 id, u32 lvl)
-{
- struct expr e = scev->exprs[id]; /* snapshot the expr before potential realloc */
- int a;
-
- switch (op_params_num(e.op)) {
- case 0:
- return e.def.set ? expr_with_def(scev, id, no_def) : id;
- case 1:
- if (lvl == EXPR_STACK_DEPTH)
- return id;
- return expr1(scev, e.op, strip_defs(scev, e.params[0], lvl + 1));
- default:
- if (lvl == EXPR_STACK_DEPTH)
- return id;
- a = strip_defs(scev, e.params[0], lvl + 1);
- return expr2(scev, e.op, a, strip_defs(scev, e.params[1], lvl + 1));
- }
-}
-
-/*
- * Construct minimal 'ANY' expression by traversing 'a' and 'b'
- * and accumulating non-duplicated non-ANY entries, without their defs.
- * 'def' is the def of the result, no_def if it has none.
- */
-static int mk_any(struct scev *scev, int a, int b, struct expr_def def)
-{
- struct expr_stack_elt *elt;
- struct expr *expr;
- u32 i, j, ids_buf_sz;
- u32 roots[2] = {a, b};
- int id;
-
- if (a < 0)
- return a;
- if (b < 0)
- return b;
- ids_buf_sz = 0;
- for (i = 0; i < ARRAY_SIZE(roots); i++) {
- scev->stack_sz = 0;
- expr_stack_push(scev, roots[i]);
- while (scev->stack_sz) {
- elt = &scev->expr_stack[--scev->stack_sz];
- id = elt->id;
- expr = &scev->exprs[elt->id];
- if (expr->op == ANY) {
- if (!expr_stack_push(scev, expr->params[1]) ||
- !expr_stack_push(scev, expr->params[0]))
- return expr_with_def(scev, UNKNOWN_EXPR_ID, def);
- } else {
- /* the operands of an ANY have no defs already */
- if (id == roots[i])
- id = strip_defs(scev, id, 0);
- if (id < 0)
- return id;
- for (j = 0; j < ids_buf_sz; j++)
- if (same_exprs(scev, scev->ids_buf[j], id))
- goto next;
- if (ids_buf_sz == ARRAY_SIZE(scev->ids_buf))
- return expr_with_def(scev, UNKNOWN_EXPR_ID, def);
- scev->ids_buf[ids_buf_sz++] = id;
- }
-next:;
- }
- }
- if (WARN_ON(ids_buf_sz == 0))
- return -EFAULT;
- id = scev->ids_buf[0];
- for (i = 1; i < ids_buf_sz; i++) {
- id = expr2(scev, ANY, id, scev->ids_buf[i]);
- if (id < 0)
- return id;
- }
- return expr_with_def(scev, id, def);
-}
-
-static u32 split_const_offset(struct bpf_verifier_env *env, u32 id, u32 *base, s64 *off);
-
-/* @id == *@base + *@off in 64 bits: no add that may become ALU32 is stripped */
-static void split_const_offset64(struct bpf_verifier_env *env, u32 id, u32 *base, s64 *off)
-{
- struct scev *scev = env->scev;
- u32 l, r;
-
- if (split_const_offset(env, id, base, off) == 64)
- return;
- *base = id;
- *off = 0;
- if (!scev->exprs[id].def.set && is_add(scev, id, &l, &r) && is_imm(scev, r, off))
- *base = l;
-}
-
-/*
- * if (c) { n = f(); m = n; } else { n = g(); m = n; } if (n > 100) return 0; use(m);
- * The phi of @reg before the insn at @idx. It merges @a, which came over the edges so far, and @b.
- * Phis are value numbered as GVN does it, by join point and operands, up to a constant offset:
- * a == base_a + off_a and b == base_b + off_b, so
- * phi == phi(base_a, base_b + off_b - off_a) + off_a
- * The phi of the bases is hashed without @reg. All registers that have a constant difference on
- * each edge get the same one, and const_diff() finds them as it finds 'x' and 'x + 4'. The next
- * edge merges the phi of the bases with the next base: the id stands for the list of the operands,
- * for any number of edges. No edge brings a phi of this join point, so every phi here gets a new
- * base with every edge.
- */
-static int mk_phi(struct bpf_verifier_env *env, u32 a, u32 b, int idx, u32 reg)
-{
- struct scev *scev = env->scev;
- u32 base_a, base_b;
- s64 off_a, off_b;
- struct expr e;
- int id;
-
- id = mk_any(scev, a, b, no_def);
- if (id < 0)
- return id;
- e = scev->exprs[id];
- e.def = phi_def(idx, reg);
- e.base = 0;
- split_const_offset64(env, a, &base_a, &off_a);
- split_const_offset64(env, b, &base_b, &off_b);
- if (!expr_is_vn(scev, &e) || !scev->exprs[base_a].is_vn || !scev->exprs[base_b].is_vn)
- return add_expr(scev, e);
- id = base_b;
- if (off_a != off_b)
- id = expr2(scev, BPF_ADD, base_b, imm_expr(scev, (u64)off_b - off_a));
- if (id < 0)
- return id;
- id = add_expr(scev, (struct expr){ .op = ANY, .params = { base_a, id },
- .def = phi_def(idx, BASE_REG) });
- if (id < 0)
- return id;
- e.base = id;
- e.off = off_a;
- return add_expr(scev, e);
-}
-
-/* @acc is the env before the insn at @idx */
-static int join(struct bpf_verifier_env *env, struct env *acc, struct env *cur, int idx)
-{
- struct scev *scev = env->scev;
- int i, j, id;
-
- if (acc->empty) {
- memcpy(acc, cur, sizeof(*acc));
- acc->empty = false;
- return 0;
- }
-
- for (i = 0; i < ENV_REGS_NUM; i++) {
- /* the address names a memory location. It has a phi if all edges track it. */
- j = i < REGS_NUM ? i : REGS_NUM + find_mem_reg(cur, acc->mem_addr[i - REGS_NUM]);
- if (j == ENV_REGS_NUM) {
- forget_mem_reg(acc, i - REGS_NUM);
- continue;
- }
- if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[j])) {
- /* no phi for a dead register: pruned SSA */
- id = reg_alive_at(env, i, idx) ?
- mk_phi(env, acc->reg2expr[i], cur->reg2expr[j], idx, i) :
- UNKNOWN_EXPR_ID;
- if (id < 0)
- return id;
- acc->reg2expr[i] = id;
- }
- }
- return 0;
-}
-
-/* Mark any register modified in 'e' as unknown in 'acc'. */
-static void forget_modified_regs(struct scev *scev, struct env *acc, struct env *e)
-{
- u32 i, reg;
-
- for (i = 0; i < REGS_NUM; i++) {
- if (e && is_reg(scev, e->reg2expr[i], ®) && reg == i)
- continue;
- acc->reg2expr[i] = UNKNOWN_EXPR_ID;
- }
-}
-
-/* For a loop H at `header`, mark any register modified as a result of H execution as unknown. */
-static int forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_loop *loop = aux[header].loop;
- struct scev *scev = env->scev;
- int i, idx, h, err;
-
- acc->empty = false;
- forget_mem_regs(env, acc, 0, 0);
- forget_modified_regs(scev, acc, find_header_env(scev, header));
- for (i = 0; i < loop->exits_cnt; i++) {
- idx = loop->exits[i].from;
- h = bpf_loop_at_index(env, idx);
- /*
- * In order to account for situations like this:
- *
- * header:
- * do {
- * saved = rX;
- * rX = -1000;
- * do { // H
- * if (random())
- * goto out; // E
- * } while (inner_again);
- * rX = saved;
- * } while (outer_again);
- * out:
- *
- * For exits originating in nested loops check that rX is invariant
- * both at the header (H) and at exit (E).
- */
- while (h != header) {
- forget_modified_regs(scev, acc, find_header_env(scev, h));
- forget_modified_regs(scev, acc, find_loop_env(scev, h, idx));
- idx = h;
- h = aux[h].loop_header;
- }
- /*
- * For exits originating at the same loop the header is already accounted for,
- * check only the exit environment.
- */
- forget_modified_regs(scev, acc, find_loop_env(scev, header, idx));
- }
- /* the nested loop defines them at its header */
- for (i = 0; i < REGS_NUM; i++) {
- if (acc->reg2expr[i] != UNKNOWN_EXPR_ID)
- continue;
- err = forget_reg(scev, acc, i, UNKNOWN_EXPR_ID, header);
- if (err)
- return err;
- }
- return 0;
-}
-
-/* Order worklist in reverse post-order. */
-static bool reverse_postorder_less(const void *a, const void *b, void *arg)
-{
- const int *rank = arg;
-
- return rank[*(const int *)a] > rank[*(const int *)b];
-}
-
-static const struct min_heap_callbacks worklist_cb = {
- .less = reverse_postorder_less,
-};
-
-static int worklist_push(struct bpf_verifier_env *env, int loop_header, int idx)
-{
- struct scev *scev = env->scev;
- struct scev_worklist *worklist = &scev->worklist;
- u32 tag = loop_header + 2; /* -1 is a subprog */
-
- if (scev->discovered[idx] == tag)
- return 0;
-
- if (min_heap_full_inline(worklist)) {
- size_t size = worklist->size ? worklist->size * 2 : 16;
- int *data;
-
- data = kvrealloc(worklist->data, array_size(size, sizeof(*data)),
- BPF_VERIFIER_GFP);
- if (!data)
- return -ENOMEM;
- worklist->data = data;
- worklist->size = size;
- }
- if (!min_heap_push_inline(worklist, &idx, &worklist_cb, env->cfg.postorder_nums))
- return -EFAULT;
-
- scev->discovered[idx] = tag;
- return 0;
-}
-
-static bool reg_alive_at_succ(struct bpf_verifier_env *env, u32 r, u32 insn_idx)
-{
- struct bpf_iarray *succ = bpf_insn_successors(env, insn_idx);
- u32 i;
-
- for (i = 0; i < succ->cnt; i++)
- if (reg_alive_at(env, r, succ->items[i]))
- return true;
- return false;
-}
-
-enum changes_at {
- LOG_AT_TRANSFER,
- LOG_AT_JOIN,
-};
-
-static void log_env_changes(struct bpf_verifier_env *env, enum changes_at at,
- struct env *old, struct env *new, int insn_idx)
-{
- struct bpf_verifier_log *log = &env->log;
- struct scev *scev = env->scev;
- u64 null_pos = log->end_pos;
- u32 old_id, new_id, regno;
- bool any_changes = false;
- bool is_self_scev;
- u64 len;
- int r;
-
- bpf_log(log, "%s %4d: ", at == LOG_AT_TRANSFER ? "t" : "j", insn_idx);
- bpf_verbose_insn(env, &env->prog->insnsi[insn_idx]);
- len = log->end_pos - null_pos;
- bpf_log(log, "%*s", max(37 - (int)len, 1), " ");
- bpf_log(log, " ; ");
- for (r = 0; r < ENV_REGS_NUM; r++) {
- old_id = old->reg2expr[r];
- new_id = new->reg2expr[r];
- if (at == LOG_AT_TRANSFER && !reg_alive_at_succ(env, r, insn_idx))
- continue;
- if (at == LOG_AT_JOIN && !reg_alive_at(env, r, insn_idx))
- continue;
- if (same_exprs(scev, old_id, new_id))
- continue;
- is_self_scev = is_reg(scev, new_id, ®no) && r == regno;
- if (old->empty && is_self_scev)
- continue;
- if (any_changes)
- bpf_log(log, ", ");
- log_reg(env, r);
- if (r >= REGS_NUM && new->mem_addr[r - REGS_NUM]) {
- bpf_log(log, "=*");
- log_expr(env, new->mem_addr[r - REGS_NUM]);
- }
- bpf_log(log, " ");
- log_expr(env, old_id);
- bpf_log(log, " -> ");
- log_expr(env, new_id);
- any_changes = true;
- }
- if (old->empty && !any_changes)
- bpf_log(log, "<empty join>");
- bpf_log(log, "\n");
- if (!old->empty && !any_changes)
- bpf_vlog_reset(log, null_pos);
-}
-
-static bool is_probe_read_helper(u32 func_id)
-{
- return func_id == BPF_FUNC_probe_read ||
- func_id == BPF_FUNC_probe_read_kernel ||
- func_id == BPF_FUNC_probe_read_user ||
- func_id == BPF_FUNC_probe_read_str ||
- func_id == BPF_FUNC_probe_read_kernel_str ||
- func_id == BPF_FUNC_probe_read_user_str;
-}
-
-/*
- * If instruction is an indirect write to stack, invalidate SCEVs for spi's
- * that this instruction can touch.
- */
-static int reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- struct scev *scev = env->scev;
- u8 class = BPF_CLASS(insn->code);
- const unsigned long *mask;
- u32 spi, reg;
- bool opaque;
- int err;
-
- /* Direct fp stores are fine. */
- if ((class == BPF_ST || class == BPF_STX) && insn->dst_reg == BPF_REG_FP)
- return 0;
-
- mask = bpf_may_write_mask(env, idx);
- opaque = bpf_helper_call(insn) && is_probe_read_helper(insn->imm);
- for_each_set_bit(spi, mask, MAX_BPF_STACK_SLOTS) {
- reg = MAX_BPF_REG + spi;
- /* '?' stays '?' */
- err = forget_reg(scev, cur_env, reg,
- opaque && !is_op(scev, cur_env->reg2expr[reg], UNKNOWN) ?
- OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID, idx);
- if (err)
- return err;
- }
- return 0;
-}
-
-/*
- * Collect loop-entry registers referenced by expr 'id', best effort.
- * DEPTH_LIMIT may leave dependencies unrecorded. This mask only avoids
- * widening that would lose stack-access precision; missing dependencies
- * may cause false rejections, but this is not a soundness issue.
- */
-static void or_expr_regs(struct bpf_verifier_env *env, u32 id, unsigned long *mask)
-{
- struct scev *scev = env->scev;
- u32 order;
-
- scev->stack_sz = 0;
- expr_stack_push(scev, id);
- while (expr_next(scev, &id, &order)) {
- if ((order & PRE) && scev->exprs[id].op == REG)
- __set_bit(scev->exprs[id].params[0], mask);
- }
-}
-
-/* Mask of argument registers (R1..R5) a call at 'idx' passes by register. */
-static u16 call_params_mask(struct bpf_verifier_env *env, int idx)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- struct bpf_call_summary cs;
- int n = bpf_get_call_summary(env, insn, &cs) ? cs.arg_slot_cnt : MAX_BPF_FUNC_REG_ARGS;
-
- return n ? GENMASK(BPF_REG_1 + n - 1, BPF_REG_1) : 0;
-}
-
-/*
- * For instructions like:
- * - *(u64 *)(rBase + off) = rX
- * - rX = *(u64 *)(rBase + off)
- * - calls that construct objects on stack (e.g. dynptr_from_mem(rBase, ...))
- * When 'rBase' can be a stack pointer and is derived from some registers Rs
- * defined at loop entry, record Rs into 'mask'.
- */
-static void collect_store_base_regs(struct bpf_verifier_env *env,
- struct env *cur_env, int idx, unsigned long *mask)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- u8 class = BPF_CLASS(insn->code);
- u8 size = BPF_SIZE(insn->code);
- u16 base_regs = 0;
- u32 r;
-
- if ((size == BPF_W || size == BPF_DW) &&
- !is_arena_sp_st(insn) && !is_arena_sp_ldx(insn)) {
- if ((class == BPF_STX || class == BPF_ST) && insn->dst_reg != BPF_REG_FP)
- base_regs |= BIT(insn->dst_reg);
- else if (class == BPF_LDX && insn->src_reg != BPF_REG_FP)
- base_regs |= BIT(insn->src_reg);
- }
-
- if (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL &&
- bpf_needs_fixed_stack_off(env, idx))
- base_regs |= call_params_mask(env, idx);
-
- base_regs &= aux[idx].stack_ptrs;
- for (r = 0; r < MAX_BPF_REG; r++)
- if (base_regs & BIT(r))
- or_expr_regs(env, cur_env->reg2expr[r], mask);
-}
-
-/*
- * Record the loop-entry registers that the operands of the conditional jump at 'idx' are
- * computed from, best effort like or_expr_regs().
- */
-static void collect_tested_regs(struct bpf_verifier_env *env, struct env *cur_env, int idx,
- unsigned long *mask)
-{
- struct bpf_insn *insn = &env->prog->insnsi[idx];
-
- if (!bpf_insn_is_cond_jump(insn->code) || BPF_OP(insn->code) == BPF_JCOND)
- return;
- or_expr_regs(env, cur_env->reg2expr[insn->dst_reg], mask);
- if (BPF_SRC(insn->code) == BPF_X)
- or_expr_regs(env, cur_env->reg2expr[insn->src_reg], mask);
-}
-
-/* Find the topmost loop header containing idx inside cur_header, or -1 if none. */
-static int topmost_nested_loop(struct bpf_verifier_env *env, int idx, int cur_header)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- int header;
-
- for (header = bpf_loop_at_index(env, idx); header >= 0;
- header = aux[header].loop_header)
- if (aux[header].loop_header == cur_header)
- return header;
-
- return -1;
-}
-
-/*
- * Join cur_env into the successor's environment and schedule their traversal.
- * Map successors in nested loops to their topmost nested header.
- * Ignore successors outside the current loop and its nested loops.
- */
-static int join_successor(struct bpf_verifier_env *env, int cur_header, int succ_idx,
- struct env *old_env, struct env *cur_env)
-{
- bool log_level2 = env->log.level & BPF_LOG_LEVEL2;
- struct scev *scev = env->scev;
- int err, nested_header, idom;
- struct env *succ_env;
-
- /*
- * There are several possibilities for a successor:
- * - succ_idx can be a part of a loop outside of the cur_header's loop,
- * such edges are ignored.
- * - succ_idx can be a part of the same loop as cur_header,
- * for such edges succ_idx environment is updated:
- * e[succ_idx] = join(e[succ_idx], cur_env)
- * - succ_idx can be a part of some loop inner to cur_header,
- * in such a case there exists some loop header H,
- * such that H.loop_header == cur_header
- * and H is the same as succ_idx's loop or contains it.
- */
- if (bpf_loop_at_index(env, succ_idx) != cur_header) {
- /*
- * topmost_nested_loop() either finds H or returns -1,
- * in case if succ_idx is a part of a loop outer to cur_header.
- */
- nested_header = topmost_nested_loop(env, succ_idx, cur_header);
- if (nested_header < 0)
- return 0;
- succ_idx = nested_header;
- }
- succ_env = get_loop_env(scev, cur_header, succ_idx);
- if (!succ_env)
- return -ENOMEM;
- /*
- * The header of an irreducible loop does not dominate its body. An insn whose immediate
- * dominator is outside of the loop is reached from a secondary entry, without the header.
- * The env at the source of an entering edge is not known: all '?', like a new env. So the
- * insn is a join point with a phi for every live register, see join_added_edges().
- */
- idom = env->idoms[succ_idx];
- if (cur_header >= 0 && succ_idx != cur_header &&
- env->insn_aux_data[cur_header].loop->irreducible &&
- bpf_loop_at_index(env, idom) != cur_header &&
- topmost_nested_loop(env, idom, cur_header) < 0)
- succ_env->empty = false;
- if (log_level2)
- memcpy(old_env, succ_env, sizeof(*old_env));
- err = join(env, succ_env, cur_env, succ_idx);
- if (err)
- return err;
- err = worklist_push(env, cur_header, succ_idx);
- if (err)
- return err;
- if (log_level2)
- log_env_changes(env, LOG_AT_JOIN, old_env, succ_env, succ_idx);
- return 0;
-}
-
-/*
- * Recurrences need a natural loop. Unsupported loops get an unknown header environment.
- * A nested loop is processed as a single instruction, its exits are the successors: all of
- * them are needed. No result for the loop itself depends on the list of its own exits:
- * mark_latches() prepares them for the loop around, and compute_max_iters() asks if there is
- * only one, which a list that is cut does not have.
- */
-static bool can_compute_loop_scev(const struct bpf_loop *loop)
-{
- return !loop->irreducible && !loop->nested_exits_overflow;
-}
-
-/* A loop is a single instruction for the subprog too: no exit list may be truncated. */
-static bool can_compute_subprog_scev(struct bpf_verifier_env *env, struct bpf_subprog_info *sub)
-{
- struct bpf_loop *loop;
- int i;
-
- for (i = sub->start; i < (sub + 1)->start; i++) {
- loop = env->insn_aux_data[i].loop;
- if (loop && (loop->exits_overflow || loop->nested_exits_overflow))
- return false;
- }
- return true;
-}
-
-/*
- * The edges that the verifier adds, see bpf_head_guard_target(): bpf_insn_successors() doesn't have
- * them. They leave all loops. The env at their source is not known: @cur_env is all '?'. So their
- * target is a join point with a phi for every live register.
- */
-static int join_added_edges(struct bpf_verifier_env *env, struct bpf_subprog_info *sub,
- struct env *old_env, struct env *cur_env)
-{
- int i, t, err;
-
- for (i = sub->start; i < (sub + 1)->start; i++) {
- t = bpf_head_guard_target(env, i);
- if (t < 0)
- continue;
- err = join_successor(env, -1, t, old_env, cur_env);
- if (err)
- return err;
- }
- return sub->bail_insn < 0 ? 0 : join_successor(env, -1, sub->bail_insn, old_env, cur_env);
-}
-
-/*
- * @id == *@base + *@off: constant offsets are stripped, see set_expr_base(). Returns the number of
- * bits in which that holds. An add with a def holds modulo 2^32 whether its insn is converted to
- * ALU32 or not, see alu64_with_def(). It holds modulo 2^64 once the main pass has set prevent_zext:
- * it verified the insn without a pointer to the arena and rejects a state that brings one. The def
- * dominates every use, and prevent_zext is only ever set: such an add is not checked again.
- */
-static u32 split_const_offset(struct bpf_verifier_env *env, u32 id, u32 *base, s64 *off)
-{
- struct scev *scev = env->scev;
- struct expr *add, *e = &scev->exprs[id];
-
- *base = e->base;
- *off = e->off;
- while (e->arena_add) {
- add = &scev->exprs[e->arena_add];
- if (!env->insn_aux_data[add->def.idx].prevent_zext)
- return 32;
- e->arena_add = scev->exprs[add->params[0]].arena_add;
- }
- return 64;
-}
-
-static bool const_diff_at_header(struct bpf_verifier_env *env, int h, u32 ra, u32 rb, s64 *delta);
-
-/*
- * @id_a and @id_b are expressions of one env of the loop at @h, or of a subprog: @h is -1. Do they
- * have a constant difference, @id_b == @id_a + *@delta (mod 2^*@bits)?
- * n = (u16)x; m = n + 600; if ((u32)(n - 1) < 256) use(m);
- * m and the tested value are x + a constant modulo 2^16, and n + a constant modulo 2^32: take the
- * outermost base that both have.
- */
-static bool const_diff(struct bpf_verifier_env *env, int h, u32 id_a, u32 id_b, s64 *delta,
- u32 *bits)
-{
- struct scev *scev = env->scev;
- u32 base_a, base_b, bits_a = 64, bits_b, ra, rb;
- s64 off_a, off_b, c;
-
- *delta = 0;
- *bits = 64;
- /* copies are equal in all bits */
- if (id_a == id_b && scev->exprs[id_a].is_vn)
- return true;
- *bits = split_const_offset(env, id_a, &base_a, &off_a);
- do {
- *bits = min(*bits, split_const_offset(env, id_b, &base_b, &off_b));
- bits_b = 64;
- do {
- c = 0;
- /* header phis: rb == ra + c */
- if (base_a == base_b ? !scev->exprs[base_a].is_vn :
- h < 0 || !is_reg(scev, base_a, &ra) || !is_reg(scev, base_b, &rb) ||
- !const_diff_at_header(env, h, ra, rb, &c))
- continue;
- *delta = (u64)off_b - off_a + c;
- *bits = min3(*bits, bits_a, bits_b);
- return true;
- } while (next_base(scev, &base_b, &off_b, &bits_b));
- } while (next_base(scev, &base_a, &off_a, &bits_a));
- return false;
-}
-
-/*
- * old = i = 0; do { i++; if (old > 1021) break; c = buf[old]; old = i; } while (c);
- * Does @rb == @ra + *@delta hold at the header of the loop at @h in every iteration? It does if it
- * holds at loop entry, by the env of the enclosing loop at @h, and every backedge preserves it: the
- * backedge values have this constant difference, or both are the header phi plus one constant.
- * Loops nest LOOP_STACK_SIZE deep at most, see bpf_flow_place_guards().
- */
-static bool const_diff_at_header(struct bpf_verifier_env *env, int h, u32 ra, u32 rb, s64 *delta)
-{
- int outer_header = env->insn_aux_data[h].loop_header;
- struct env *entry_env = find_loop_env(env->scev, outer_header, h);
- struct env *header_env = find_header_env(env->scev, h);
- u32 base_a, base_b, bits;
- s64 off_a, c;
-
- if (!entry_env || !can_compute_loop_scev(env->insn_aux_data[h].loop) ||
- !const_diff(env, outer_header, entry_env->reg2expr[ra], entry_env->reg2expr[rb],
- delta, &bits) || bits != 64)
- return false;
- if (const_diff(env, -1, header_env->reg2expr[ra], header_env->reg2expr[rb], &c, &bits))
- return bits == 64 && c == *delta;
- return split_const_offset(env, header_env->reg2expr[ra], &base_a, &off_a) == 64 &&
- split_const_offset(env, header_env->reg2expr[rb], &base_b, &c) == 64 &&
- base_a == REG_EXPR_ID(ra) && base_b == REG_EXPR_ID(rb) && off_a == c;
-}
-
-/*
- * const_diff() for the SCEV register @reg, which has the expression @id_b. A fill from byte
- * *@fill_off of a slot reads these bits. On big endian the low bytes of a spill are the last ones,
- * and a narrow fill from the start of a slot is a right shift, see mk_fill().
- */
-static bool const_diff_reg(struct bpf_verifier_env *env, int h, u32 id_a, u32 reg, u32 id_b,
- u32 *fill_off, s64 *delta, u32 *bits)
-{
- struct scev *scev = env->scev;
- struct expr *e;
- u32 x;
- s64 k;
-
- *fill_off = 0;
- if (const_diff(env, h, id_a, id_b, delta, bits)) {
- if (is_big_endian_slot(reg))
- *fill_off = expr_spill_size(scev, id_b, &x) - *bits / BITS_PER_BYTE;
- return true;
- }
- if (!is_big_endian_slot(reg) || !scev->exprs[id_b].is_vn)
- return false;
- *bits = split_const_offset(env, id_a, &x, delta);
- while (next_base(scev, &x, delta, bits))
- ;
- e = &scev->exprs[x];
- if ((e->op != BPF_RSH && e->op != BPF_ARSH) || e->def.set || e->params[0] != id_b ||
- !is_imm(scev, e->params[1], &k) || (k != 32 && k != 48 && k != 56))
- return false;
- *delta = -(u64)*delta;
- *bits = min_t(u32, *bits, 64 - k);
- *fill_off = (64 - k - *bits) / BITS_PER_BYTE;
- return true;
-}
-
-/* The first of the registers and stack slots @live that holds @id + a constant. REGS_NUM: none. */
-static u32 find_leader(struct bpf_verifier_env *env, struct env *e, int h,
- const unsigned long *live, u32 id)
-{
- u32 r, bits;
- s64 delta;
-
- for_each_set_bit(r, live, REGS_NUM)
- if (const_diff(env, h, e->reg2expr[r], id, &delta, &bits))
- break;
- return r;
-}
-
-/* The linked registers of a conditional jump */
-struct jump_links {
- struct scev_linked_reg *regs;
- u32 regs_cnt;
- /* the loop of the expressions, -1: a subprog */
- int h;
- /* r0 - r10: what the jump refined */
- u16 refined_regs;
- /* their innermost bases */
- u32 tested_base[2];
- /* One is a header phi: these can have a constant difference, see const_diff_at_header() */
- bool tested_phi;
-};
-
-static bool is_refined_reg(const struct jump_links *t, u32 reg)
-{
- return reg < MAX_BPF_REG && (t->refined_regs & BIT(reg));
-}
-
-/* Make room for element @cnt of the array *@p, whose elements have @size bytes */
-static int grow_array(void *p, u32 cnt, u32 *cap, size_t size)
-{
- u32 new_cap = max_t(u32, roundup_pow_of_two(cnt + 1), 16);
- void *tmp;
-
- if (cnt < *cap)
- return 0;
- tmp = kvrealloc(*(void **)p, array_size(new_cap, size), BPF_VERIFIER_GFP);
- if (!tmp)
- return -ENOMEM;
- *(void **)p = tmp;
- *cap = new_cap;
- return 0;
-}
-
-/*
- * The op and the operands that the value of the base @id is computed from. NULL: it is a leaf, its
- * value is what a leader holds.
- */
-static struct expr *expr_to_eval(struct bpf_verifier_env *env, int h, u32 id)
-{
- struct scev *scev = env->scev;
- struct expr *e = &scev->exprs[id];
- u32 reg;
-
- /* A header phi can be an add of two header phis. Its other SCEVs have entry values. */
- if (h >= 0 && is_reg(scev, id, ®)) {
- e = &scev->exprs[find_header_env(scev, h)->reg2scev[reg]];
- if (e->op != ADD_SCEV)
- return NULL;
- }
- /* nothing looks through a phi, see struct expr_def */
- if (e->def.phi)
- return NULL;
- return op_params_num(e->op) ? e : NULL;
-}
-
-static struct scev_linked_expr *find_linked_expr(struct scev *scev, u32 id)
-{
- u32 n = scev->exprs[id].linked_expr;
-
- if (n >= scev->linked_exprs_cnt || scev->linked_exprs[n].id != id)
- return NULL;
- return &scev->linked_exprs[n];
-}
-
-/*
- * Add the base of @id to the linked expressions, after its operands. Returns its position. An
- * expression is a DAG, not a tree: a base that is there is not entered again. Below depth
- * EXPR_STACK_DEPTH a base is a leaf.
- */
-static int add_linked_expr(struct bpf_verifier_env *env, const struct jump_links *t, u32 id,
- u32 depth)
-{
- struct scev *scev = env->scev;
- struct scev_linked_expr x = { .id = scev->exprs[id].base, .params = { NO_IDX, NO_IDX },
- .leaders = { NO_IDX, NO_IDX, NO_IDX } };
- struct expr *e;
- u32 i, base;
- int n;
-
- if (find_linked_expr(scev, x.id))
- return scev->exprs[x.id].linked_expr;
- base = innermost_base(scev, x.id);
- x.tested_base = base == t->tested_base[0] || base == t->tested_base[1] ||
- (t->tested_phi && is_op(scev, base, REG));
- x.has_tested = x.tested_base;
- e = expr_to_eval(env, t->h, x.id);
- for (i = 0; e && depth < EXPR_STACK_DEPTH && i < op_params_num(e->op); i++) {
- /* the operands of ADD_SCEV are leaves: they could be adds of each other */
- n = add_linked_expr(env, t, e->params[i],
- e->op == ADD_SCEV ? EXPR_STACK_DEPTH : depth + 1);
- if (n < 0)
- return n;
- x.params[i] = n;
- x.has_tested |= scev->linked_exprs[n].has_tested;
- }
- n = scev->linked_exprs_cnt;
- if (n >= NO_IDX)
- return -E2BIG;
- if (grow_array(&scev->linked_exprs, n, &scev->linked_exprs_cap, sizeof(x)) ||
- grow_array(&scev->vals, n, &scev->vals_cap, sizeof(*scev->vals)))
- return -ENOMEM;
- scev->linked_exprs[scev->linked_exprs_cnt++] = x;
- scev->exprs[x.id].linked_expr = n;
- return n;
-}
-
-/* The linked register at position @k holds @x + a constant (mod 2^@bits) */
-static void add_leader(struct scev_linked_expr *x, u32 k, u32 bits, bool tested)
-{
- u32 i = 0;
-
- if (tested) {
- while (x->leaders[i] != NO_IDX && x->leaders[i] != k)
- i++;
- x->leaders[i] = k;
- } else if (bits > x->leader_bits) {
- x->leaders[LEADERS_NUM - 1] = k;
- x->leader_bits = bits;
- }
-}
-
-/*
- * List the bases below the expressions of the linked registers at the positions @roots, and find
- * their leaders among all of @t. Most registers don't depend on a tested register: they leave
- * @roots. -ENOENT: nothing but tested registers is left.
- */
-static int add_linked_exprs(struct bpf_verifier_env *env, struct jump_links *t,
- unsigned long *roots)
-{
- struct scev_linked_reg *q, *linked = t->regs;
- struct scev *scev = env->scev;
- struct scev_linked_expr *x;
- u32 i, k, id, base, bits, fill_off, min_id, linked_cnt = 0;
- s64 off;
- int n;
-
- scev->linked_exprs_cnt = 0;
- for (k = 0; k < t->regs_cnt; k++) {
- linked[k].expr = NO_IDX;
- if (!is_refined_reg(t, linked[k].reg))
- continue;
- t->tested_base[1] = innermost_base(scev, linked[k].id);
- if (!t->tested_base[0])
- t->tested_base[0] = t->tested_base[1];
- }
- t->tested_phi = t->h >= 0 && (is_op(scev, t->tested_base[0], REG) ||
- is_op(scev, t->tested_base[1], REG));
- min_id = t->tested_phi ? 0 : min(t->tested_base[0], t->tested_base[1]);
- for_each_set_bit(k, roots, t->regs_cnt) {
- q = &linked[k];
- id = q->id;
- /* an expression is constructed after its operands */
- if (scev->exprs[id].base < min_id && !is_big_endian_slot(q->reg)) {
- __clear_bit(k, roots);
- continue;
- }
- /* the start of the slot was filled and tested, see const_diff_reg() */
- for (i = 0; is_big_endian_slot(q->reg) && i < k; i++)
- if (is_refined_reg(t, linked[i].reg) &&
- !const_diff(env, t->h, linked[i].id, q->id, &off, &bits) &&
- const_diff_reg(env, t->h, linked[i].id, q->reg, q->id, &fill_off, &off,
- &bits))
- id = linked[i].id;
- n = add_linked_expr(env, t, id, 0);
- if (n < 0)
- return n;
- if (!scev->linked_exprs[n].has_tested) {
- __clear_bit(k, roots);
- continue;
- }
- q->expr = n;
- linked_cnt += !is_refined_reg(t, q->reg);
- }
- if (!linked_cnt)
- return -ENOENT;
- /*
- * A register holds each of its bases + a constant: a leader is found by base. Memory is no
- * leader: which memory it is depends on the state.
- */
- for (k = 0; k < t->regs_cnt && linked[k].reg < REGS_NUM; k++) {
- if (is_refined_reg(t, linked[k].reg)) {
- for (x = scev->linked_exprs;
- x < scev->linked_exprs + scev->linked_exprs_cnt; x++)
- if (x->tested_base &&
- const_diff(env, t->h, linked[k].id, x->id, &off, &bits))
- add_leader(x, k, bits, true);
- continue;
- }
- base = scev->exprs[linked[k].id].base;
- bits = 64;
- off = 0;
- do {
- x = find_linked_expr(scev, base);
- if (x)
- add_leader(x, k, bits, false);
- } while (next_base(scev, &base, &off, &bits));
- }
- return 0;
-}
-
-/*
- * n = bpf_get_stack(..); m = n; if (n < 0) ..; use(m);
- * p = &hdr[i]; if (i > 1) return 0; p->len = 0;
- * The main pass refines the registers that the conditional jump at @idx tests. Find the live SCEV
- * registers whose expression in @e, the env before the jump, can be evaluated from a tested
- * register and other live SCEV registers. Keep what bpf_scev_next_linked_reg() needs of @e: these
- * registers and the leaders, with their expressions.
- */
-static int find_linked_regs(struct bpf_verifier_env *env, struct env *e, int h, int idx)
-{
- struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- struct jump_links t = { .h = h, .refined_regs = BIT(insn->dst_reg) };
- struct insn_envs *envs = env->scev->envs[idx];
- struct scev *scev = env->scev;
- u32 first = scev->linked_cnt;
- DECLARE_BITMAP(live, ENV_REGS_NUM) = {};
- DECLARE_BITMAP(roots, ENV_REGS_NUM);
- DECLARE_BITMAP(found, ENV_REGS_NUM) = {};
- struct scev_linked_expr *x, *l, *p;
- struct scev_linked_reg *q;
- u32 i, k, r;
- bool op;
- int err;
-
- if (!bpf_insn_is_cond_jump(insn->code) || BPF_OP(insn->code) == BPF_JCOND)
- return 0;
- if (BPF_SRC(insn->code) == BPF_X)
- t.refined_regs |= BIT(insn->src_reg);
- bitmap_copy(live, aux->live_stack_before, MAX_BPF_STACK_SLOTS);
- bitmap_shift_left(live, live, MAX_BPF_REG, REGS_NUM);
- live[0] |= aux->live_regs_before;
- for (r = REGS_NUM; r < ENV_REGS_NUM; r++)
- if (e->mem_addr[r - REGS_NUM])
- __set_bit(r, live);
- for_each_set_bit(r, live, ENV_REGS_NUM) {
- if (grow_array(&scev->linked, scev->linked_cnt, &scev->linked_cap,
- sizeof(*scev->linked)))
- return -ENOMEM;
- scev->linked[scev->linked_cnt++] = (struct scev_linked_reg){
- .reg = r, .id = e->reg2expr[r],
- .addr = r < REGS_NUM ? 0 : e->mem_addr[r - REGS_NUM] };
- }
- t.regs = &scev->linked[first];
- t.regs_cnt = scev->linked_cnt - first;
- scev->linked_cnt = first;
- bitmap_fill(roots, t.regs_cnt);
- err = add_linked_exprs(env, &t, roots);
- /* -E2BIG: too many expressions */
- if (err)
- return err == -E2BIG || err == -ENOENT ? 0 : err;
- /* operands come first */
- for (x = scev->linked_exprs; x < scev->linked_exprs + scev->linked_exprs_cnt; x++) {
- l = x->params[0] == NO_IDX ? NULL : &scev->linked_exprs[x->params[0]];
- p = x->params[1] == NO_IDX ? l : &scev->linked_exprs[x->params[1]];
- op = l && l->has_value && p->has_value;
- x->has_value = op || x->leader_bits || x->leaders[0] != NO_IDX ||
- is_op(scev, x->id, IMM);
- x->has_tested = x->leaders[0] != NO_IDX ||
- (op && (l->has_tested || p->has_tested));
- }
- for_each_set_bit(k, roots, t.regs_cnt) {
- q = &t.regs[k];
- x = &scev->linked_exprs[q->expr];
- r = q->reg < REGS_NUM ? q->reg : find_leader(env, e, h, live, q->addr);
- /* a tested register learns from the other one only */
- if (!x->has_tested || (is_refined_reg(&t, q->reg) && x->leaders[1] == NO_IDX) ||
- /* the main pass finds memory by a live register that holds its address */
- r == REGS_NUM) {
- __clear_bit(k, roots);
- continue;
- }
- x->needed = true;
- __set_bit(bitmap_weight(live, r), found);
- }
- if (bitmap_empty(roots, t.regs_cnt))
- return 0;
- bitmap_or(found, found, roots, t.regs_cnt);
- /* operands come first */
- for (x = scev->linked_exprs + scev->linked_exprs_cnt; x-- > scev->linked_exprs;) {
- for (i = 0; x->needed && i < LEADERS_NUM; i++)
- if (x->leaders[i] != NO_IDX)
- __set_bit(x->leaders[i], found);
- for (i = 0; x->needed && i < 2 && x->params[i] != NO_IDX; i++)
- scev->linked_exprs[x->params[i]].needed = true;
- }
- for_each_set_bit(k, found, t.regs_cnt) {
- q = &scev->linked[scev->linked_cnt++];
- *q = t.regs[k];
- q->refine = test_bit(k, roots);
- }
- if (!envs) {
- envs = kzalloc(sizeof(*envs), BPF_VERIFIER_GFP);
- if (!envs)
- return -ENOMEM;
- scev->envs[idx] = envs;
- }
- envs->linked_first = first;
- envs->linked_cnt = scev->linked_cnt - first;
- return 0;
-}
-
-/*
- * The memory location @regno is a linked register of the conditional jump at @insn_idx. Its address
- * is *@base_regno + *@off. *@base_regno is a live register or stack slot. Which memory it is
- * depends on the state.
- */
-bool bpf_scev_mem_addr(struct bpf_verifier_env *env, int insn_idx, u32 regno, u32 *base_regno,
- s64 *off)
-{
- struct insn_envs *envs = env->scev->envs[insn_idx];
- struct scev_linked_reg *linked = &env->scev->linked[envs->linked_first];
- int h = bpf_loop_at_index(env, insn_idx);
- u32 i, j, bits;
-
- for (i = 0; linked[i].reg != regno; i++)
- if (i + 1 == envs->linked_cnt)
- return false;
- for (j = 0; linked[j].reg < REGS_NUM; j++) {
- if (!const_diff(env, h, linked[j].id, linked[i].addr, off, &bits) || bits != 64)
- continue;
- *base_regno = linked[j].reg;
- return true;
- }
- return false;
-}
-
-/*
- * @cur_header and @first are a loop header. Or -1 and the first insn of a subprog: the subprog is
- * the outermost loop, which runs once. (REG r) is the entry value of r, the loops in it are single
- * instructions.
- * Value numbering works on any CFG: the body of an irreducible loop is walked like any other. Every
- * retreating edge of the depth-first search goes to a loop header, see compute_loops_in_subprog(),
- * and bpf_compute_postorder() does the same search. So reverse post-order is a topological order of
- * this body too.
- */
-static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header, int first)
-{
- struct scev_worklist *worklist = &env->scev->worklist;
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_loop *cur_loop = cur_header < 0 ? NULL : aux[cur_header].loop;
- struct bpf_subprog_info *sub = bpf_find_containing_subprog(env, first);
- struct env *header_env;
- struct scev *scev = env->scev;
- struct bpf_loop *nested_loop;
- struct env *cur_env = NULL;
- struct env *old_env = NULL;
- struct bpf_iarray *succ;
- bool log_level2 = env->log.level & BPF_LOG_LEVEL2;
- int s, i, err, idx, succ_idx, start;
- u32 r;
-
- if (log_level2)
- bpf_log(&env->log, "Computing SCEV for %s at %d:\n",
- cur_loop ? "loop" : "subprog", first);
-
- if (!cur_loop && !can_compute_subprog_scev(env, sub))
- return 0;
-
- cur_env = kzalloc_obj(*cur_env, BPF_VERIFIER_GFP);
- if (!cur_env)
- goto nomem;
- if (log_level2) {
- old_env = kzalloc_obj(*old_env, BPF_VERIFIER_GFP);
- if (!old_env)
- goto nomem;
- }
- header_env = get_loop_env(scev, cur_header, first);
- if (!header_env)
- goto nomem;
- setup_initial_loop_env(header_env);
- /* The successors of a nested loop with a truncated exit list are not known: no walk */
- if (!cur_loop || !cur_loop->nested_exits_overflow) {
- err = worklist_push(env, cur_header, first);
- if (err)
- goto out;
- }
- if (!cur_loop) {
- /* no backedge overwrites it */
- header_env->empty = false;
- err = join_added_edges(env, sub, old_env, cur_env);
- if (err)
- goto out;
- }
-
- while (worklist->nr) {
- idx = *min_heap_peek_inline(worklist);
- min_heap_pop_inline(worklist, &worklist_cb, env->cfg.postorder_nums);
- start = idx;
- if (bpf_must_abort()) {
- err = -EINTR;
- goto out;
- }
-
- /* join_successor() maps nested-loop entries to their representative header. */
- nested_loop = idx != cur_header ? aux[idx].loop : NULL;
- memcpy(cur_env, find_loop_env(scev, cur_header, idx), sizeof(*cur_env));
- if (nested_loop) {
- /*
- * Process nested loop as a single instruction by
- * forgetting anything non-invariant in the nested loop
- */
- if (log_level2)
- memcpy(old_env, cur_env, sizeof(*old_env));
- if (cur_loop) {
- /* Pull the nested loop's stack-store base dependencies up. */
- for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM)
- or_expr_regs(env, cur_env->reg2expr[r],
- cur_loop->store_base_regs);
- /* And what its jumps depend on. */
- for_each_set_bit(r, nested_loop->tested_regs, BPF_SCEV_REGS_NUM)
- or_expr_regs(env, cur_env->reg2expr[r],
- cur_loop->tested_regs);
- }
- err = forget_non_invariants(env, cur_env, idx);
- if (err)
- goto out;
- if (log_level2)
- log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx);
- /*
- * Treat nested loop exits as successors,
- * join cur_env into successor's envs.
- */
- for (i = 0; i < nested_loop->exits_cnt; i++) {
- s = nested_loop->exits[i].to;
- err = join_successor(env, cur_header, s, old_env, cur_env);
- if (err)
- goto out;
- }
- } else {
- /*
- * Iterate instructions within a single basic block
- * starting at 'idx' mutating 'cur_env'.
- */
- for (;;) {
- if (log_level2)
- memcpy(old_env, cur_env, sizeof(*old_env));
- if (cur_loop) {
- collect_store_base_regs(env, cur_env, idx,
- cur_loop->store_base_regs);
- collect_tested_regs(env, cur_env, idx,
- cur_loop->tested_regs);
- }
- /* the others wait for the envs of all loop headers */
- if (!aux[idx].need_scev) {
- err = find_linked_regs(env, cur_env, -1, idx);
- if (err)
- goto out;
- }
- err = transfer(env, cur_env, idx);
- if (err)
- goto out;
- err = reset_scevs_at_indirect_writes(env, cur_env, idx);
- if (err)
- goto out;
- if (log_level2)
- log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx);
- succ = bpf_insn_successors(env, idx);
- if (succ->cnt != 1)
- break;
- succ_idx = succ->items[0];
- if (aux[idx].bb_end || aux[succ_idx].need_scev ||
- bpf_loop_at_index(env, succ_idx) != cur_header)
- break;
- idx = succ_idx;
- }
- /* Join cur_env into basic block successor's envs. */
- iarray_for_each(s, succ) {
- err = join_successor(env, cur_header, s, old_env, cur_env);
- if (err)
- goto out;
- }
- }
- /* Reverse post-order: all predecessors came first, the env has no users any more */
- if (!aux[start].need_scev)
- free_loop_env(scev, cur_header, start);
- }
-
- /* For the enclosing loop and for the main pass it clobbers everything */
- if (cur_loop && !can_compute_loop_scev(cur_loop)) {
- memset(header_env, 0, sizeof(*header_env));
- bitmap_fill(cur_loop->store_base_regs, REGS_NUM);
- bitmap_fill(cur_loop->tested_regs, REGS_NUM);
- }
- err = 0;
-out:
- kfree(cur_env);
- kfree(old_env);
- return err;
-nomem:
- err = -ENOMEM;
- goto out;
-}
-
-static void mark_latches(struct bpf_verifier_env *env)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_loop *loop;
- int len = env->prog->len;
- int i, j, k, from;
-
- for (i = 0; i < len; i++) {
- /* An exit by way of a flag. It may have no place yet, see bpf_scev_jump_counts() */
- if (aux[i].edge_leaves &&
- can_compute_loop_scev(aux[bpf_loop_at_index(env, i)].loop))
- aux[i].need_scev = true;
- loop = aux[i].loop;
- if (!loop)
- continue;
- aux[i].need_scev = true;
- if (!can_compute_loop_scev(loop))
- continue;
- /*
- * Make sure to only set .need_scev for exits and latches
- * belonging to the loop at `i`. If can_compute_loop_scev()
- * fails for a loop, it should hide it's instructions from
- * compute_insn_scevs().
- */
- for (j = 0; j < loop->backedges_cnt; j++)
- for (k = 0; k < loop->backedges[j].latches_cnt; k++)
- aux[loop->backedges[j].latches[k]].need_scev = true;
- for (j = 0; j < loop->exits_cnt; j++) {
- from = loop->exits[j].from;
- if (bpf_loop_at_index(env, from) == i)
- aux[from].need_scev = true;
- }
- }
-}
-
-static bool is_any_imm_reg_opaque(struct scev *scev, u32 id)
-{
- u32 l, r, reg, order;
- s64 imm;
-
- scev->stack_sz = 0;
- expr_stack_push(scev, id);
- while (expr_next(scev, &id, &order)) {
- if (order & DEPTH_LIMIT)
- return false;
- if (!(order & PRE) ||
- is_imm(scev, id, &imm) ||
- is_reg(scev, id, ®) ||
- is_opaque(scev, id) ||
- is_any(scev, id, &l, &r))
- continue;
- return false;
- }
- return true;
-}
-
-/*
- * Can implement explicit stack version, but it is harder to read.
- * Stick with recursive version for now.
- */
-static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv,
- int (*fn)(struct scev *scev, u32 id, void *priv))
-{
- struct expr expr;
- int p0, p1, id;
-
- if (lvl >= EXPR_STACK_DEPTH)
- return UNKNOWN_EXPR_ID;
-
- expr = scev->exprs[root]; /* snapshot the expr before potential realloc */
- switch (op_params_num(expr.op)) {
- case 0:
- id = root;
- break;
- case 1:
- p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn);
- id = expr1(scev, expr.op, p0);
- break;
- case 2:
- p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn);
- p1 = transform_expr_once(scev, lvl + 1, expr.params[1], priv, fn);
- id = expr2(scev, expr.op, p0, p1);
- break;
- }
- return id < 0 ? id : fn(scev, id, priv);
-}
-
-static int simplify(struct scev *scev, u32 id)
-{
- int a, b, l, r, t, base, slope;
- u32 op = scev->exprs[id].op, bits = op_low_bits(op);
- s64 imm1, imm2;
-
- /* (- x imm) -> (+ x -imm) */
- if (is_binop(scev, BPF_SUB, id, &l, &r) &&
- is_imm(scev, r, &imm1) && imm1 != S64_MIN)
- return expr2(scev, BPF_ADD, l, imm_expr(scev, -imm1));
-
- /* (+ x 0) -> x */
- if (is_add(scev, id, &l, &r) && is_imm(scev, r, &imm1) && !imm1)
- return l;
-
- /* (+ (+ x imm) imm) -> (+ x imm). Not through a def, see alu64_with_def() */
- if (is_add(scev, id, &l, &r) &&
- is_imm(scev, r, &imm2) &&
- is_add(scev, l, &a, &b) &&
- is_imm(scev, b, &imm1) && !scev->exprs[l].def.set)
- return expr2(scev, BPF_ADD, a, imm_expr(scev, (s64)((u64)imm1 + (u64)imm2)));
-
- /* 'r1 <<= 32; r1 >>= 32' is (u32)r1 for cpu v1 and v2: (>> (<< x 32) 32) -> (zext32 x) */
- if (is_binop(scev, BPF_RSH, id, &l, &r) &&
- is_imm(scev, r, &imm2) && imm2 == 32 &&
- is_binop(scev, BPF_LSH, l, &a, &b) &&
- is_imm(scev, b, &imm1) && imm1 == 32 && !scev->exprs[l].def.set)
- return expr1(scev, ZEXT32, a);
-
- /* 'r1 &= 0xff' is (u8)r1, 'r1 &= 0xffff' is (u16)r1: (& x 0xff) -> (zext8 x) */
- if (is_binop(scev, BPF_AND, id, &l, &r) &&
- is_imm(scev, r, &imm1) && (imm1 == 0xff || imm1 == 0xffff))
- return expr1(scev, imm1 == 0xff ? ZEXT8 : ZEXT16, l);
-
- /* (+ (linear base slope) imm) -> (linear (+ base imm) slope) */
- if (is_add(scev, id, &l, &r) &&
- is_linear(scev, l, &base, &slope) &&
- is_imm(scev, r, &imm1))
- return expr2(scev, LINEAR_SCEV, expr2(scev, BPF_ADD, base, r), slope);
-
- /* (+ imm imm) -> imm */
- if (is_add(scev, id, &l, &r) &&
- is_imm(scev, l, &imm1) &&
- is_imm(scev, r, &imm2))
- return imm_expr(scev, imm1 + imm2);
-
- /* (zext32 (imm X)) -> (imm (u32)X) */
- if (is_zext32(scev, id, &l) && is_imm(scev, l, &imm1))
- return imm_expr(scev, (u64)(u32)imm1);
-
- /*
- * An extension and a partial spill depend on the low bits of their operand only: look
- * through an extension and a partial spill that is not narrower, below an add too:
- * (sext32 (zext32 $a)) -> (sext32 $a)
- * (sext8 (spill16 $a)) -> (sext8 $a)
- * (zext32 (+ (sext32 $a) $b)) -> (zext32 (+ $a $b))
- * (spill32 (+ $a (zext32 $b))) -> (spill32 (+ $a $b))
- */
- if (!bits)
- return id;
- t = scev->exprs[id].params[0];
- /* 'w1 &= 0xff': (zext32 (zext8 $a)) -> (zext8 $a) */
- if (is_zext(op) && is_zext(scev->exprs[t].op) && !scev->exprs[t].def.set &&
- op_low_bits(scev->exprs[t].op) <= bits)
- return t;
- if (is_low_bits(scev, t, bits, &a))
- return expr1(scev, op, a);
- if (is_add(scev, t, &l, &r) && !scev->exprs[t].def.phi &&
- (is_low_bits(scev, l, bits, &l) || is_low_bits(scev, r, bits, &r)))
- return expr1(scev, op, expr2(scev, BPF_ADD, l, r));
-
- return id;
-}
-
-static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv)
-{
- u32 reg, *reg2scev = priv;
-
- /* (reg r) -> ... equation at header ... */
- if (is_reg(scev, id, ®))
- return reg2scev[reg];
- /* keep expression shapes */
- return id;
-}
-
-static int compute_header_scevs(struct bpf_verifier_env *env, int h)
-{
- struct scev *scev = env->scev;
- struct env *header_env = find_header_env(scev, h);
- int l, r, t, id, ra, rb, ra_expr;
- bool zext32;
- s64 imm;
- u32 op;
-
- /*
- * Note:
- * reg2expr[ra] describes a change in rA's value over a single loop iteration.
- * It does not account for rA's entry value. However, the SCEV constructed for rA
- * must include an additional rA entry edge. For example:
- *
- * r0 = ..., r1 = 7
- * l: if r0 == 10 goto e // reg2expr[r1] == (imm 42) => reg2scev[r1] == (any r1 42)
- * r0 += 1
- * r1 = 42
- * goto l
- * e:
- * ... at exit r1 is in {7,42} ...
- */
- for (ra = 0; ra < REGS_NUM; ra++) {
- ra_expr = header_env->reg2expr[ra];
- /* Constructed again it has no defs, see alu64_with_def(): they hide the shape */
- if (env->prog->aux->arena)
- ra_expr = transform_expr_once(scev, 0, ra_expr, scev->identity_env.reg2expr,
- instantiate_header_scevs);
- if (ra_expr < 0)
- return ra_expr;
- /* rA = (+ rA IMM) -> (linear rA IMM) */
- if (is_add(scev, ra_expr, &l, &r) &&
- is_reg(scev, l, &rb) &&
- is_imm(scev, r, &imm) &&
- ra == rb) {
- id = expr2(scev, LINEAR_SCEV, l, r);
- if (id < 0)
- return id;
- header_env->reg2scev[ra] = id;
- continue;
- }
- /*
- * A 32-bit induction variable:
- * rA = (zext32 (+ rA IMM)) -> (zext32 (linear rA IMM))
- * rA = (sext32 (+ rA IMM)) -> (sext32 (linear rA IMM))
- * rA = (spill32 (+ rA IMM)) -> (spill32 (linear (zext32 rA) IMM))
- * The last SCEV leaves the upper 32 bits of the slot unknown, so it holds for the
- * first two as well. They get it if these bits are dead: the states have a
- * 32-bit spill then, see bpf_clean_verifier_state().
- */
- if (is_low_bits(scev, ra_expr, 32, &t) &&
- is_add(scev, t, &l, &r) &&
- is_reg(scev, l, &rb) &&
- is_imm(scev, r, &imm) &&
- ra == rb) {
- op = hi32_alive_at(env, ra, h) ? scev->exprs[ra_expr].op : SPILL32;
- if (op == SPILL32)
- l = expr1(scev, ZEXT32, l);
- id = expr1(scev, op, expr2(scev, LINEAR_SCEV, l, r));
- if (id < 0)
- return id;
- header_env->reg2scev[ra] = id;
- continue;
- }
- /* rA = rA -> rA*/
- if (is_reg(scev, ra_expr, &rb) && ra == rb) {
- header_env->reg2scev[ra] = ra_expr;
- continue;
- }
- /* rA = (any 1 2 3 4 ...) -> (any rA 1 2 3 4 ...) */
- if (is_any_imm_reg_opaque(scev, ra_expr)) {
- id = mk_any(scev, expr1(scev, REG, ra), ra_expr, no_def);
- if (id < 0)
- return id;
- header_env->reg2scev[ra] = id;
- continue;
- }
-
- }
- /*
- * do { i++; if (old > 1021) break; c = buf[old]; old = i; } while (c);
- * rA = (+ rB IMM) and rB = (+ rB IMM): an iteration leaves the same number in both. If
- * they are equal at loop entry too, rA is an induction variable with the same step.
- */
- for (ra = 0; ra < REGS_NUM; ra++) {
- if (header_env->reg2scev[ra] != UNKNOWN_EXPR_ID ||
- header_env->reg2expr[ra] == UNKNOWN_EXPR_ID)
- continue;
- for (rb = 0; rb < REGS_NUM; rb++) {
- id = header_env->reg2scev[rb];
- if (header_env->reg2expr[rb] != header_env->reg2expr[ra] ||
- !is_linear_or_linear32(scev, id, &l, &r, &zext32) ||
- !is_reg(scev, l, &t) || t != rb ||
- !const_diff_at_header(env, h, rb, ra, &imm) || imm)
- continue;
- id = expr2(scev, LINEAR_SCEV, expr1(scev, REG, ra), r);
- if (zext32)
- id = expr1(scev, ZEXT32, id);
- if (id < 0)
- return id;
- header_env->reg2scev[ra] = id;
- break;
- }
- }
- return 0;
-}
-
-/*
- * Find a SCEV register with the entry value @x and the backedge value @l, and one with @y and @r.
- * All that match are copies of each other at the header: take the first.
- */
-static bool find_operands(struct env *entry_env, struct env *header_env, u32 x, u32 y, u32 l, u32 r,
- u32 *rb, u32 *rc)
-{
- for (*rb = 0; *rb < REGS_NUM; ++*rb)
- if (entry_env->reg2expr[*rb] == x && header_env->reg2expr[*rb] == l)
- break;
- for (*rc = 0; *rc < REGS_NUM; ++*rc)
- if (entry_env->reg2expr[*rc] == y && header_env->reg2expr[*rc] == r)
- break;
- return *rb < REGS_NUM && *rc < REGS_NUM;
-}
-
-/*
- * p = buf; i = 0; while (*p) { if (i > 512) break; use(p); i += len(p); p = buf + i; }
- * rA == rB + rC holds at the header in every iteration: it holds at loop entry and every backedge
- * preserves it. The backedge values are rA = (+ l r), rB = l, rC = r. The entry values, by the env
- * of the enclosing loop at @h, are rA = x, rB = x, rC = 0 or rA = (+ x y), rB = x, rC = y.
- * (add rB rC) keeps the def of (+ l r), see alu64_with_def().
- */
-static int compute_header_adds(struct bpf_verifier_env *env, int h)
-{
- struct scev *scev = env->scev;
- struct env *entry_env = find_loop_env(scev, env->insn_aux_data[h].loop_header, h);
- struct env *header_env = find_header_env(scev, h);
- u32 ra, rb, rc, l, r, x, y, entry_expr;
- struct expr_def def;
- int id, zero = imm_expr(scev, 0);
-
- if (zero < 0)
- return zero;
- for (ra = 0; entry_env && ra < REGS_NUM; ra++) {
- entry_expr = entry_env->reg2expr[ra];
- if (header_env->reg2scev[ra] != UNKNOWN_EXPR_ID ||
- !is_add(scev, header_env->reg2expr[ra], &l, &r) ||
- !scev->exprs[entry_expr].is_vn || !scev->exprs[l].is_vn ||
- !scev->exprs[r].is_vn)
- continue;
- if (!find_operands(entry_env, header_env, entry_expr, zero, l, r, &rb, &rc) &&
- !find_operands(entry_env, header_env, zero, entry_expr, l, r, &rb, &rc) &&
- (scev->exprs[entry_expr].def.set || !is_add(scev, entry_expr, &x, &y) ||
- (!find_operands(entry_env, header_env, x, y, l, r, &rb, &rc) &&
- !find_operands(entry_env, header_env, y, x, l, r, &rb, &rc))))
- continue;
- /* expr2() can move exprs[] */
- def = scev->exprs[header_env->reg2expr[ra]].def;
- id = expr_with_def(scev, expr2(scev, ADD_SCEV, REG_EXPR_ID(rb), REG_EXPR_ID(rc)),
- def);
- if (id < 0)
- return id;
- header_env->reg2scev[ra] = id;
- }
- return 0;
-}
-
-/*
- * 'sum += a[i]', 'if (..) { len -= n; len -= 2; }': is 'id', what an iteration makes of
- * register 'r', the register plus or minus something, or the register as it was?
- * 'low32': it's computed in 32 bits.
- */
-static bool expr_is_sum(struct scev *scev, u32 id, u32 r, bool low32, int depth)
-{
- u32 a, b;
-
- if (is_reg(scev, id, &a))
- return a == r;
- if (!depth--)
- return false;
- if (low32 && is_zext32(scev, id, &a))
- return expr_is_sum(scev, a, r, low32, depth);
- if (is_any(scev, id, &a, &b))
- return expr_is_sum(scev, a, r, low32, depth) &&
- expr_is_sum(scev, b, r, low32, depth);
- if (is_add(scev, id, &a, &b) && expr_is_sum(scev, b, r, low32, depth))
- return true;
- return (is_add(scev, id, &a, &b) || is_binop(scev, BPF_SUB, id, &a, &b)) &&
- expr_is_sum(scev, a, r, low32, depth);
-}
-
-/*
- * Find the registers and stack slots that nothing but a jump stops: what gets a constant added in
- * every iteration of the loop, and a sum that no jump of the loop depends on.
- */
-static void scev_compute_linear_regs(struct scev *scev, struct bpf_loop *loop,
- struct env *header_env)
-{
- u32 r, same, base, slope, id;
- bool low32;
- int bits;
-
- for (r = 0; r < REGS_NUM; r++) {
- id = header_env->reg2expr[r];
- bits = 0;
- if (is_linear_or_linear32(scev, header_env->reg2scev[r], &base, &slope, &low32) &&
- is_reg(scev, base, &same) && same == r)
- bits = low32 ? 32 : 64;
- /* what the loop leaves alone is no sum */
- else if (!is_reg(scev, id, &same) && !test_bit(r, loop->tested_regs))
- bits = expr_is_sum(scev, id, r, false, EXPR_STACK_DEPTH) ? 64 :
- expr_is_sum(scev, id, r, true, EXPR_STACK_DEPTH) ? 32 : 0;
- if (bits)
- __set_bit(r, bits == 32 ? loop->linear32_regs : loop->linear_regs);
- }
-}
-
-static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, struct env *einsn)
-{
- struct scev *scev = env->scev;
- int id, reg;
-
- for (reg = 0; reg < REGS_NUM; reg++) {
- id = transform_expr_once(scev, 0, einsn->reg2expr[reg], eheader->reg2scev,
- instantiate_header_scevs);
- if (id < 0)
- return id;
- einsn->reg2scev[reg] = id;
- }
-
- return 0;
-}
-
-static int log_scevs(struct bpf_verifier_env *env)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_verifier_log *log = &env->log;
- struct env *header_env, *latch_env;
- struct scev *scev = env->scev;
- struct bpf_loop *loop;
- int i, j, k, len, latch;
-
- len = env->prog->len;
- for (i = 0; i < len; i++) {
- loop = aux[i].loop;
- if (!loop)
- continue;
- header_env = find_header_env(scev, i);
- bpf_log(log, "scev at header %d:\n", i);
- print_env(env, header_env, i);
- if (!can_compute_loop_scev(loop))
- continue;
- for (j = 0; j < loop->backedges_cnt; j++) {
- for (k = 0; k < loop->backedges[j].latches_cnt; k++) {
- latch = loop->backedges[j].latches[k];
- bpf_log(log, " scev at %s %d:\n", k ? "exit" : "latch", latch);
- latch_env = find_loop_env(scev, i, latch);
- if (verifier_bug_if(!latch_env, env, "i=%d, latch=%d", i, latch))
- return -EFAULT;
- print_env(env, latch_env, latch);
- }
- }
- }
- return 0;
-}
-
-/*
- * A slice comes from a call that may write to the buffer: the kfunc, or a subprog that calls it.
- * See expr_is_vn().
- */
-static void find_slice_slots(struct bpf_verifier_env *env)
-{
- struct bpf_insn *insn = env->prog->insnsi;
- struct scev *scev = env->scev;
- bool slices = false;
- u32 i, spi;
-
- for (i = 0; i < env->prog->len; i++)
- slices |= bpf_is_dynptr_slice_rdwr_kfunc(&insn[i]);
- for (i = 0; slices && i < env->prog->len; i++)
- if (BPF_CLASS(insn[i].code) == BPF_JMP && BPF_OP(insn[i].code) == BPF_CALL)
- bitmap_or(scev->slice_slots, scev->slice_slots, bpf_may_write_mask(env, i),
- MAX_BPF_STACK_SLOTS);
- for_each_set_bit(spi, scev->slice_slots, MAX_BPF_STACK_SLOTS)
- scev->exprs[REG_EXPR_ID(MAX_BPF_REG + spi)].is_vn = false;
-}
-
-int bpf_compute_scev(struct bpf_verifier_env *env)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct scev *scev = env->scev;
- int *postorder = env->cfg.insn_postorder;
- int cnt = env->cfg.cur_postorder;
- int i, idx, err, header;
-
- find_slice_slots(env);
- mark_latches(env);
- /*
- * Visit loop headers in postorder, to guarantee that scevs
- * for innermost loops are computed first.
- */
- for (i = 0; i < cnt; i++) {
- idx = postorder[i];
- if (!aux[idx].loop)
- continue;
- err = compute_scev_for_loop(env, idx, idx);
- if (err)
- return err;
- }
- for (i = 0; i < env->subprog_cnt; i++) {
- err = compute_scev_for_loop(env, -1, env->subprog_info[i].start);
- if (err)
- return err;
- }
-
- /*
- * Compute scevs from exprs collected on a previous step. Iterate instructions in
- * reverse post-order so that each loop header is processed before instructions
- * reachable from it.
- */
- for (i = cnt - 1; i >= 0; i--) {
- idx = postorder[i];
- if (!aux[idx].need_scev)
- continue;
- if (bpf_must_abort())
- return -EINTR;
- header = bpf_loop_at_index(env, idx);
- err = aux[idx].loop
- ? compute_header_scevs(env, header)
- : compute_insn_scevs(env,
- find_header_env(scev, header),
- find_loop_env(scev, header, idx));
- if (err)
- return err;
- if (aux[idx].loop) {
- scev_compute_linear_regs(scev, aux[idx].loop,
- find_header_env(scev, header));
- err = compute_header_adds(env, idx);
- if (err)
- return err;
- }
- err = find_linked_regs(env, aux[idx].loop ? &scev->identity_env :
- find_loop_env(scev, header, idx), header, idx);
- if (err)
- return err;
- }
-
- /* Now it's known which jumps test a counter: the places of a back edge are for them */
- err = bpf_find_latches(env, false);
- if (err)
- return err;
-
- if (env->log.level & BPF_LOG_LEVEL2) {
- err = log_scevs(env);
- if (err)
- return err;
- }
-
- return 0;
-}
-
-/*
- * Are all slots of the buffer [off, off + size) in slice_slots? Without @fixed_off the buffer is
- * somewhere in the range, and the range can be wider than slice_slots: two offsets become a range
- * at a join point.
- */
-bool bpf_scev_in_slice_slots(struct bpf_verifier_env *env, int off, u32 size, bool fixed_off)
-{
- int first = bpf_get_spi(off + size - 1), last = bpf_get_spi(off);
- const unsigned long *slots = env->scev->slice_slots;
-
- if (fixed_off)
- return find_next_zero_bit(slots, last + 1, first) > last;
- return find_next_bit(slots, last + 1, first) <= last;
-}
-
-void bpf_free_scev(struct bpf_verifier_env *env)
-{
- struct scev *scev = env->scev;
- struct insn_envs *envs;
- int i;
- u32 j;
-
- if (!scev)
- return;
- for (i = 0; i < scev->envs_cnt; i++) {
- envs = scev->envs[i];
- if (!envs)
- continue;
- for (j = 0; j < envs->cnt; j++)
- kfree(envs->entries[j].env);
- kfree(envs);
- }
- kvfree(scev->exprs_ht);
- kvfree(scev->exprs_next);
- kvfree(scev->worklist.data);
- kvfree(scev->envs);
- kvfree(scev->exprs);
- kvfree(scev->linked);
- kvfree(scev->linked_exprs);
- kvfree(scev->vals);
- kvfree(scev->discovered);
- kfree(scev);
- env->scev = NULL;
-}
-
-int bpf_init_scev(struct bpf_verifier_env *env)
-{
- struct scev *scev;
- u32 r;
-
- scev = kzalloc_obj(struct scev, BPF_VERIFIER_GFP);
- if (!scev)
- return -ENOMEM;
- env->scev = scev;
- bpf_mark_reg_known_scalar(&scev->zero_reg, 0);
- bpf_mark_reg_unknown(&scev->unknown_reg);
- setup_initial_loop_env(&scev->identity_env);
- if (grow_exprs(scev, roundup_pow_of_two(REG_EXPR_ID(REGS_NUM))))
- goto nomem;
- scev->exprs[UNKNOWN_EXPR_ID] = (struct expr){ .op = UNKNOWN };
- scev->exprs[OPAQUE_EXPR_ID] = (struct expr){ .op = OPAQUE };
- for (r = 0; r < REGS_NUM; r++)
- scev->exprs[REG_EXPR_ID(r)] = (struct expr){ .op = REG, .is_vn = true,
- .params = { r, 0 } };
- for (r = 0; r < REG_EXPR_ID(REGS_NUM); r++)
- set_expr_base(scev, &scev->exprs[r], r);
- scev->exprs_cnt = REG_EXPR_ID(REGS_NUM);
- scev->envs = kvzalloc_objs(*scev->envs, env->prog->len,
- BPF_VERIFIER_GFP);
- scev->discovered = kvzalloc_objs(*scev->discovered, env->prog->len,
- BPF_VERIFIER_GFP);
- if (!scev->envs || !scev->discovered)
- goto nomem;
- /*
- * Remember original program length, in case bpf_free_scev()
- * is called after bpf program rewrites that increase program
- * length.
- */
- scev->envs_cnt = env->prog->len;
- return 0;
-nomem:
- bpf_free_scev(env);
- return -ENOMEM;
-}
-
-static struct bpf_stack_state *scev_regno_to_stack(const struct bpf_func_state *st, u32 r)
-{
- int spi, slots_available;
-
- if (r < MAX_BPF_REG)
- return NULL;
-
- slots_available = st->allocated_stack / BPF_REG_SIZE;
- spi = r - MAX_BPF_REG;
- if (spi < slots_available)
- return &st->stack[spi];
-
- return NULL;
-}
-
-static struct bpf_reg_state *scev_regno_to_reg(struct bpf_func_state *st, u32 r)
-{
- return r < MAX_BPF_REG ? &st->regs[r] : NULL;
-}
-
-/* Linear widening requires a physical register or a full spill. */
-static struct bpf_reg_state *scev_reg_for_linear(struct bpf_func_state *st, u32 r)
-{
- struct bpf_stack_state *stack;
-
- if (r < MAX_BPF_REG)
- return scev_regno_to_reg(st, r);
- stack = scev_regno_to_stack(st, r);
- if (!stack || bpf_spill_size(stack) != BPF_REG_SIZE)
- return NULL;
- return &stack->spilled_ptr;
-}
-
-/* '*(u32 *)(r10 - 8) = rX' left a number in the low half of the slot. */
-static struct bpf_reg_state *scev_reg_for_spill32(struct bpf_func_state *st, u32 r)
-{
- struct bpf_stack_state *stack = scev_regno_to_stack(st, r);
-
-#ifdef __BIG_ENDIAN
- /* a 32-bit spill is in the upper 32 bits of the slot */
- return NULL;
-#endif
- if (!stack || bpf_spill_size(stack) != 4 || stack->spilled_ptr.type != SCALAR_VALUE)
- return NULL;
- return &stack->spilled_ptr;
-}
-
-/*
- * Return a borrowed value; never use it as a widening destination.
- * NULL denotes no readable value.
- */
-static const struct bpf_reg_state *
-scev_reg_for_read(struct bpf_verifier_env *env, const struct bpf_func_state *st, u32 r)
-{
- const struct bpf_stack_state *stack;
- bool zero = true;
- int i;
-
- if (r < MAX_BPF_REG)
- return st->regs[r].type == NOT_INIT ? NULL : &st->regs[r];
- if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE)
- return NULL;
-
- stack = scev_regno_to_stack(st, r);
- if (!stack)
- return &env->scev->unknown_reg;
- if (bpf_spill_size(stack) == BPF_REG_SIZE)
- return &stack->spilled_ptr;
-
- /* Partial spills do not describe the value of the whole slot. */
- for (i = 0; i < BPF_REG_SIZE; i++) {
- switch (stack->slot_type[i]) {
- case STACK_ZERO:
- break;
- case STACK_INVALID:
- case STACK_POISON:
- case STACK_MISC:
- zero = false;
- break;
- case STACK_SPILL:
- if (!bpf_is_spilled_scalar_reg(stack))
- return NULL;
- zero = false;
- break;
- default:
- return NULL;
- }
- }
- return zero ? &env->scev->zero_reg : &env->scev->unknown_reg;
-}
-
-/*
- * Materialize a writable value in the current state, growing its stack if needed.
- * This does not go through check_stack_write(). The buffer of a live dynptr slice, see
- * bpf_reg_state::buf, does not become a spill here. The callers skip the slots that are
- * is_unbound_scev_reg() when the loop is entered. A slot that a call in the loop may write
- * has no description: it is in forget_regs or fresh_regs, and they skip these too.
- */
-static int scev_reg_for_write(struct bpf_verifier_env *env, struct bpf_func_state *st,
- u32 r, struct bpf_reg_state **out)
-{
- const struct bpf_reg_state *value;
- struct bpf_stack_state *stack;
- u32 spi;
- int err;
-
- *out = scev_regno_to_reg(st, r);
- if (*out)
- return 0;
- if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE)
- return -EINVAL;
-
- spi = r - MAX_BPF_REG;
- err = bpf_grow_stack_state(env, st, (spi + 1) * BPF_REG_SIZE);
- if (err)
- return err;
-
- /* Resolve pointers after growth, which may relocate the stack. */
- value = scev_reg_for_read(env, st, r);
- if (!value)
- return -EINVAL;
- stack = scev_regno_to_stack(st, r);
- stack->spilled_ptr = *value;
- memset(stack->slot_type, STACK_SPILL, sizeof(stack->slot_type));
- *out = &stack->spilled_ptr;
- return 0;
-}
-
-static bool scev_reg_alive(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 r)
-{
- int insn_idx = bpf_frame_insn_idx(st, st->curframe);
- u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
- int spi;
-
- if (r < MAX_BPF_REG) {
- return BIT(r) & live_regs;
- } else {
- spi = r - MAX_BPF_REG;
- return bpf_stack_slot_alive(env, st->curframe, spi * 2) ||
- bpf_stack_slot_alive(env, st->curframe, spi * 2 + 1);
- }
-}
-
-/*
- * Latch is a condition deciding if execution remains inside a loop.
- * Linear latch represents a condition 'if <reg> <op> <loop invariant> goto <loop-header>',
- * where equation '<base> + i * <step> <op> <bound>' describes values taken by register <reg>,
- * 'i' is the loop iteration number, starting from 0.
- * 'zext32' indicates the latch is for (zext32 (linear ...)) SCEV.
- * 'jmp32' indicates that the latch is a JMP32 instruction.
- * 'base_reg' is the register in <base>.
- */
-struct linear_latch {
- u32 insn_idx;
- u32 base_expr;
- u32 step_expr;
- u32 bound_expr;
- u32 base_reg;
- u8 op;
- bool zext32;
- bool jmp32;
-};
-
-static bool loop_invariant(struct scev *scev, u32 id)
-{
- s64 imm;
- u32 reg;
-
- return is_reg(scev, id, ®) || is_imm(scev, id, &imm);
-}
-
-static int match_linear_latch(struct bpf_verifier_env *env,
- u32 header,
- u32 latch_idx, bool taken_leaves,
- struct linear_latch *latch)
-{
- struct bpf_insn *insn = &env->prog->insnsi[latch_idx];
- struct scev *scev = env->scev;
- struct env *latch_env;
- u32 src_reg_scev;
- u32 dst_reg_scev;
- u32 l, r, op;
- int id;
-
- op = BPF_OP(insn->code);
- latch->jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
- /* Flip the condition if true branch jumps out of the loop. */
- if (taken_leaves)
- op = bpf_rev_opcode(op);
- switch (op) {
- case BPF_JSLT:
- case BPF_JSLE:
- case BPF_JSGT:
- case BPF_JSGE:
- case BPF_JLT:
- case BPF_JLE:
- case BPF_JGT:
- case BPF_JGE:
- case BPF_JNE:
- break;
- default:
- return false;
- }
-
- latch->insn_idx = latch_idx;
-
- latch_env = find_loop_env(scev, header, latch_idx);
- if (verifier_bug_if(!latch_env, env, "header_idx=%d, latch_idx=%d\n", header, latch_idx))
- return -EFAULT;
- dst_reg_scev = latch_env->reg2scev[insn->dst_reg];
- src_reg_scev = latch_env->reg2scev[insn->src_reg];
- /*
- * Make sure (linear ...) is in dst_reg_scev:
- * 'if <invariant> <op> <linear>' ⇔
- * 'if <linear> <flip(op)> <invariant>'.
- */
- if (BPF_SRC(insn->code) == BPF_X &&
- is_linear_or_linear32(scev, src_reg_scev, &l, &r, &latch->zext32)) {
- swap(dst_reg_scev, src_reg_scev);
- op = bpf_flip_opcode(op);
- }
- latch->op = op;
- /*
- * simplify() produces two shapes for linear latches:
- * - (linear register step)
- * ^----------------------------+-- base_expr
- * base_expr |
- * - (linear (+ register imm) step) |
- * ^----------------------------'
- * 'register' is (zext32 *fp-8) for a 32-bit counter on the stack.
- */
- if (!is_linear_or_linear32(scev, dst_reg_scev,
- &latch->base_expr, &latch->step_expr, &latch->zext32))
- return false;
-
- if (BPF_SRC(insn->code) == BPF_K) {
- id = imm_expr(scev, insn->imm);
- if (id < 0)
- return id;
- latch->bound_expr = id;
- } else {
- latch->bound_expr = src_reg_scev;
- }
-
- if (!loop_invariant(scev, latch->step_expr) ||
- !loop_invariant(scev, latch->bound_expr))
- return false;
-
- /* (linear register step) */
- if (is_reg_or_low32(scev, latch->base_expr, &latch->base_reg))
- return true;
-
- /* (linear (+ register imm) step) */
- if (is_add(scev, latch->base_expr, &l, &r) &&
- is_reg_or_low32(scev, l, &latch->base_reg) &&
- loop_invariant(scev, r))
- return true;
-
- return false;
-}
-
-/*
- * Can the jump at @n tell the number of iterations of the loop at @header? The status of a call
- * and a NULL test cannot. 1 as long as scalar evolution has not looked at the program.
- */
-int bpf_scev_jump_counts(struct bpf_verifier_env *env, int header, int n, bool taken_leaves)
-{
- struct linear_latch latch;
-
- if (!env->scev)
- return 1;
- if (!find_loop_env(env->scev, header, n))
- return 0;
- return match_linear_latch(env, header, n, taken_leaves, &latch);
-}
-
-static void log_scev_value(struct bpf_verifier_env *env, struct bpf_func_state *st,
- const struct scev_value *v)
-{
- const struct bpf_reg_state *reg;
-
- if (v->ptr_reg >= 0) {
- reg = scev_reg_for_read(env, st, v->ptr_reg);
- log_reg(env, v->ptr_reg);
- bpf_log(&env->log, " (%s)", reg ? reg_type_str(env, reg->type) : "unavailable");
- } else if (cnum64_is_const(v->r64)) {
- bpf_log(&env->log, "scalar %llu", v->r64.base);
- } else {
- bpf_log(&env->log, "scalar %llu..%llu", cnum64_umin(v->r64), cnum64_umax(v->r64));
- }
-}
-
-static void log_incompatible_scev_values(struct bpf_verifier_env *env, const char *reason,
- struct bpf_func_state *st,
- const struct scev_value *a, const struct scev_value *b)
-{
- bpf_log(&env->log, "scev: %s ", reason);
- log_scev_value(env, st, a);
- bpf_log(&env->log, ", ");
- log_scev_value(env, st, b);
- bpf_log(&env->log, "\n");
-}
-
-static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st,
- const struct scev_value *a, const struct scev_value *b,
- const struct linear_latch *latch)
-{
- const struct bpf_reg_state *ra, *rb;
-
- if (a->ptr_reg < 0 || b->ptr_reg < 0) {
- if (a->ptr_reg < 0 && b->ptr_reg < 0)
- return true;
- goto incompatible;
- }
- ra = scev_reg_for_read(env, st, a->ptr_reg);
- rb = scev_reg_for_read(env, st, b->ptr_reg);
- if (ra && rb && bpf_same_memory_origin(ra, rb) && !latch->zext32 && !latch->jmp32)
- return true;
-
-incompatible:
- if (env->log.level & BPF_LOG_LEVEL2)
- log_incompatible_scev_values(env, "incompatible latch operands", st, a, b);
- return false;
-}
-
-/* @v = the SCEV register @r in the state @st */
-static void eval_reg(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 r,
- struct scev_value *v)
-{
- const struct bpf_reg_state *reg;
-
- v->ptr_reg = -1;
- v->bits = 64;
- /* a 32-bit spill, or the upper half of the slot is dead: the low 32 bits of the value */
- reg = scev_reg_for_spill32(st, r);
- if (reg)
- v->bits = 32;
- else
- reg = scev_reg_for_read(env, st, r);
- if (!reg || reg->type == NOT_INIT) {
- v->bits = 0;
- return;
- }
- *scalar_of(v) = *scalar_of(reg);
- if (reg->type != SCALAR_VALUE)
- v->ptr_reg = r;
-}
-
-static bool scalar_is_unknown(const struct scalar *s)
-{
- return s->step <= 1 && tnum_is_unknown(s->var_off) && s->r64.size == U64_MAX &&
- s->r32.size == U32_MAX;
-}
-
-static void scev_value_set_const(struct scev_value *v, s64 imm)
-{
- scalar_set_const(scalar_of(v), imm);
- v->ptr_reg = -1;
- v->bits = 64;
-}
-
-/*
- * @v = @v @op @r, by the transfer functions of the main pass. @r is not read if @op has one
- * operand. An op that is not evaluated: nothing is known.
- * A value is known modulo 2^bits: the run-time value and a member of the abstract value have the
- * same low bits. + - * & | ^ and << by an exact amount respect that. An extension of N bits wants N
- * bits and makes 64. The other ops want 64.
- * Expressions have no types, the state has them. A pointer can be an operand of an add, with a
- * scalar, in 64 bits.
- */
-static void scev_value_alu(u32 op, struct scev_value *v, const struct scev_value *r)
-{
- struct bpf_insn insn = { .code = BPF_ALU64 | BPF_X | op };
- u32 low = op_low_bits(op);
-
- if (low) {
- if (v->ptr_reg >= 0)
- goto unknown;
- /* a partial spill: the other bits of the slot are not known */
- if (op == SPILL8 || op == SPILL16 || op == SPILL32) {
- v->bits = min(v->bits, low);
- return;
- }
- if (v->bits < low)
- goto unknown;
- scalar_ext(scalar_of(v), low, !is_zext(op));
- v->bits = 64;
- return;
- }
- if (op_params_num(op) != 2)
- goto unknown;
- v->bits = min(v->bits, r->bits);
- switch (op) {
- case BPF_ADD:
- if (v->ptr_reg < 0 && r->ptr_reg < 0)
- break;
- if ((v->ptr_reg >= 0 && r->ptr_reg >= 0) || v->bits < 64)
- goto unknown;
- v->ptr_reg = max(v->ptr_reg, r->ptr_reg);
- scalar_add(scalar_of(v), scalar_of(v), scalar_of(r));
- return;
- case BPF_SUB:
- case BPF_MUL:
- case BPF_AND:
- case BPF_OR:
- case BPF_XOR:
- break;
- case BPF_LSH:
- if (r->bits < 64)
- goto unknown;
- break;
- case SDIV:
- case SMOD:
- insn.code = BPF_ALU64 | BPF_X | (op == SDIV ? BPF_DIV : BPF_MOD);
- insn.off = 1;
- fallthrough;
- case BPF_DIV:
- case BPF_MOD:
- case BPF_RSH:
- case BPF_ARSH:
- if (v->bits < 64)
- goto unknown;
- break;
- default:
- goto unknown;
- }
- if (v->ptr_reg >= 0 || r->ptr_reg >= 0)
- goto unknown;
- if (v->bits)
- scalar_alu(scalar_of(v), scalar_of(r), &insn);
- return;
-unknown:
- v->bits = 0;
-}
-
-/* @v = @v + @delta (mod 2^@bits) */
-static void scev_value_add_const(struct scev_value *v, s64 delta, u32 bits)
-{
- struct scev_value c;
-
- if (delta) {
- scev_value_set_const(&c, delta);
- scev_value_alu(BPF_ADD, v, &c);
- }
- v->bits = min(v->bits, bits);
-}
-
-/*
- * @v and @w are two values of one expression: what a leader holds, what the operands give. Both
- * hold: @v becomes their intersection. An empty one: the path is infeasible, @v is left as it is.
- * Two pointers are not intersected, see recompute_ptr(): the one that has news is taken.
- */
-static void scev_value_meet(struct scev_value *v, struct scev_value *w)
-{
- if (!v->bits || !w->bits) {
- if (w->bits)
- *v = *w;
- return;
- }
- if (w->bits > v->bits)
- swap(*v, *w);
- if (v->ptr_reg >= 0 || w->ptr_reg >= 0) {
- if (v->ptr_reg >= 0 && w->ptr_reg >= 0 && w->bits == v->bits && w->refined)
- *v = *w;
- return;
- }
- scalar_intersect(scalar_of(v), scalar_of(w), w->bits);
- v->refined |= w->refined;
-}
-
-/* The value of the operand @id. Its base is the linked expression at position @n. */
-static void eval_operand(struct bpf_verifier_env *env, u32 id, u32 n, struct scev_value *v)
-{
- u32 base, bits;
- s64 off;
-
- *v = env->scev->vals[n];
- bits = split_const_offset(env, id, &base, &off);
- scev_value_add_const(v, off, bits);
-}
-
-/*
- * LLVM: ScalarEvolution::getRangeRef(). Compute scev->vals[], the values of the linked expressions
- * in the state @st. Operands come first: one walk over the array, every expression is evaluated
- * once, from what its leaders hold in @st and from the values of its operands.
- */
-static void eval_linked_exprs(struct bpf_verifier_env *env, struct bpf_func_state *st,
- const struct jump_links *t)
-{
- struct scev_value w, r, *v, *vals = env->scev->vals;
- const struct scev_linked_expr *x;
- const struct scev_linked_reg *q;
- struct scev *scev = env->scev;
- struct expr *e;
- u32 n, i, bits;
- s64 delta;
-
- for (n = 0; n < scev->linked_exprs_cnt; n++) {
- x = &scev->linked_exprs[n];
- v = &vals[n];
- v->ptr_reg = -1;
- v->bits = 0;
- v->refined = false;
- if (is_imm(scev, x->id, &delta))
- scev_value_set_const(v, delta);
- for (i = 0; i < LEADERS_NUM; i++) {
- /* a tested register holds all bits: the others learn from it */
- if (x->leaders[i] == NO_IDX || (i == LEADERS_NUM - 1 && v->bits == 64))
- continue;
- q = &t->regs[x->leaders[i]];
- if (!const_diff(env, t->h, q->id, x->id, &delta, &bits))
- continue;
- eval_reg(env, st, q->reg, &w);
- w.refined = is_refined_reg(t, q->reg);
- scev_value_add_const(&w, delta, bits);
- scev_value_meet(v, &w);
- }
- /* operands without news: the main pass evaluated the leader from them */
- if (x->params[0] == NO_IDX || (v->bits == 64 && !vals[x->params[0]].refined &&
- (x->params[1] == NO_IDX || !vals[x->params[1]].refined)))
- continue;
- /* not NULL: it has operands */
- e = expr_to_eval(env, t->h, x->id);
- eval_operand(env, e->params[0], x->params[0], &w);
- if (x->params[1] != NO_IDX) {
- eval_operand(env, e->params[1], x->params[1], &r);
- w.refined |= r.refined;
- }
- scev_value_alu(e->op == ADD_SCEV ? BPF_ADD : e->op, &w, &r);
- /* The insn may become ALU32, see alu64_with_def(). The main pass knows. */
- if (e->def.set && !env->insn_aux_data[e->def.idx].prevent_zext)
- w.bits = e->op == BPF_ADD || e->op == ADD_SCEV ? min(w.bits, 32) : 0;
- /* a product that can wrap: no news for those who are made of it */
- if (w.bits && !scalar_is_unknown(scalar_of(&w)))
- scev_value_meet(v, &w);
- }
-}
-
-/*
- * The jump at @insn_idx refined the registers @refined_regs in the state @st. Find the next SCEV
- * register that is linked to one of them, and evaluate it: on every path to the jump
- * *@linked_regno == *@val (mod 2^val->bits). It is of the same frame, and live at @next_idx, where
- * @st goes on. It is a register, or MAX_BPF_REG + the index of a stack slot: a fill from byte
- * *@fill_off of the slot reads these bits. *@pos is the position of the search, 0 at first. False:
- * no more.
- * Expressions have no types: a pointer is val->ptr_reg as it is in @st, with the offset *@val.
- * BPF_SCEV_REGS_NUM and above is a memory location, see struct env and bpf_scev_mem_addr().
- */
-bool bpf_scev_next_linked_reg(struct bpf_verifier_env *env, struct bpf_func_state *st, int insn_idx,
- int next_idx, u16 refined_regs, u32 *pos, u32 *linked_regno,
- u32 *fill_off, struct scev_value *val)
-{
- struct insn_envs *envs = env->scev->envs[insn_idx];
- DECLARE_BITMAP(roots, ENV_REGS_NUM) = {};
- const struct scev_linked_reg *q;
- struct scev *scev = env->scev;
- struct jump_links t;
- s64 delta;
- u32 k, bits;
-
- if (!envs || !envs->linked_cnt)
- return false;
- t = (struct jump_links){ .regs = &scev->linked[envs->linked_first],
- .regs_cnt = envs->linked_cnt,
- .h = bpf_loop_at_index(env, insn_idx),
- .refined_regs = refined_regs };
- if (!*pos) {
- for (k = 0; k < t.regs_cnt; k++)
- if (t.regs[k].refine && reg_alive_at(env, t.regs[k].reg, next_idx))
- __set_bit(k, roots);
- if (add_linked_exprs(env, &t, roots))
- return false;
- eval_linked_exprs(env, st, &t);
- }
- for (; *pos < t.regs_cnt; ++*pos) {
- q = &t.regs[*pos];
- if (q->expr == NO_IDX || !scev->vals[q->expr].bits ||
- !scev->vals[q->expr].refined ||
- !const_diff_reg(env, t.h, scev->linked_exprs[q->expr].id, q->reg, q->id,
- fill_off, &delta, &bits))
- continue;
- *val = scev->vals[q->expr];
- scev_value_add_const(val, delta, bits);
- if (!val->bits)
- continue;
- /* the low bytes are the last ones */
- if (is_big_endian_slot(q->reg))
- *fill_off += (bits - val->bits) / BITS_PER_BYTE;
- *linked_regno = q->reg;
- return true;
- }
- return false;
-}
-
-/*
- * The value of @id in the state @st at a loop header: every SCEV register is its header phi there,
- * (REG r) is r. All 64 bits are wanted.
- */
-static bool eval_expr(struct bpf_verifier_env *env, struct scev *scev,
- struct bpf_func_state *st, u32 id, u32 lvl, struct scev_value *result)
-{
- struct expr *e = &scev->exprs[id];
- struct scev_value rval;
- u32 regno;
- s64 imm;
-
- result->ptr_reg = -1;
- result->bits = 0;
- if (is_reg(scev, id, ®no)) {
- eval_reg(env, st, regno, result);
- } else if (is_imm(scev, id, &imm)) {
- scev_value_set_const(result, imm);
- } else if (op_params_num(e->op) && !e->def.set && lvl < EXPR_STACK_DEPTH) {
- eval_expr(env, scev, st, e->params[0], lvl + 1, result);
- if (op_params_num(e->op) == 2)
- eval_expr(env, scev, st, e->params[1], lvl + 1, &rval);
- scev_value_alu(e->op, result, &rval);
- }
- return result->bits == 64;
-}
-
-/* The range of @v. With @low32 that of its low 32 bits, in the upper half of a u64. */
-static struct cnum64 scalar_arc(const struct scalar *v, bool low32)
-{
- return low32 ? cnum64_lshift32(v->r32) : v->r64;
-}
-
-/* Euclid. gcd64(a, 0) is a. gcd() is for unsigned long. */
-static u64 gcd64(u64 a, u64 b)
-{
- u64 rem;
-
- while (b) {
- div64_u64_rem(a, b, &rem);
- a = b;
- b = rem;
- }
- return a;
-}
-
-/*
- * The values of @v are among the members of an arithmetic progression on the circle:
- *
- * arc->base, arc->base + stride, arc->base + 2 * stride, .. , arc->base + arc->size
- *
- * @arc is scalar_arc() without the values at its ends that @v cannot have.
- * Return the stride, 0 if there is one value.
- *
- * base and step are a congruence of the s64: the distance between two values is a multiple of
- * step if r64 does not cross S64_MAX / S64_MIN. The low 32 bits have the same distances if r32
- * is r64 moved by a multiple of 1 << 32.
- * The known low bits of var_off are a congruence modulo a power of two. That divides 1 << 64 and
- * 1 << 32, so it holds wherever the arc is. reg_bounds_sync() does not round a range to them:
- * 1..0xff8 with three zero bits is 8..0xff8 here.
- */
-static u64 scalar_stride(const struct scalar *v, bool low32, struct cnum64 *arc)
-{
- struct tnum t = low32 ? tnum_lshift(v->var_off, 32) : v->var_off;
- s64 smin = cnum64_smin(v->r64), smax = cnum64_smax(v->r64);
- u64 low, d;
-
- *arc = scalar_arc(v, low32);
- if (cnum64_is_empty(*arc) || cnum64_is_const(*arc))
- return 0;
- if (v->step > 1 && smin == (s64)v->r64.base &&
- imod(smin, v->step) == v->base && imod(smax, v->step) == v->base &&
- (!low32 || (v->r32.base == (u32)v->r64.base && v->r32.size == v->r64.size)))
- return (u64)v->step << (low32 ? 32 : 0);
-
- low = t.mask ? BIT_ULL(__ffs64(t.mask)) - 1 : U64_MAX;
- d = (t.value - arc->base) & low;
- /* range and var_off don't overlap: the path is infeasible */
- if (d > arc->size)
- return low32 ? BIT_ULL(32) : 1;
- arc->base += d;
- arc->size -= d;
- arc->size -= arc->size & low;
- return arc->size ? low + 1 : 0;
-}
-
-static bool is_signed_op(u8 op)
-{
- switch (op) {
- case BPF_JSLT:
- case BPF_JSLE:
- case BPF_JSGT:
- case BPF_JSGE:
- return true;
- default:
- return false;
- }
-}
-
-/*
- * What a linear latch tells about the loop. LLVM's ScalarEvolution::ExitLimit has the counts,
- * @false_until and @iv are ours.
- */
-struct exit_limit {
- /* The backedge is taken at most @max times. Exact if initial value and bound are known. */
- u64 max;
- /* The latch is false in iterations @max .. @false_until, whatever the initial value is. */
- u64 false_until;
- /* The induction variable at the latch in all iterations, the one that ends the loop too. */
- struct cnum64 iv;
-};
-
-/*
- * The range of @a as unsigned numbers after the circle is rotated such that @origin is 0.
- * An arc that crosses @origin gives [0, @ut_max].
- */
-static void rotated_urange(struct cnum64 a, u64 origin, u64 ut_max, u64 *min, u64 *max)
-{
- a = cnum64_sub(a, cnum64_const(origin));
- *min = cnum64_umin(a);
- *max = cnum64_umax(a) & ut_max;
-}
-
-/*
- * The backedge-taken count of
- *
- * for (x = initial; x <op> bound; x += step)
- *
- * The numbers are the points of a circle, see cnum.h. The latch is true on one arc,
- * [origin, end), and false on the rest of the circle, [end, origin).
- * min_value is 0, with @sign it is S64_MIN:
- *
- * x < b [min_value, b) 3 + 2*i < 7: i ∈ [0,1]
- * x <= b [min_value, b + 1) 3 + 2*i <= 7: i ∈ [0,1,2]
- * x > b [b + 1, min_value)
- * x >= b [b, min_value)
- * x != b [b + 1, b) 3 + 2*i != 7: i ∈ [0,1]
- *
- * 'x <= b' is always true if b + 1 is min_value, and so is 'x >= b' if b is.
- *
- * x moves clockwise. If @step is negative, both sides are inverted, as LLVM's howManyLessThans()
- * does: '7 - 2*i > 3' is '~7 + 2*i < ~3': i ∈ [0,1]. Then the circle is rotated such that origin
- * is 0. One comparison of unsigned numbers is left, x < end:
- *
- * origin x x+step end x+step*count ut_max
- * |---------------|-------|-------|-- .. --|----|-----------|------------------|
- * |<----------- diff ---------->|<overshoot>|
- *
- * The count is diff / step, rounded up, as in LLVM's computeMaxBECountForLT(). The loop ends only
- * if x does not step over [end, origin): end + overshoot <= ut_max.
- * E.g. 3 + 2*i < 8: diff is 5, the count is 3, overshoot is 1: x is 9.
- * For '!=' end is ut_max: step has to divide diff.
- *
- * With ranges the count is that of the smallest initial value and the largest end. The value that
- * ends the loop does not get smaller when end grows, so the largest end has the largest
- * end + overshoot too. Initial values are different:
- *
- * for (u64 i = initial; i > 0; i -= 2) 10: 5 times | 11: i wraps | 12: 6 times
- *
- * Both bounds of 10..12 have a count and 11 has none. overshoot is -diff modulo step: when the
- * initial value grows by 1, so does overshoot, modulo step. For the initial values
- * imin .. imin + size it is at most 'overshoot of imin' + size while that is below step: exact
- * for a constant. Else it is known modulo g = gcd(step, @stride), where the initial values are
- * imin, imin + @stride, imin + 2 * @stride ..: at most step - g + 'overshoot of imin' % g.
- * 10, 12, 14 has a count: the step divides the distance. With a stride of 1 it is step - 1, which
- * is what LLVM's canIVOverflowOnLT() assumes.
- * '!=' has to hit each bound: what holds for initial values holds for its bounds too.
- *
- * 32-bit numbers are compared on the same circle, in the upper half of a u64: cnum64_lshift32().
- * @unit is the 1 of the text above, 1 or 1 << 32, and ut_max is the largest number, -unit.
- */
-static bool compute_backedge_taken_count(u8 op, bool sign, u64 unit, struct cnum64 initial,
- u64 step, struct cnum64 bound, u64 stride,
- struct exit_limit *el)
-{
- u64 min_value = sign ? (u64)S64_MIN : 0, ut_max = -unit, origin = min_value;
- u64 imin, imax, emin, emax, bmin, bmax, diff, rem, overshoot, size, g;
- bool down = (s64)step < 0;
- struct cnum64 end;
-
- *el = (struct exit_limit){ .iv = initial };
- if (down) {
- /* ~x is ut_max - x. It reverses the order of signed and of unsigned numbers. */
- initial = cnum64_sub(cnum64_const(ut_max), initial);
- bound = cnum64_sub(cnum64_const(ut_max), bound);
- step = -step;
- op = bpf_flip_opcode(op);
- }
- rotated_urange(bound, min_value, ut_max, &bmin, &bmax);
- end = bound;
-
- switch (op) {
- case BPF_JLT:
- case BPF_JSLT:
- break;
- case BPF_JLE:
- case BPF_JSLE:
- if (bmax == ut_max)
- return false;
- end = cnum64_add(bound, cnum64_const(unit));
- break;
- case BPF_JGE:
- case BPF_JSGE:
- if (!bmin)
- return false;
- bmin -= unit;
- fallthrough;
- case BPF_JGT:
- case BPF_JSGT:
- /*
- * x moves away from the bound and wraps: 'i >= 0 && i < 128' as one unsigned
- * comparison of a variable that goes down. The smallest bound has the longest arc.
- */
- origin += bmin + unit;
- end = cnum64_const(min_value);
- break;
- case BPF_JNE:
- /* after each bound and, see below, before each initial value */
- origin = bound.base + bound.size + unit;
- break;
- default:
- return false;
- }
-
- rotated_urange(initial, origin, ut_max, &imin, &imax);
- rotated_urange(end, origin, ut_max, &emin, &emax);
-
- /* An initial value after its bound begins the arc: it goes all the way around. */
- if (op == BPF_JNE && imax > emin)
- return false;
- /*
- * The backedge is not taken. That is a count, and a better one than that of
- * another exit, which may be true for a long time.
- */
- if (imin >= emax)
- return true;
- if (!step)
- return false;
-
- diff = emax - imin;
- div64_u64_rem(diff, step, &rem);
- overshoot = rem ? step - rem : 0;
- size = imax - imin;
- if (op == BPF_JNE && check_add_overflow(size, emax - emin, &size))
- size = ut_max;
- if (size < step - overshoot) {
- overshoot += size;
- } else {
- /* Initial values that cross origin are 0 .. ut_max here: imin is none of them. */
- g = imin == initial.base - origin ? gcd64(step, stride) : unit;
- div64_u64_rem(overshoot, g, &rem);
- overshoot = step - g + rem;
- }
- if (overshoot > ut_max - emax)
- return false;
-
- el->max = div64_u64(diff - 1, step) + 1;
- /*
- * In iteration @max each x is on [end, origin). It stays there until it gets to origin.
- * '!=' is false for one number: the end of the arc of one bound is on the arc of another.
- */
- if (op != BPF_JNE || cnum64_is_const(bound))
- el->false_until = div64_u64(ut_max - imax, step);
- el->iv = cnum64_add(cnum64_from_urange(imin, emax + overshoot), cnum64_const(origin));
- el->iv = cnum64_union(initial, el->iv);
- if (down)
- el->iv = cnum64_sub(cnum64_const(ut_max), el->iv);
- return true;
-}
-
-/*
- * A 64-bit jump tests x, which is zero extended. For a bound in 0 .. U32_MAX it is an unsigned
- * 32-bit jump. For any other bound it is true for each x or false for each x.
- * Return false if it can be true for each x. Else drop the bounds for which it is false.
- */
-static bool bound_of_zext32(u8 op, struct scev_value *bound)
-{
- switch (op) {
- case BPF_JGT:
- case BPF_JGE:
- break;
- case BPF_JSGT:
- case BPF_JSGE:
- if (cnum64_smin(bound->r64) < 0)
- return false;
- break;
- case BPF_JSLT:
- case BPF_JSLE:
- if (cnum64_smax(bound->r64) > U32_MAX)
- return false;
- break;
- default:
- if (cnum64_umax(bound->r64) > U32_MAX)
- return false;
- }
- if (cnum64_umax(bound->r64) > U32_MAX) {
- bound->r64 = cnum64_intersect(bound->r64, cnum64_from_urange(0, U32_MAX));
- bound->r32 = cnum32_from_cnum64(bound->r64);
- }
- return true;
-}
-
-/*
- * The iteration count depends on the initial value, so 'initial + slope * count' is far off when
- * the initial value is a range. There is a better bound for the variable that the latch tests:
- * at the loop header it is what it is at the latch, @iv, minus what the iteration adds on the way.
- *
- * A 32-bit latch tells that about the low 32 bits of the register. It can test a copy of a 64-bit
- * induction variable, 'r6 += 1; w3 = w6; if r3 < 10 goto l': (u32)(a + b) is (u32)a + (u32)b, so
- * the range holds whatever the upper 32 bits of the register and of the step are.
- */
-static void latch_iv_range(struct bpf_verifier_env *env, struct bpf_func_state *st,
- const struct linear_latch *latch, struct cnum64 iv,
- struct bpf_loop_iters *iters)
-{
- struct scev *scev = env->scev;
- struct scev_value off;
- u32 l, r, regno;
- u64 c = 0;
-
- l = latch->base_expr;
- if (!is_reg_or_low32(scev, l, ®no)) {
- if (!is_add(scev, latch->base_expr, &l, &r) || !is_reg_or_low32(scev, l, ®no) ||
- !eval_expr(env, scev, st, r, 0, &off) || !cnum64_is_const(off.r64) ||
- off.ptr_reg >= 0)
- return;
- c = off.r64.base;
- }
- scalar_set_unknown(&iters->iv_range);
- if (latch->jmp32 || latch->zext32)
- iters->iv_range.r32 = cnum32_sub(cnum32_rshift32(iv), cnum32_const(c));
- else
- iters->iv_range.r64 = cnum64_sub(iv, cnum64_const(c));
- iters->iv_reg = regno;
-}
-
-/*
- * Loop with post-condition:
- *
- * r0 = 0
- * l: ... r0 ∈ [0,1,2] header executed 3 times
- * r0 += 1 r0 ∈ [0,1,2]
- * ... r0 ∈ [1,2,3]
- * if r0 != 3 goto l r0 ∈ [1,2,3] backedge taken 2 times
- * ... r0 ∈ [3]
- *
- * SCEV at header: r0 = k
- * SCEV at latch: r0 = 1 + k
- *
- * Loop with pre-condition:
- *
- * r0 = 0
- * l: ... r0 ∈ [0,1,2,3] header executed 4 times
- * if r0 == 3 goto e r0 ∈ [0,1,2,3] backedge taken 3 times
- * r0 += 1 r0 ∈ [0,1,2]
- * ... r0 ∈ [1,2,3]
- * goto l r0 ∈ [1,2,3]
- * e: ... r0 ∈ [3]
- *
- * SCEV at header: r0 = k
- * SCEV at latch: r0 = k
- */
-static bool compute_max_iters(struct bpf_verifier_env *env,
- struct bpf_func_state *st,
- struct linear_latch *latch,
- struct bpf_loop_iters *iters, u64 *false_until)
-{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_loop *loop = aux[bpf_loop_at_index(env, latch->insn_idx)].loop;
- struct scev *scev = env->scev;
- struct scev_value initial_val, step_val, bound_val, reg_val;
- bool sign = is_signed_op(latch->op), konst;
- struct cnum64 initial, bound, off;
- u64 step, unit = 1, stride;
- struct exit_limit el;
- bool low32;
- u32 l, r;
-
- iters->iv_reg = -1;
-
- if (!eval_expr(env, scev, st, latch->base_expr, 0, &initial_val) ||
- !eval_expr(env, scev, st, latch->step_expr, 0, &step_val) ||
- !eval_expr(env, scev, st, latch->bound_expr, 0, &bound_val))
- return false;
-
- if (step_val.ptr_reg >= 0) {
- if (env->log.level & BPF_LOG_LEVEL2) {
- bpf_log(&env->log, "scev: latch step is a pointer ");
- log_scev_value(env, st, &step_val);
- bpf_log(&env->log, "\n");
- }
- return false;
- }
- if (!scev_value_compatible(env, st, &initial_val, &bound_val, latch))
- return false;
-
- if (!cnum64_is_const(step_val.r64))
- return false;
- konst = cnum64_is_const(initial_val.r64) && cnum64_is_const(bound_val.r64);
-
- if (latch->zext32 && !latch->jmp32) {
- /*
- * (zext32 (linear ...)) is zext32() of something in all iterations but
- * the first one at the loop header, where it is what the register has:
- * no difference if that fits in 32 bits. (+ register imm) is zero extended
- * in the first iteration too, simplify() makes this shape of
- * (zext32 (+ ... imm)) only. Either the register or the sum has to fit.
- */
- l = latch->base_expr;
- if (!is_reg_or_low32(scev, l, &r) && !is_add(scev, latch->base_expr, &l, &r))
- return false;
- if (cnum64_umax(initial_val.r64) > U32_MAX &&
- (!eval_expr(env, scev, st, l, 0, ®_val) ||
- cnum64_umax(reg_val.r64) > U32_MAX))
- return false;
- if (!bound_of_zext32(latch->op, &bound_val))
- return false;
- sign = false;
- }
- low32 = initial_val.ptr_reg < 0 && (latch->jmp32 || latch->zext32);
- stride = scalar_stride(scalar_of(&initial_val), low32, &initial);
- bound = scalar_arc(scalar_of(&bound_val), low32);
- /* '!=' has to hit each bound. For the rest the largest or the smallest bound counts. */
- if (latch->op == BPF_JNE)
- stride = gcd64(stride, scalar_stride(scalar_of(&bound_val), low32, &bound));
- step = step_val.r64.base;
- if (initial_val.ptr_reg >= 0) {
- /*
- * The jump compares addresses, the ranges are those of the offsets from one base.
- * fp-1 is below fp+0: whatever the op is, addresses have the order of signed
- * offsets, as long as base + offset does not wrap around. The limit is that of
- * reg_not_null(). Nothing limits the offset of a pointer that is compared only.
- */
- off = cnum64_from_srange(-BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF);
- if (!cnum64_is_subset(off, initial) || !cnum64_is_subset(off, bound))
- return false;
- sign = true;
- } else if (low32) {
- step <<= 32;
- unit <<= 32;
- }
-
- /* no bound is left for which the latch can be true */
- if (cnum64_is_empty(bound))
- el = (struct exit_limit){};
- else if (!compute_backedge_taken_count(latch->op, sign, unit, initial, step, bound, stride,
- &el))
- return false;
- /*
- * max_header_count is u32 and U32_MAX means "infinite",
- * check to avoid overflow below.
- */
- if (el.max >= U32_MAX - 1)
- return false;
- if (el.max && initial_val.ptr_reg >= 0) {
- /* el.iv has the value that ends the loop. Past the limit the latch may be true. */
- if (!cnum64_is_subset(off, el.iv))
- return false;
- el.false_until = min(el.false_until, (s64)step < 0 ?
- div64_u64(cnum64_smin(initial) + BPF_MAX_VAR_OFF, -step) :
- div64_u64(BPF_MAX_VAR_OFF - cnum64_smax(initial), step));
- }
- /* linear_bounds() is exact for constants */
- if (el.max && !konst)
- latch_iv_range(env, st, latch, el.iv, iters);
-
- *false_until = el.false_until;
- /* The header executes once on entry and once for each taken backedge, hence +1. */
- iters->max_header_count = el.max + 1;
- /*
- * The latch is a conditional jump with one jump target exiting the loop.
- * The loop still, however can have multiple exits.
- * In such case, conservatively assume that non-latch exit can happen
- * at any iteration, thus setting minimal number of iterations as 0.
- */
- iters->min_header_count = konst && loop->exits_cnt == 1 ? iters->max_header_count : 0;
- return true;
-}
-
-static void mark_scev_reg_scratched(struct bpf_verifier_env *env, u32 r)
-{
- if (r < MAX_BPF_REG)
- mark_reg_scratched(env, r);
- else
- mark_stack_slot_scratched(env, r - __MAX_BPF_REG);
-}
-
-/*
- * An unknown scalar is its own widening.
- * Stack bytes without a value are represented by scev->unknown_reg.
- */
-static bool is_unbound_scev_reg(struct bpf_verifier_env *env, const struct bpf_func_state *st,
- u32 r)
-{
- const struct bpf_reg_state *reg = scev_reg_for_read(env, st, r);
-
- return reg && reg->type == SCALAR_VALUE && scalar_is_unknown(scalar_of(reg));
-}
-
-/*
- * scev_reg_for_read() has scev->unknown_reg for a partial spill, because a part does not tell
- * what the 8 bytes of the slot are. But the state has the value of the spilled bytes.
- * Does it know something about them? If not, they are their own widening too.
- */
-static bool is_known_partial_spill(const struct bpf_func_state *st, u32 r)
-{
- const struct bpf_stack_state *stack = scev_regno_to_stack(st, r);
- const struct bpf_reg_state *reg;
- u64 max;
-
- if (!stack || !bpf_is_spilled_reg(stack) || bpf_spill_size(stack) == BPF_REG_SIZE)
- return false;
- reg = &stack->spilled_ptr;
- max = GENMASK_ULL(bpf_spill_size(stack) * BITS_PER_BYTE - 1, 0);
- return (reg->var_off.mask & max) != max || reg->r64.size < max || reg->r32.size < max ||
- reg->step > 1;
-}
-
-/*
- * Is @id made from a partial spill that the loop changes? '?' can be made from anything.
- * @self is the slot whose next value is @id, as long as only (any ...) is above @id:
- * (any self ...) tells that one of the paths does not change the slot.
- */
-static bool made_from_partial_spill(struct scev *scev, const struct bpf_func_state *st,
- const struct env *header_env, u32 id, int self, u32 lvl)
-{
- const struct expr *e = &scev->exprs[id];
- u32 i;
-
- if (lvl >= EXPR_STACK_DEPTH || e->op == UNKNOWN)
- return true;
- if (e->op == REG)
- return (int)e->params[0] != self && is_known_partial_spill(st, e->params[0]) &&
- header_env->reg2expr[e->params[0]] != id;
- for (i = 0; i < op_params_num(e->op); i++)
- if (made_from_partial_spill(scev, st, header_env, e->params[i],
- e->op == ANY ? self : -1, lvl + 1))
- return true;
- return false;
-}
-
-/*
- * bpf_compute_const_regs() has @r as a constant at @insn_idx, the head of a loop: it's the same in
- * every iteration. do_check() takes a value that a range adds to it, or to what is computed
- * from it, for a bug.
- * That pass knows no types: arithmetic on a pointer to arena runs in 32 bits, and a pointer and a
- * number can have one constant. As for a pointer to arena below, this is not relied on.
- */
-static bool is_const_scev_reg(struct bpf_verifier_env *env, int insn_idx, u32 r)
-{
- struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
-
- return r < ARRAY_SIZE(aux->const_reg_vals) &&
- ((aux->const_reg_mask | aux->const_reg_wide_mask) & BIT(r));
-}
-
-/* Main logic in verifier.c forbids varying offsets for certain register types. */
-bool bpf_is_widenable_reg_type(const struct bpf_reg_state *reg)
-{
- if (type_may_be_null(reg->type))
- return false;
-
- switch (base_type(reg->type)) {
- case SCALAR_VALUE:
- case PTR_TO_MAP_VALUE:
- case PTR_TO_MAP_KEY:
- case PTR_TO_STACK:
- case PTR_TO_PACKET:
- case PTR_TO_PACKET_META:
- case PTR_TO_MEM:
- case PTR_TO_BTF_ID:
- return true;
- default:
- return false;
- }
-}
-
-static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
-{
- switch (base_type(reg->type)) {
- case PTR_TO_PACKET:
- case PTR_TO_PACKET_META:
- reg->id = ++env->id_gen;
- break;
- default:
- break;
- }
-}
-
-/* Check that ANY leaves can be unioned with r's loop-entry value. */
-static bool is_widenable_any(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry,
- struct env *header_env, u32 r, u32 id)
-{
- const struct bpf_reg_state *reg, *leaf_reg;
- struct scev *scev = env->scev;
- u32 l, ra, rr, leaf, order;
- s64 imm;
-
- if (!is_any(scev, id, &l, &rr))
- return false;
- reg = scev_reg_for_read(env, loop_entry, r);
- if (!reg || !bpf_is_widenable_reg_type(reg))
- return false;
-
- scev->stack_sz = 0;
- expr_stack_push(scev, id);
- while (expr_next(scev, &id, &order)) {
- if (order & DEPTH_LIMIT)
- return false;
- if (!(order & PRE) || is_any(scev, id, &l, &rr))
- continue;
- if (is_imm(scev, id, &imm)) {
- if (reg->type != SCALAR_VALUE)
- return false;
- } else if (is_reg(scev, id, &leaf)) {
- /* Other leaves must denote loop-invariant registers. */
- if (leaf != r &&
- !(is_reg(scev, header_env->reg2scev[leaf], &ra) && ra == leaf))
- return false;
- leaf_reg = scev_reg_for_read(env, loop_entry, leaf);
- if (!leaf_reg || !bpf_reg_union_compatible(reg, leaf_reg))
- return false;
- } else {
- return false;
- }
- }
- return true;
-}
-
-static void log_bounds(struct bpf_verifier_env *env, const struct scalar *bounds)
-{
- struct bpf_verifier_log *log = &env->log;
-
- bpf_verbose_snum(env, cnum64_smin(bounds->r64));
- bpf_log(log, "..");
- bpf_verbose_snum(env, cnum64_smax(bounds->r64));
- if (bounds->r32.size < U32_MAX &&
- (cnum32_smin(bounds->r32) != cnum64_smin(bounds->r64) ||
- cnum32_smax(bounds->r32) != cnum64_smax(bounds->r64))) {
- bpf_log(log, " (s32 ");
- bpf_verbose_snum32(env, cnum32_smin(bounds->r32));
- bpf_log(log, "..");
- bpf_verbose_snum32(env, cnum32_smax(bounds->r32));
- bpf_log(log, ")");
- }
- bpf_log(log, " step %u", bounds->step);
-}
-
-struct linear_coefs {
- u32 base_reg;
- s64 slope;
- bool zext32;
- /* with zext32: the low 32 bits of a stack slot */
- bool spill32;
- /* with zext32: the low 32 bits are sign extended */
- bool sext32;
-};
-
-static bool is_simple_slope(struct scev *scev, u32 id, s64 *slope)
-{
- return is_imm(scev, id, slope) &&
- *slope <= S16_MAX &&
- *slope >= S16_MIN &&
- *slope != 0;
-}
-
-static bool is_simple_linear(struct scev *scev, u32 id, struct linear_coefs *out)
-{
- u32 base, slope;
-
- out->spill32 = false;
- out->sext32 = is_unop(scev, SEXT32, id, &id);
- if (!is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) ||
- (out->sext32 && out->zext32))
- return false;
- out->zext32 |= out->sext32;
- return is_reg(scev, base, &out->base_reg) &&
- is_simple_slope(scev, slope, &out->slope);
-}
-
-/*
- * Is @r 'its entry value + slope * k' at the header in iteration k? @id is its SCEV there. At a
- * jump in the loop @id can be linear in the entry value of another register: the entry value of @r
- * does not bound it then.
- */
-static bool is_linear_at_header(struct scev *scev, u32 id, u32 r, struct linear_coefs *out)
-{
- return is_simple_linear(scev, id, out) && out->base_reg == r;
-}
-
-/*
- * A 32-bit counter that lives on the stack: the SCEV of slot @r is
- * (spill32 (linear (zext32 @r) slope)) and there is a 32-bit number in the slot in @st.
- * Return what is known about that number.
- */
-static struct bpf_reg_state *spill32_counter(struct scev *scev, struct bpf_func_state *st,
- u32 id, u32 r, struct linear_coefs *out)
-{
- u32 t, base, slope;
-
- if (!is_spill32(scev, id, &t) ||
- !is_linear(scev, t, &base, &slope) ||
- !is_zext32(scev, base, &t) ||
- !is_reg(scev, t, &out->base_reg) || out->base_reg != r ||
- !is_simple_slope(scev, slope, &out->slope))
- return NULL;
- out->zext32 = true;
- out->spill32 = true;
- out->sext32 = false;
- return scev_reg_for_spill32(st, r);
-}
-
-/* Compute the range an induction variable in `reg` spans over the loop. */
-static void linear_bounds(const struct scalar *reg, u32 max_header_count,
- struct linear_coefs *coefs, struct scalar *out)
-{
- s64 slope = coefs->slope;
- s64 slope_abs = slope < 0 ? -slope : slope;
- struct scalar low;
- s64 min_val, max_val;
- struct cnum64 r64;
- struct cnum32 r32;
- s64 total_change;
- bool overflow, konst;
- u16 step;
-
- if (coefs->spill32) {
- /* what 'rX = *(u32 *)(r10 - 8)' loads */
- low = *reg;
- scalar_zext32(&low);
- reg = &low;
- }
- min_val = cnum64_smin(reg->r64);
- max_val = cnum64_smax(reg->r64);
- overflow = check_mul_overflow(slope, (s64)max_header_count - 1, &total_change) ||
- (slope > 0 && check_add_overflow(max_val, total_change, &max_val)) ||
- (slope < 0 && check_add_overflow(min_val, total_change, &min_val));
- r64 = overflow ? CNUM64_UNBOUNDED : cnum64_from_srange(min_val, max_val);
- r32 = CNUM32_UNBOUNDED;
- if (coefs->zext32) {
- r32 = cnum32_from_cnum64(r64);
- r64 = coefs->sext32 ? cnum64_sext32(r32) : cnum64_zext32(r32);
- /*
- * 32-bit iteration overflow is equivalent to subtracting N * 2**32:
- * (base + slope * k) % slope ≠ (base + slope * k - N * 2**32) % slope.
- */
- overflow |= coefs->sext32 ?
- cnum32_contains(r32, S32_MAX) && cnum32_contains(r32, S32_MIN) :
- cnum32_contains(r32, 0) && cnum32_contains(r32, U32_MAX);
- /*
- * The entry and zero-extended values must agree on how they define `base`,
- * otherwise `base + slope * k` would define different sets of points between
- * the two.
- */
- overflow |= imod(cnum64_smin(reg->r64), slope_abs) !=
- imod(cnum64_smin(r64), slope_abs);
- /*
- * Full range is a union of a reg's value at loop entry,
- * plus a range produced by 32-bit iteration.
- */
- r64 = cnum64_union(reg->r64, r64);
- r32 = cnum32_union(reg->r32, r32);
- }
- /*
- * The value set is 'v + slope * k'. It has the power-of-two alignment shared by the entry
- * value and the slope. The entry value is congruent to base modulo step, so the sum is,
- * modulo gcd(step, |slope|), unless it wraps. A single point is its own base, its step
- * is 0: every number divides 0. One step is kept: the gcd if it implies the alignment.
- */
- out->step = 1u << min_t(u32, tnum_alignment(reg->var_off), __ffs(slope_abs));
- out->base = 0;
- konst = cnum64_is_const(reg->r64);
- step = gcd(konst ? 0 : reg->step, slope_abs);
- if (!overflow && step > out->step && !(step % out->step)) {
- out->step = step;
- out->base = konst ? imod(cnum64_smin(reg->r64), step) : reg->base % step;
- }
- out->var_off = tnum_unknown;
- out->r64 = r64;
- out->r32 = r32;
-}
-
-/*
- * Is @r the induction variable that the latch tests, or congruent to it: equal to it at the header
- * of the loop at @h in every iteration? Then the range of the one is the range of the other.
- * Expressions have no types: after 'r9 = le64 r9' a scalar holds the address, and the range of the
- * pointer is that of its offset. So in @loop_entry both are scalars, or pointers to one object.
- */
-static bool is_iv_at_header(struct bpf_verifier_env *env, const struct bpf_func_state *loop_entry,
- int h, int iv_reg, u32 r)
-{
- const struct bpf_reg_state *iv, *reg;
- s64 delta;
-
- if (iv_reg < 0)
- return false;
- if (r == iv_reg)
- return true;
- iv = scev_reg_for_read(env, loop_entry, iv_reg);
- reg = scev_reg_for_read(env, loop_entry, r);
- return iv && reg && bpf_reg_union_compatible(iv, reg) &&
- const_diff_at_header(env, h, iv_reg, r, &delta) && !delta;
-}
-
-/*
- * A 32-bit latch bounds the low 32 bits of the induction variable. What follows for all 64 bits
- * is the reduction that reg_bounds_sync() makes. It is made here for the log, and because two
- * ranges that are not empty can have no number in common.
- */
-static void iv_bounds(struct scalar *bounds, const struct scalar *iv_range)
-{
- struct cnum32 r32 = cnum32_intersect(bounds->r32, iv_range->r32);
- struct cnum64 r64 = cnum64_intersect(bounds->r64, iv_range->r64);
-
- r64 = cnum64_cnum32_intersect(r64, r32);
- if (cnum64_is_empty(r64))
- return;
- bounds->r64 = r64;
- bounds->r32 = r32;
-}
-
-/*
- * Nothing that @id is computed from is what a register had at the head of the loop, and
- * some of it is not known: it's made anew in every iteration. @call: by a call.
- */
-static bool expr_is_fresh(struct bpf_verifier_env *env, u32 id, bool *call)
-{
- struct scev *scev = env->scev;
- bool opaque = false;
- u32 order, regno;
-
- *call = false;
- scev->stack_sz = 0;
- expr_stack_push(scev, id);
- while (expr_next(scev, &id, &order)) {
- if (order & DEPTH_LIMIT)
- return false;
- if (!(order & PRE))
- continue;
- if (is_reg(scev, id, ®no) || is_op(scev, id, UNKNOWN))
- return false;
- if (is_opaque(scev, id)) {
- opaque = true;
- *call |= scev->exprs[id].params[0];
- }
- }
- return opaque;
-}
-
-/*
- * Does @id have no bit that register @r doesn't have at the head of the loop? 'x & e' and the
- * low bits of x lose bits only. 'x >> e', 'x / e' and 'x % e' are not greater than x as well, but
- * iterations one by one tell more about them: a number that is halved 8 times.
- */
-static bool expr_loses_bits_only(struct scev *scev, u32 id, u32 r, int depth)
-{
- struct expr *e = &scev->exprs[id];
- u32 regno;
-
- if (is_reg(scev, id, ®no))
- return regno == r;
- if (depth > 8)
- return false;
- switch (e->op) {
- case BPF_AND:
- return expr_loses_bits_only(scev, e->params[0], r, depth + 1) ||
- expr_loses_bits_only(scev, e->params[1], r, depth + 1);
- case ANY:
- return expr_loses_bits_only(scev, e->params[0], r, depth + 1) &&
- expr_loses_bits_only(scev, e->params[1], r, depth + 1);
- case ZEXT8:
- case ZEXT16:
- case ZEXT32:
- return expr_loses_bits_only(scev, e->params[0], r, depth + 1);
- default:
- return false;
- }
-}
-
-/* The ranges of @reg are [0, max]. So no bit is known to be 1. */
-static bool reg_is_zero_to_max(const struct bpf_reg_state *reg)
-{
- struct cnum64 r64 = cnum64_from_urange(0, reg_umax(reg));
-
- return cnum64_is_subset(reg->r64, r64) &&
- cnum32_is_subset(reg->r32, cnum32_from_cnum64(r64)) && reg->step <= 1;
-}
-
-/*
- * A jump that leaves the loop and dominates @backedge tells in which iterations @backedge can
- * be taken: in none from the count of the jump to @false_until. That bounds the iterations of
- * the loop only if it has no other back edge. Keep the jump with the smallest count.
- * Return 1 if there is one.
- */
-static int count_backedge(struct bpf_verifier_env *env, struct bpf_func_state *st, int insn_idx,
- struct bpf_backedge *backedge, struct bpf_loop_iters *iters,
- struct linear_latch *best, u64 *false_until)
-{
- struct bpf_verifier_log *log = &env->log;
- struct bpf_loop_iters tmp;
- struct linear_latch latch;
- bool found = false;
- int linear_latch;
- int latch_idx;
- u64 until;
- int j;
-
- /* If this backedge has a latch */
- if (!backedge->latches_cnt) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log, "loop header at %d, latch not identified\n", insn_idx);
- return 0;
- }
-
- for (j = 0; j < backedge->latches_cnt; j++) {
- latch_idx = backedge->latches[j];
- linear_latch = match_linear_latch(env, insn_idx, latch_idx,
- backedge->taken_leaves & BIT(j), &latch);
- if (linear_latch < 0)
- return linear_latch;
- if (!linear_latch) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log, "loop header at %d, non-linear %s at %d\n",
- insn_idx, j ? "exit" : "latch", latch_idx);
- continue;
- }
- if (!compute_max_iters(env, st, &latch, &tmp, &until)) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log,
- "loop header at %d, can't compute iterations count at %d\n",
- insn_idx, latch_idx);
- continue;
- }
- /* a flag can have other reasons to end the loop */
- if (!bpf_jump_leaves_loop(env, insn_idx, latch_idx))
- tmp.min_header_count = 0;
- if (!found || tmp.max_header_count < iters->max_header_count) {
- iters->max_header_count = tmp.max_header_count;
- iters->iv_reg = tmp.iv_reg;
- iters->iv_range = tmp.iv_range;
- *false_until = until;
- *best = latch;
- }
- iters->min_header_count = found ? min(iters->min_header_count, tmp.min_header_count)
- : tmp.min_header_count;
- found = true;
- }
- if (!found && (log->level & BPF_LOG_LEVEL2))
- bpf_log(log, "loop header at %d, can't compute iterations count\n", insn_idx);
- return found;
-}
-
-/* Two jumps that test the same thing: in each iteration both are true or both are false. */
-static bool same_latch(const struct linear_latch *a, const struct linear_latch *b)
-{
- return a->base_expr == b->base_expr && a->step_expr == b->step_expr &&
- a->bound_expr == b->bound_expr && a->op == b->op &&
- a->zext32 == b->zext32 && a->jmp32 == b->jmp32;
-}
-
-int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- struct bpf_loop_iters *iters)
-{
- struct bpf_func_state *cur_func = st->frame[st->curframe];
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
- struct bpf_verifier_log *log = &env->log;
- struct scev *scev = env->scev;
- struct env *header_env;
- struct linear_latch latch, first;
- struct linear_coefs coefs;
- struct bpf_loop_iters tmp;
- struct bpf_loop *loop;
- int insn_idx = st->insn_idx;
- u64 until, false_until;
- bool same = true, walk = false, call;
- int err, i;
- u32 r;
-
- if (!env->allow_uninit_stack)
- return 0;
-
- /*
- * If insn_idx is a loop header for a reducible loop.
- * loop is NULL for secondary entries to irreducible loops.
- */
- loop = aux[insn_idx].loop;
- if (!loop || loop->irreducible || loop->backedges_overflow ||
- loop->nested_exits_overflow) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log, "loop header at %d, unsupported loop:%s%s%s\n", insn_idx,
- !loop || loop->irreducible ? " irreducible" : "",
- loop && loop->backedges_overflow ? " too many backedges" : "",
- loop && loop->nested_exits_overflow ?
- " too many exits of a nested loop" : "");
- return 0;
- }
-
- /*
- * A register that is linear at the header is 'first value + slope * k' in iteration k
- * whatever back edges were taken. So the jumps that dominate a back edge say in
- * which iterations it can be taken. The loop ends, at the latest, in an iteration
- * in which no back edge can be taken.
- */
- for (i = 0; i < loop->backedges_cnt; i++) {
- err = count_backedge(env, cur_func, insn_idx, &loop->backedges[i],
- i ? &tmp : iters, &latch, &until);
- if (err <= 0)
- return err;
- if (!i) {
- false_until = until;
- first = latch;
- continue;
- }
- same &= same_latch(&first, &latch);
- false_until = min(false_until, until);
- iters->max_header_count = max(iters->max_header_count, tmp.max_header_count);
- iters->min_header_count = min(iters->min_header_count, tmp.min_header_count);
- /* at the header the variable is what one of the latches lets through */
- if (iters->iv_reg >= 0 && iters->iv_reg == tmp.iv_reg)
- scalar_union(&iters->iv_range, &tmp.iv_range);
- else
- iters->iv_reg = -1;
- }
- /*
- * When the latches that were kept are the same test, the first iteration in which it's
- * false is the last one. Otherwise take the largest count and make sure that the latches
- * which became false earlier are false still.
- */
- if (!same && iters->max_header_count - 1 > false_until) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log,
- "loop header at %d, a latch may be true again in iteration %u\n",
- insn_idx, iters->max_header_count - 1);
- return 0;
- }
-
- /*
- * It leaves the loop itself: a flag does that later. If one edge does the one and the
- * other edge the other, the last iteration goes on behind the jump as well.
- */
- iters->count_insn = bpf_jump_leaves_loop(env, insn_idx, first.insn_idx) &&
- aux[first.insn_idx].edge_leaves != 3 ? first.insn_idx : -1;
-
- if (iters->max_header_count == 0) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log, "loop header at %d, 0 iterations count\n", insn_idx);
- return 0;
- }
-
- if (iters->max_header_count == U32_MAX) {
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log, "loop header at %d, inf iterations count\n", insn_idx);
- return 0;
- }
-
- if (log->level & BPF_LOG_LEVEL2) {
- bpf_log(log, "loop header at %d, header_count is ", insn_idx);
- if (iters->min_header_count == iters->max_header_count)
- bpf_log(log, "%u ", iters->max_header_count);
- else
- bpf_log(log, "[%u..%u] ", iters->min_header_count, iters->max_header_count);
- bpf_log(log, "\n");
- }
-
- err = bpf_live_stack_query_init(env, st);
- if (err)
- return err;
-
- bitmap_zero(iters->forget_regs, REGS_NUM);
- bitmap_zero(iters->fresh_regs, REGS_NUM);
- bitmap_zero(iters->mask_regs, REGS_NUM);
- iters->spill_base = false;
- header_env = find_header_env(scev, insn_idx);
- for (r = 0; r < REGS_NUM; r++) {
- struct bpf_reg_state *reg = NULL;
- u32 ra, r_scev, r_expr;
- bool spill_base = false;
-
- if (!scev_reg_alive(env, st, r))
- continue;
-
- r_scev = header_env->reg2scev[r];
- if (spill32_counter(scev, cur_func, r_scev, r, &coefs)) {
- /* There is no exact offset to lose if nothing is known about the number */
- if (test_bit(r, loop->store_base_regs) &&
- is_known_partial_spill(cur_func, r)) {
- spill_base = true;
- goto cant_widen;
- }
- continue;
- }
- /*
- * A heuristic:
- * Ignore SCEVs computed for unbound values and treat those as widened.
- * This is poor man's modelling of e.g. bpf_probe_read_kernel() effect
- * on stack, when called from a subprogram. Currently this would produce
- * an unknown ('?') SCEV as a summary of a subprogram call effects.
- * Ideally, this should be computed as an OPAQUE value.
- * However, that would require tracking STACK_MISC writes in the
- * liveness analysis. Not implemented yet.
- */
- if (is_unbound_scev_reg(env, cur_func, r)) {
- /*
- * Nothing widens the value that the state has for a partial spill. A
- * value that is made from itself can be a new one in each iteration:
- * treat it like a register that has no formula. It's in the half
- * of the slot that is farther from fp.
- */
- if (is_known_partial_spill(cur_func, r) &&
- bpf_stack_slot_alive(env, st->curframe, (r - MAX_BPF_REG) * 2 + 1) &&
- made_from_partial_spill(scev, cur_func, header_env,
- header_env->reg2expr[r], r, 0))
- goto cant_widen;
- continue;
- }
-
- if (is_const_scev_reg(env, insn_idx, r))
- continue;
-
- reg = scev_reg_for_linear(cur_func, r);
- /*
- * The offset of a pointer to arena is not tracked, so there is nothing to lose.
- * If the loop makes something else of it the state that returns to the head
- * is not covered by the state at the head.
- */
- if (reg && reg->type == PTR_TO_ARENA) {
- __set_bit(r, iters->forget_regs);
- continue;
- }
- /* If SCEV for r is (linear <reg> <slope>) */
- if (reg && is_linear_at_header(scev, r_scev, r, &coefs) &&
- bpf_is_widenable_reg_type(reg)) {
- /* 32-bit ALU operations on pointers convert them to scalars */
- if (coefs.zext32 && reg->type != SCALAR_VALUE)
- goto cant_widen;
- /* Spills at varying offsets lose precision */
- if (test_bit(r, loop->store_base_regs)) {
- spill_base = true;
- goto cant_widen;
- }
- continue;
- }
-
- /* rA = rA, loop does not change this reg */
- if (is_reg(scev, r_scev, &ra) && r == ra)
- continue;
-
- /* (any 1 (any 2 (any 3 4))) */
- if (is_widenable_any(env, cur_func, header_env, r, r_scev)) {
- if (test_bit(r, loop->store_base_regs)) {
- spill_base = true;
- goto cant_widen;
- }
- continue;
- }
-
- /*
- * [0, max] that loses bits is in [0, max], a bit that is 0 stays 0: what enters
- * the loop stands for all iterations as it is. 64 or [64, 127] does not: it
- * can become 0.
- */
- if (reg && reg->type == SCALAR_VALUE && !test_bit(r, loop->store_base_regs) &&
- reg_is_zero_to_max(reg) &&
- expr_loses_bits_only(scev, header_env->reg2expr[r], r, 0)) {
- __set_bit(r, iters->mask_regs);
- continue;
- }
-cant_widen:
- if (log->level & BPF_LOG_LEVEL2) {
- r_expr = header_env->reg2expr[r];
- bpf_log(log, "loop header at %d, can't widen ", insn_idx);
- log_reg(env, r);
- bpf_log(log, ", expr is ");
- log_expr(env, r_expr);
- if (spill_base)
- bpf_log(log, ", requires exact stack-offset tracking");
- bpf_log(log, "\n");
- }
- /* the count stands, it is this one that is not known in the loop */
- if (spill_base) {
- iters->spill_base = true;
- walk = true;
- continue;
- }
- /*
- * What is loaded from memory or returned by a call in every iteration is the same
- * kind of thing in all of them. One iteration tells what. A walk through the
- * iterations tells more: what goes with it, like a flag that is set if it's not
- * NULL, and each number of a table in .rodata. So a walk comes first. What is
- * loaded is often a leftover in a stack slot: not if nothing looks at it.
- */
- if (expr_is_fresh(env, header_env->reg2expr[r], &call)) {
- __set_bit(r, iters->fresh_regs);
- if (!call && !bpf_flow_is_needed(env, insn_idx, r))
- continue;
- } else {
- __set_bit(r, iters->forget_regs);
- }
- /*
- * for (i = 0; i < 40; i++) { if (i) sum += prev; prev = (x ^ i) * 3; }
- * It's not set in the first iteration only. That counts if the loop is not walked:
- * nothing looks at prev, or there are too many iterations.
- */
- if (reg && reg->type == NOT_INIT)
- __set_bit(r, iters->fresh_regs);
- /*
- * Only a walk through the iterations tells what it is in each of them. That is not
- * worth it for a number that nothing looks at: a sum, a hash, what a call leaves in
- * r1 - r5. What comes back to the head is joined with what entered then. A pointer,
- * or a slot that holds something else than a register, has no such join.
- */
- if (!reg || (reg->type != SCALAR_VALUE && reg->type != NOT_INIT) ||
- bpf_flow_is_needed(env, insn_idx, r)) {
- walk = true;
- continue;
- }
- if (log->level & BPF_LOG_LEVEL2)
- bpf_log(log, "loop header at %d, nothing looks at it\n", insn_idx);
- }
-
- /* 2: the count is known, what some of the live registers are is not and it matters */
- return walk ? 2 : 1;
-}
-
-/* bpf_compute_loop_iters() checked that every leaf can be unioned into acc. */
-static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry,
- struct bpf_reg_state *acc, u32 id)
-{
- struct bpf_reg_state *tmp = &env->fake_reg[0];
- const struct bpf_reg_state *leaf_reg;
- struct scev *scev = env->scev;
- u32 l, r, leaf, order;
- s64 smax = reg_smax(acc);
- s64 imm;
- int err;
-
- scev->stack_sz = 0;
- expr_stack_push(scev, id);
- while (expr_next(scev, &id, &order)) {
- if (order & DEPTH_LIMIT) {
- verifier_bug(env, "scev ANY union exceeds expression depth limit");
- return -EFAULT;
- }
- if (!(order & PRE) || is_any(scev, id, &l, &r))
- continue;
- if (is_imm(scev, id, &imm)) {
- bpf_mark_reg_known_scalar(tmp, imm);
- leaf_reg = tmp;
- } else if (is_reg(scev, id, &leaf)) {
- leaf_reg = scev_reg_for_read(env, loop_entry, leaf);
- if (verifier_bug_if(!leaf_reg, env, "unreadable SCEV ANY leaf %u", leaf))
- return -EFAULT;
- } else {
- verifier_bug(env, "scev ANY union has an unsupported leaf");
- return -EFAULT;
- }
- smax = max(smax, reg_smax(leaf_reg));
- err = bpf_reg_union(env, acc, leaf_reg);
- if (err)
- return err;
- }
- /* All leaves have the same range. It goes with the largest offset of all leaves. */
- bpf_trim_pkt_range(acc, smax);
- return 0;
-}
-
-/*
- * Make @st, which is at the head of a counted loop, stand for all iterations. @loop_entry is the
- * state that entered the loop and @iters is what bpf_compute_loop_iters() says about it.
- *
- * @st is a copy of @loop_entry, or, if the loop has iters->fresh_regs, the state that comes back to
- * the head after the first iteration, @back. These registers are left as the first iteration
- * made them. At the entry they are something else: 0, a pointer, not set.
- */
-int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
- struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters,
- bool back)
-{
- struct bpf_func_state *cur_func = st->frame[st->curframe];
- struct bpf_func_state *entry_func = loop_entry->frame[loop_entry->curframe];
- const struct bpf_reg_state *val, *entry_val;
- struct bpf_verifier_log *log = &env->log;
- struct scev *scev = env->scev;
- struct bpf_reg_state *reg, *spill32;
- struct env *header_env;
- struct linear_coefs coefs;
- struct scalar bounds;
- u32 r, r_expr, a, b;
- int insn_idx = st->insn_idx;
- int err;
-
- header_env = find_header_env(scev, insn_idx);
- for (r = 0; r < REGS_NUM; r++) {
- if (!scev_reg_alive(env, st, r))
- continue;
-
- /*
- * It stays what it is in @st. What comes back to the head is joined with it and
- * widened to where the jumps of the loop stop it, like in a loop without a count.
- * NULL or not is all that is known about a pointer to arena.
- */
- if (test_bit(r, iters->forget_regs)) {
- reg = scev_reg_for_linear(cur_func, r);
- if (reg && reg->type == PTR_TO_ARENA) {
- scalar_set_unknown(scalar_of(reg));
- mark_scev_reg_scratched(env, r);
- }
- continue;
- }
- if (test_bit(r, iters->fresh_regs))
- continue;
-
- r_expr = header_env->reg2scev[r];
- spill32 = spill32_counter(scev, cur_func, r_expr, r, &coefs);
- /* @iters says nothing about a register that is not known in @loop_entry */
- if (!spill32 && (is_unbound_scev_reg(env, entry_func, r) ||
- is_const_scev_reg(env, insn_idx, r)))
- continue;
-
- /*
- * A call in the loop can change the type, see bpf_clamp_scev_regs(). Then the
- * register is left alone. What comes back to the head next is compared with it.
- */
- val = scev_reg_for_read(env, cur_func, r);
- entry_val = scev_reg_for_read(env, entry_func, r);
- if (back && (!val || !entry_val || !bpf_reg_union_compatible(val, entry_val)))
- continue;
- if (test_bit(r, iters->mask_regs)) {
- /* the range stays */
- err = scev_reg_for_write(env, cur_func, r, ®);
- if (!err && back)
- err = bpf_reg_union(env, reg, entry_val);
- if (err)
- return err;
- if (log->level & BPF_LOG_LEVEL2) {
- bpf_log(log, "loop header at %d, ", insn_idx);
- log_reg(env, r);
- bpf_log(log, " stays in 0..%llu, it loses bits only\n",
- reg_umax(reg));
- }
- mark_scev_reg_scratched(env, r);
- continue;
- }
-
- /* If SCEV for r is (linear <reg> <slope>)*/
- if (spill32 || is_linear_at_header(scev, r_expr, r, &coefs)) {
- /* the range is derived from @loop_entry: bpf_clamp_scev_regs() */
- if (back)
- continue;
- /* the other half of the slot stays as it is */
- reg = spill32;
- err = spill32 ? 0 : scev_reg_for_write(env, cur_func, r, ®);
- if (err)
- return err;
- linear_bounds(scalar_of(reg), iters->max_header_count, &coefs, &bounds);
- if (is_iv_at_header(env, entry_func, insn_idx, iters->iv_reg,
- coefs.base_reg))
- iv_bounds(&bounds, &iters->iv_range);
- if (log->level & BPF_LOG_LEVEL2) {
- bpf_log(log, "loop header at %d, widening ", insn_idx);
- log_reg(env, r);
- bpf_log(log, " to ");
- log_bounds(env, &bounds);
- bpf_log(log, "\n");
- }
- scratch_widened_reg_id(env, reg);
- /* "at pkt_end" and "beyond pkt_end" are not valid in all iterations */
- if (type_is_pkt_pointer(reg->type))
- reg->range = max(reg->range, 0);
- err = bpf_set_reg_range(env, reg, &bounds);
- if (err)
- return err;
- mark_scev_reg_scratched(env, r);
- } else if (is_any(scev, r_expr, &a, &b)) {
- err = scev_reg_for_write(env, cur_func, r, ®);
- if (err)
- return err;
- err = union_any_reg(env, entry_func, reg, r_expr);
- if (err)
- return err;
- if (log->level & BPF_LOG_LEVEL2) {
- bpf_log(log, "loop header at %d, widening ", insn_idx);
- log_reg(env, r);
- bpf_log(log, " to ");
- bpf_verbose_snum(env, reg_smin(reg));
- bpf_log(log, "..");
- bpf_verbose_snum(env, reg_smax(reg));
- bpf_log(log, " step %u\n", reg->step);
- }
- scratch_widened_reg_id(env, reg);
- mark_scev_reg_scratched(env, r);
- }
- }
- return 1;
-}
-
-/* (linear (+ rX imm) slope): the iteration has added a constant to rX so far */
-static bool is_moved_linear(struct scev *scev, u32 id, struct linear_coefs *out, s64 *moved)
-{
- u32 base, slope, l, r;
-
- out->spill32 = false;
- out->sext32 = false;
- return is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) && !out->zext32 &&
- is_add(scev, base, &l, &r) && is_reg(scev, l, &out->base_reg) &&
- is_imm(scev, r, moved) && is_simple_slope(scev, slope, &out->slope);
-}
-
-/*
- * The state came back to the head of a counted loop. Or, with @behind, it goes on in the loop
- * behind the jump that counts. That jump leaves the loop in the last iteration, so this is
- * not the last one:
- * for (i = 0; ; i++, p += 8) { if (i == 4) break; *p = 0; }
- * At the head p is up to 32 bytes from where it was, at the store up to 24. The jump tells that
- * about i only. It holds for all that is linear at the jump.
- */
-int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur_func_state, u32 insn_idx,
- const struct loop_stack_entry *entry, bool behind)
-{
- struct bpf_verifier_state *entry_state = entry->entry_state;
- struct bpf_func_state *entry_st = entry_state->frame[entry_state->curframe];
- u32 max_header_count = entry->max_header_count - behind;
- struct bpf_verifier_log *log = &env->log;
- struct bpf_reg_state *reg, *entry_reg;
- struct scev *scev = env->scev;
- struct linear_coefs coefs;
- struct scalar bounds, off;
- struct env *header_env;
- u32 r, r_scev;
- s64 moved;
- int err;
-
- if (entry_state->curframe != cur_func_state->frameno) {
- verifier_bug(env, "clamping registers for a wrong frame: %d vs %d\n",
- entry_state->curframe, cur_func_state->frameno);
- return -EFAULT;
- }
-
- header_env = find_loop_env(scev, insn_idx, behind ? entry->count_insn : insn_idx);
- if (!max_header_count)
- return 0;
- for (r = 0; r < REGS_NUM; r++) {
- r_scev = header_env->reg2scev[r];
- moved = 0;
- entry_reg = spill32_counter(scev, entry_st, r_scev, r, &coefs);
- reg = spill32_counter(scev, cur_func_state, r_scev, r, &coefs);
- if (entry_reg && reg)
- goto clamp;
-
- /* If SCEV for r is (linear <reg> <slope>)*/
- if (!is_linear_at_header(scev, r_scev, r, &coefs) &&
- !(behind && is_moved_linear(scev, r_scev, &coefs, &moved) &&
- coefs.base_reg == r))
- continue;
-
- entry_reg = scev_reg_for_linear(entry_st, r);
- reg = scev_reg_for_linear(cur_func_state, r);
- if (!entry_reg || !reg || entry_reg->type == NOT_INIT || reg->type == NOT_INIT ||
- is_unbound_scev_reg(env, entry_st, r))
- continue;
-
- /*
- * A call in the loop can change the type: it invalidates packet pointers or
- * releases a reference. The bounds are the offsets of the pointer at the entry.
- * They say nothing about the value that is in the register now.
- */
- if (reg->type != entry_reg->type)
- continue;
-
- /*
- * bpf_compute_loop_iters() gives no range to a pointer to arena that moves, and
- * it does not look at dead registers. Set a range only where it would.
- */
- if (!bpf_is_widenable_reg_type(entry_reg) ||
- (coefs.zext32 && entry_reg->type != SCALAR_VALUE))
- continue;
-clamp:
- linear_bounds(scalar_of(entry_reg), max_header_count, &coefs, &bounds);
- /*
- * iv_range is the range at the header. After the latch @r can be the zero extension
- * of its header value. @reg went through that insn and has what holds of the range.
- */
- if (!behind &&
- is_iv_at_header(env, entry_st, insn_idx, entry->iv_reg, coefs.base_reg))
- iv_bounds(&bounds, &entry->iv_range);
- if (moved) {
- scalar_set_const(&off, moved);
- scalar_add(&bounds, &bounds, &off);
- }
- if (behind) {
- /* jumps on the way may have told more */
- bounds.r64 = cnum64_intersect(bounds.r64, reg->r64);
- bounds.r32 = cnum32_intersect(bounds.r32, reg->r32);
- if (type_is_pkt_pointer(reg->type) || cnum64_is_empty(bounds.r64) ||
- cnum32_is_empty(bounds.r32) ||
- (cnum64_is_subset(bounds.r64, reg->r64) &&
- cnum32_is_subset(bounds.r32, reg->r32)))
- continue;
- }
- if (log->level & BPF_LOG_LEVEL2) {
- bpf_log(log, "loop header at %d, clamping ", insn_idx);
- log_reg(env, r);
- bpf_log(log, " to ");
- log_bounds(env, &bounds);
- bpf_log(log, "\n");
- }
- if (!behind)
- scratch_widened_reg_id(env, reg);
- /*
- * The range of a packet pointer goes together with its id and with the offsets
- * that it had in the compare with pkt_end. Both are replaced here. What holds for
- * the new offsets is the range at the entry.
- */
- if (type_is_pkt_pointer(reg->type))
- reg->range = max(entry_reg->range, 0);
- err = bpf_set_reg_range(env, reg, &bounds);
- if (err)
- return err;
- mark_scev_reg_scratched(env, r);
- }
- return 0;
-}
diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c
index 1f620facd1b..ec9c310cf5d 100644
--- a/kernel/bpf/tnum.c
+++ b/kernel/bpf/tnum.c
@@ -317,14 +317,3 @@ u64 tnum_step(struct tnum t, u64 z)
inc = (filled + 1) & t.mask;
return t.value | inc;
}
-
-/*
- * Return the number of trailing bits known to be zero in a tnum.
- * Return 64 for a tnum representing zero.
- */
-u32 tnum_alignment(struct tnum a)
-{
- u64 v = a.value | a.mask;
-
- return v ? __ffs64(v) : 64;
-}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a4eca416dc2..f079435dd25 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6116,7 +6116,7 @@ static int check_buffer_access(struct bpf_verifier_env *env,
}
/* s = (u32)s. BPF architecture zero extends alu32 ops into 64-bit registers. */
-void scalar_zext32(struct scalar *s)
+static void scalar_zext32(struct scalar *s)
{
scalar_step_check_unsigned(s, 32);
s->var_off = tnum_subreg(s->var_off);
@@ -6230,15 +6230,6 @@ static void coerce_scalar_to_size_sx(struct scalar *s, int size)
set_sext64_default_val(s, size);
}
-/* @s = (uN)@s or (sN)@s. N is @bits: 8, 16 or 32. */
-void scalar_ext(struct scalar *s, u32 bits, bool sign)
-{
- if (sign)
- coerce_scalar_to_size_sx(s, bits / BITS_PER_BYTE);
- else
- coerce_scalar_to_size(s, bits / BITS_PER_BYTE);
-}
-
static void set_sext32_default_val(struct scalar *s, int size)
{
if (size == 1)
@@ -14790,55 +14781,6 @@ static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 of
return *kfunc.flags & KF_SPINLOCK_SAFE;
}
-/*
- * True if insn calls a helper/kfunc that requires one of its arguments to
- * be a stack pointer with a constant offset.
- */
-bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx)
-{
- const struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
- const struct bpf_func_proto *fn;
- struct bpf_kfunc_desc *desc;
- u32 *flags, btf_id;
- int i;
-
- if (bpf_helper_call(insn)) {
- if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)
- return false;
- } else if (bpf_pseudo_kfunc_call(insn)) {
- desc = find_kfunc_desc(env, insn->imm, insn->off);
- if (!desc)
- return false;
- fn = &desc->proto;
- } else {
- return false;
- }
-
- /* Both initialized and uninitialized stack dynptrs need a fixed offset. */
- for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++)
- if (arg_type_is_dynptr(fn->arg_type[i]))
- return true;
-
- /* vmlinux kfuncs only */
- if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0)
- return false;
- btf_id = insn->imm;
-
- flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog);
- if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)))
- return true;
-
- if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
- btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] ||
- btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
- btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] ||
- btf_id == special_kfunc_list[KF_bpf_local_irq_save] ||
- btf_id == special_kfunc_list[KF_bpf_local_irq_restore])
- return true;
-
- return false;
-}
-
static bool is_sync_callback_calling_kfunc(u32 btf_id)
{
return is_bpf_rbtree_add_kfunc(btf_id);
@@ -14857,12 +14799,6 @@ bool bpf_is_throw_kfunc(struct bpf_insn *insn)
insn->imm == special_kfunc_list[KF_bpf_throw];
}
-bool bpf_is_dynptr_slice_rdwr_kfunc(struct bpf_insn *insn)
-{
- return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
- insn->imm == special_kfunc_list[KF_bpf_dynptr_slice_rdwr];
-}
-
static bool is_bpf_wq_set_callback_kfunc(u32 btf_id)
{
return btf_id == special_kfunc_list[KF_bpf_wq_set_callback];
@@ -15624,15 +15560,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
regs[BPF_REG_0].buf = meta->slice_buf;
if (meta->slice_buf.size)
env->seen_stack_slice = true;
- /* No insn stores through a slice of 0 bytes */
- if (verifier_bug_if(meta->slice_buf.size && meta->arg_constant.value &&
- !type_is_rdonly_mem(regs[BPF_REG_0].type) &&
- !bpf_scev_in_slice_slots(env, meta->slice_buf.off,
- meta->slice_buf.size,
- meta->slice_buf.size ==
- meta->arg_constant.value),
- env, "slice buffer is not in scev slice_slots"))
- return -EFAULT;
} else {
return 0;
}
@@ -17941,29 +17868,6 @@ void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax)
reg->range = max_t(s64, reg->range - (smax - reg_smax(reg)), 0);
}
-int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
- const struct scalar *range)
-{
- *scalar_of(reg) = *range;
- reg_bounds_sync(reg); /* this should infer the tnum alignment */
- bpf_trim_pkt_range(reg, cnum64_smax(range->r64));
- return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range");
-}
-
-/*
- * Checks if pointer and packet metadata matches between two registers.
- * For use with bpf_reg_union().
- */
-bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b)
-{
- if (a->type != b->type)
- return false;
- if (a->type == SCALAR_VALUE)
- return true;
- return reg_fixed_part_equal(a, b) &&
- a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid;
-}
-
/*
* @s is what @old became after a trip around a cycle, @old included. The end
* of the range that moved is going to move again. Put it where it may come
@@ -18077,36 +17981,6 @@ void scalar_union(struct scalar *acc, const struct scalar *src)
*/
}
-/*
- * Extends acc's scalar range to include src's scalar range.
- * Caller must ensure that other kinds of metadata is compatible.
- */
-int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc,
- const struct bpf_reg_state *src)
-{
- u16 base, step;
-
- if (!bpf_reg_union_compatible(acc, src)) {
- verifier_bug(env, "union of incompatible registers");
- return -EFAULT;
- }
- acc->r64 = cnum64_union(acc->r64, src->r64);
- acc->r32 = cnum32_union(acc->r32, src->r32);
- acc->var_off = tnum_union(acc->var_off, src->var_off);
-
- /* Retain a common congruence if the bases agree modulo the gcd. */
- step = gcd(acc->step, src->step);
- base = acc->base % step;
- if (base != src->base % step) {
- scalar_step_reset(scalar_of(acc));
- } else {
- acc->base = base;
- acc->step = step;
- }
- reg_bounds_sync(acc);
- return reg_bounds_sanity_check(env, acc, "bpf_reg_union");
-}
-
/*
* There are no address spaces in Rust. The compiler doesn't cast a pointer that
* comes from the kernel. But it can't always tell, and then it casts the pointer
@@ -19098,162 +18972,12 @@ bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits)
return true;
}
-/*
- * The state of the SCEV register @r. As in check_stack_read_fixed_off() a fill of @bits bits from
- * byte @fill_off of the slot reads the low bits of the spill.
- */
-static struct bpf_reg_state *scev_reg_for_fill(struct bpf_func_state *frame, u32 r, u32 fill_off,
- u32 bits)
-{
- struct bpf_stack_state *ss;
-
- if (r < MAX_BPF_REG)
- return &frame->regs[r];
- if (r - MAX_BPF_REG >= bpf_stack_nr_slots(frame))
- return NULL;
- ss = bpf_stack_slot(frame, r - MAX_BPF_REG);
- if (!bpf_is_spilled_reg(ss) || bpf_spill_size(ss) * BITS_PER_BYTE < bits ||
- !bpf_stack_narrow_access_ok(fill_off, bits / BITS_PER_BYTE, bpf_spill_size(ss)))
- return NULL;
- return &ss->spilled_ptr;
-}
-
-/*
- * Scalar evolution promoted memory to the SCEV register *@regno, see struct env there, and names
- * its address at the jump at @idx. This state knows which memory it is: the frame that is returned,
- * its stack slot *@regno, *@fill_off counts from the start of the slot. The stack of a caller only:
- * other memory has no state, and a store through r10 is no def of memory for scalar evolution.
- * What holds the address is live: it was compared all the way from its def. Every place that
- * compares, joins or drops states cleans them of what is dead first, see flow_arrive().
- */
-static struct bpf_func_state *scev_mem_to_slot(struct bpf_verifier_env *env,
- struct bpf_verifier_state *vstate, int idx,
- u32 *regno, u32 *fill_off)
-{
- struct bpf_reg_state *ptr;
- u32 base_regno;
- s64 addr;
-
- if (!bpf_scev_mem_addr(env, idx, *regno, &base_regno, &addr))
- return NULL;
- ptr = scev_reg_for_fill(vstate->frame[vstate->curframe], base_regno, 0, 64);
- if (!ptr || ptr->type != PTR_TO_STACK || !tnum_is_const(ptr->var_off) ||
- ptr->frameno >= vstate->curframe)
- return NULL;
- addr += ptr->var_off.value + *fill_off;
- if (addr >= 0 || addr < -MAX_BPF_STACK_JIT)
- return NULL;
- *regno = MAX_BPF_REG + bpf_get_spi(addr);
- *fill_off = addr & (BPF_REG_SIZE - 1);
- return vstate->frame[ptr->frameno];
-}
-
-/*
- * p = buf + i; if (i > 100) return 0; *p = 0;
- * p is @dst_reg, x is @ptr_reg. Scalar evolution says that p == x + i. So the insns
- * 'p = x; p += i' are a no-op at run time, and a state that is sound before them is sound after
- * them: apply their transfer function. Type and object of p are taken from x as it is now, @off is
- * the offset of x + i; how p was computed does not matter.
- * The old p is replaced, not intersected: a state does not distinguish two values of one hash map,
- * and the intersection of an offset into the one with an offset into the other holds for neither.
- * It is replaced only if the new range is a subset of the old one. An empty range is a subset too:
- * the path is infeasible, p is left as it is.
- */
-static bool recompute_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *dst_reg,
- const struct bpf_reg_state *ptr_reg, const struct scalar *off)
-{
- struct bpf_reg_state sum = *ptr_reg;
-
- /* the cpu can speculate past the jump with an out-of-bounds index */
- if (!env->bypass_spec_v1)
- return false;
- /* 'p += i' assigns a new id to a packet pointer */
- if (ptr_reg->type == SCALAR_VALUE || type_is_pkt_pointer(ptr_reg->type) ||
- !bpf_is_widenable_reg_type(ptr_reg))
- return false;
- *scalar_of(&sum) = *off;
- __mark_scalar32_unbounded(scalar_of(&sum));
- reg_bounds_sync(&sum);
- if (reg_is_empty(&sum) || const_tnum_range_mismatch(scalar_of(&sum)) ||
- const_tnum_range_mismatch_32(scalar_of(&sum)))
- return false;
- /*
- * 'p += i' refuses a constant offset outside of this range, and a constant offset is put in
- * an int, see stack_slot_obj_get_spi().
- */
- if (reg_smin(&sum) <= -BPF_MAX_VAR_OFF || reg_smax(&sum) >= BPF_MAX_VAR_OFF)
- return false;
- if (!bpf_reg_union_compatible(dst_reg, &sum) || !cnum64_is_subset(dst_reg->r64, sum.r64))
- return false;
- *dst_reg = sum;
- return true;
-}
-
-/*
- * The jump at @idx refined the scalars that it tests. Scalar evolution evaluates the expression of
- * a register that is linked to them in this state. The register holds that value on every path to
- * the jump: intersect the two.
- */
-static void refine_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate,
- int idx, int next_idx)
-{
- struct bpf_func_state *frame = vstate->frame[vstate->curframe], *linked_frame;
- struct bpf_insn *insn = &env->prog->insnsi[idx];
- struct bpf_reg_state *reg, *base_reg;
- u32 pos, linked_regno, fill_off, bits;
- struct scev_value val;
- u16 refined_regs = 0;
-
- if (frame->regs[insn->dst_reg].type == SCALAR_VALUE)
- refined_regs |= BIT(insn->dst_reg);
- if (BPF_SRC(insn->code) == BPF_X && frame->regs[insn->src_reg].type == SCALAR_VALUE)
- refined_regs |= BIT(insn->src_reg);
- if (!refined_regs)
- return;
- for (pos = 0;
- bpf_scev_next_linked_reg(env, frame, idx, next_idx, refined_regs, &pos, &linked_regno,
- &fill_off, &val);
- pos++) {
- bits = val.bits;
- linked_frame = frame;
- if (linked_regno >= BPF_SCEV_REGS_NUM)
- linked_frame = scev_mem_to_slot(env, vstate, idx, &linked_regno, &fill_off);
- if (!linked_frame)
- continue;
- reg = scev_reg_for_fill(linked_frame, linked_regno, fill_off, bits);
- /*
- * The other half-slot of a 64-bit spill is dead: __clean_func_state() left
- * a 32-bit spill, the low 32 bits of the value.
- */
- if (!reg && bits == 64 &&
- bpf_stack_narrow_access_ok(fill_off, BPF_REG_SIZE / 2, BPF_REG_SIZE)) {
- bits = 32;
- reg = scev_reg_for_fill(linked_frame, linked_regno, fill_off, bits);
- }
- if (!reg || (val.ptr_reg < 0 && reg->type != SCALAR_VALUE))
- continue;
-
- if (val.ptr_reg >= 0) {
- base_reg = scev_reg_for_fill(frame, val.ptr_reg, 0, 64);
- if (!base_reg || bits < 64 ||
- !recompute_ptr(env, reg, base_reg, scalar_of(&val)))
- continue;
- } else if (!scalar_intersect(scalar_of(reg), scalar_of(&val), bits)) {
- continue;
- }
- if (linked_regno < MAX_BPF_REG)
- mark_reg_scratched(env, linked_regno);
- else if (linked_frame == frame)
- mark_stack_slot_scratched(env, linked_regno - MAX_BPF_REG);
- }
-}
-
/*
* One branch of the jump @insn is dead, the current state takes the other one. @reg1 and @reg2 are
* the operands as that branch refines them. They hold every value that takes it, whether the other
* branch is dead or not.
*/
-static int refine_live_branch(struct bpf_verifier_env *env, struct bpf_insn *insn, int next_idx,
+static int refine_live_branch(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_reg_state *reg1, struct bpf_reg_state *reg2)
{
struct bpf_reg_state *regs = cur_regs(env);
@@ -19277,9 +19001,6 @@ static int refine_live_branch(struct bpf_verifier_env *env, struct bpf_insn *ins
regs[insn->dst_reg] = *reg1;
if (BPF_SRC(insn->code) == BPF_X)
regs[insn->src_reg] = *reg2;
- if (!env->bpf_capable)
- return 0;
- refine_linked_regs(env, env->cur_state, env->insn_idx, next_idx);
return 0;
}
@@ -19375,8 +19096,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
if (err < 0)
return err;
}
- err = refine_live_branch(env, insn, *insn_idx + insn->off + 1, &env->true_reg1,
- &env->true_reg2);
+ err = refine_live_branch(env, insn, &env->true_reg1, &env->true_reg2);
if (err)
return err;
if (env->log.level & BPF_LOG_LEVEL)
@@ -19394,8 +19114,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
if (err < 0)
return err;
}
- err = refine_live_branch(env, insn, *insn_idx + 1, &env->false_reg1,
- &env->false_reg2);
+ err = refine_live_branch(env, insn, &env->false_reg1, &env->false_reg2);
if (err)
return err;
if (env->log.level & BPF_LOG_LEVEL)
@@ -19418,11 +19137,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
if (BPF_SRC(insn->code) == BPF_X)
other_branch_regs[insn->src_reg] = env->true_reg2;
- if (env->bpf_capable) {
- refine_linked_regs(env, this_branch, *insn_idx, *insn_idx + 1);
- refine_linked_regs(env, other_branch, *insn_idx, *insn_idx + insn->off + 1);
- }
-
/* if one pointer register is compared to another pointer
* register check if PTR_MAYBE_NULL could be lifted.
* E.g. register A - maybe null
@@ -20836,23 +20550,6 @@ static int loop_stack_pop(struct bpf_verifier_env *env)
return 0;
}
-static int maybe_clamp_scev_regs(struct bpf_verifier_env *env)
-{
- struct bpf_func_state *frame = cur_func(env);
- struct loop_stack_entry *entry;
-
- if (frame->loop_stack_cnt == 0) {
- verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx);
- return -EFAULT;
- }
-
- entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
- if (!entry->terminates)
- return 0;
-
- return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry, false);
-}
-
/* Can the main program return 0? */
static bool zero_is_valid_retval(struct bpf_verifier_env *env)
{
@@ -24610,10 +24307,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
- ret = bpf_find_latches(env, true);
- if (ret < 0)
- goto skip_full_check;
-
ret = bpf_compute_live_registers(env);
if (ret < 0)
goto skip_full_check;
@@ -24621,14 +24314,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (env->log.level & BPF_LOG_LEVEL2)
log_program(env);
- ret = bpf_init_scev(env);
- if (ret < 0)
- goto skip_full_check;
-
- ret = bpf_compute_scev(env);
- if (ret < 0)
- goto skip_full_check;
-
ret = bpf_flow_prepare(env);
if (ret < 0)
goto skip_full_check;
@@ -24776,7 +24461,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
kvfree(env->used_index.slots);
kvfree(env->kfunc_index.slots);
kvfree(env->insn_array_maps);
- bpf_free_scev(env);
bpf_flow_free(env);
bpf_stack_liveness_free(env);
kvfree(env->cfg.postorder_nums);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 04/49] bpf: Delete scalar evolution
2026-11-20 7:41 ` [PATCH bpf-next 04/49] bpf: Delete scalar evolution Alexei Starovoitov
@ 2026-11-20 8:03 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:03 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [High] bpf: call to undeclared bpf_add_cost() and BPF_COST_SCAN in
find_latch()
--
commit e5e45ce256ee98d6b03c38ed44f64ae0c972a296
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Delete scalar evolution
This commit removes the scalar evolution pre-pass (scev.c), its callers,
and helpers that were only used by it, and simplifies loop exit and latch
tracking to record only the first exit target and latch.
> diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
> index 1101976850515..8a63bb1799a78 100644
> --- a/kernel/bpf/loops.c
> +++ b/kernel/bpf/loops.c
[ ... ]
> @@ -340,100 +291,23 @@ int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx)
> }
>
> /*
> - * Collect conditional jumps within n's loop that dominate 'n' and leave
> - * the loop on one edge, see bpf_edge_leaves_loop(), nearest first.
> - * A jump that cannot count the iterations takes no place, see bpf_scev_jump_counts().
> + * The backedge->from and its latch can reside in an inner loop of @h, e.g.:
> + *
> + * 1: r7 += 1;
> + * 2: r6 += 1;
> + * if r7 == 2 goto 1b;
> + * if r6 < 2 goto 2b;
> + *
> + * For now, let's assume that latches are unknown for such cases.
> + * (GCC/LLVM handle this by inserting artificial cfg nodes).
> */
> -static int find_dominating_conditions(struct bpf_verifier_env *env, int n,
> - struct bpf_backedge *backedge)
> +static int find_latch(struct bpf_verifier_env *env, int h, int n)
> {
> - struct bpf_insn *insns = env->prog->insnsi;
> - bool t_exits_loop, f_exits_loop;
> - int *idoms = env->idoms;
> - int n_loop, counts;
> -
> - n_loop = bpf_loop_at_index(env, n);
> - if (idoms_intersect(env, n, n_loop) != n_loop)
> - return 0;
> - while (n >= 0) {
> - if (is_cond_jmp_insn(&insns[n]) && bpf_loop_at_index(env, n) == n_loop) {
> - t_exits_loop = bpf_edge_leaves_loop(env, n_loop, n, n + insns[n].off + 1);
> - f_exits_loop = bpf_edge_leaves_loop(env, n_loop, n, n + 1);
> - /* latches[0] is the latch: it has to leave the loop itself */
> - if ((t_exits_loop || f_exits_loop) &&
> - (backedge->latches_cnt || bpf_jump_leaves_loop(env, n_loop, n))) {
> - counts = !backedge->latches_cnt ? 1 :
> - bpf_scev_jump_counts(env, n_loop, n, t_exits_loop);
> - if (counts < 0)
> - return counts;
> - if (!counts)
> - goto next;
> - if (backedge->latches_cnt == MAX_LATCHES)
> - return 0;
> - if (t_exits_loop)
> - backedge->taken_leaves |= BIT(backedge->latches_cnt);
> - backedge->latches[backedge->latches_cnt++] = n;
> - }
> - }
> -next:
> - if (n == n_loop)
> - break;
> - n = idoms[n];
> - }
> - return 0;
> -}
> -
> -/*
> - * Without @flags: the jumps that leave a loop themselves. That tells where the verifier may add
> - * may_goto. It takes the constants that hold with these edges to follow a flag.
> - * Once more after scalar evolution, which knows the jumps that count. The flags that were found
> - * are remembered.
> - */
> -int bpf_find_latches(struct bpf_verifier_env *env, bool flags)
> -{
> - struct bpf_insn_aux_data *aux = env->insn_aux_data;
> - struct bpf_backedge *backedge;
> - struct bpf_loop *loop;
> - int i, j, k, err;
> -
> - if (flags) {
> - err = bpf_compute_leave_conds(env);
> - if (err)
> - return err;
> - }
> - for (i = 0; i < env->prog->len; i++) {
> - loop = aux[i].loop;
> - if (!loop)
> - continue;
> - for (j = 0; j < loop->backedges_cnt; j++) {
> - backedge = &loop->backedges[j];
> - backedge->latches_cnt = 0;
> - backedge->taken_leaves = 0;
> - /*
> - * In theory, the backedge->from and it's latch can reside in
> - * an inner loop of `i`, e.g.:
> - *
> - * 1: r7 += 1;
> - * 2: r6 += 1;
> - * if r7 == 2 goto 1b;
> - * if r6 < 2 goto 2b;
> - *
> - * For now, let's assume that latches are unknown for such cases.
> - * (GCC/LLVM handle this by inserting artificial cfg nodes).
> - */
> - if (bpf_loop_at_index(env, backedge->from) != i)
> - continue;
> - err = find_dominating_conditions(env, backedge->from, backedge);
> - if (err)
> - return err;
> - for (k = 1; flags && k < backedge->latches_cnt; k++)
> - if ((env->log.level & BPF_LOG_LEVEL2) &&
> - !bpf_jump_leaves_loop(env, i, backedge->latches[k]))
> - bpf_log(&env->log, "loop at %d, exit by way of a flag at %d\n",
> - i, backedge->latches[k]);
> - }
> - }
> - return 0;
> + if (bpf_loop_at_index(env, n) != h || idoms_intersect(env, n, h) != h)
> + return -1;
> + for (; n >= 0 && !bpf_insn_leaves_loop(env, h, n); n = n == h ? -1 : env->idoms[n])
> + bpf_add_cost(env, 1, BPF_COST_SCAN);
> + return n;
> }
[Severity: High]
Does find_latch() break the build at this commit by calling bpf_add_cost()
with BPF_COST_SCAN before they are defined? Those symbols are only
introduced in the following patch in the series, commit "bpf: Limit the
cost and the memory of a load", so kernel/bpf/loops.o fails to compile
with CONFIG_BPF_SYSCALL=y here, breaking bisection.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=4
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 04/49] bpf: Delete scalar evolution
2026-11-20 7:41 ` [PATCH bpf-next 04/49] bpf: Delete scalar evolution Alexei Starovoitov
2026-11-20 8:03 ` sashiko-bot
@ 2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 5165 bytes --]
> bpf: Delete scalar evolution
TL;DR:
- loops.c uses bpf_add_cost()/BPF_COST_SCAN, defined only in the next commit,
so this commit may not build (bisection)
- verifier_loop_hierarchy and verifier_scev selftests likely fail on removed
log lines
- unused helpers remain despite commit message
> diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
> --- a/kernel/bpf/loops.c
> +++ b/kernel/bpf/loops.c
[ ... ]
> +static int find_latch(struct bpf_verifier_env *env, int h, int n)
> {
> [ ... ]
> + if (bpf_loop_at_index(env, n) != h || idoms_intersect(env, n, h) != h)
> + return -1;
> + for (; n >= 0 && !bpf_insn_leaves_loop(env, h, n); n = n == h ? -1 : env->idoms[n])
> + bpf_add_cost(env, 1, BPF_COST_SCAN);
> + return n;
> }
Are bpf_add_cost() and BPF_COST_SCAN defined anywhere at this commit?
A git grep finds no definition of either. The only other user is the
bpf_add_cost() call in kernel/bpf/liveness.c added by cd07fb71e4d5 ("bpf:
Make stack liveness faster"). loops.o is built whenever CONFIG_BPF_SYSCALL
is set, so this looks like an implicit function declaration plus an
undeclared identifier, and loops.c would not compile.
The definitions arrive in the next commit, ecbd98bcc45a ("bpf: Limit the
cost and the memory of a load"), which adds BPF_COST_SCAN and a static
inline bpf_add_cost() to include/linux/bpf_verifier.h. That commit also
adds bpf_add_cost() calls to idoms_intersect(), assign_header() and
bpf_is_nested_loop(), but not to find_latch().
Should the bpf_add_cost() call in find_latch() move to that commit, or
should the cost definitions come before this one? As it stands this commit
adds a second file that does not build, so the series cannot be bisected
until ecbd98bcc45a.
[ ... ]
> @@ bpf_compute_loops()
> - for (j = 0; j < loop->exits_cnt; j++)
> - bpf_log(log, " exit from %d to %d\n",
> - loop->exits[j].from, loop->exits[j].to);
> + if (loop->exit_to >= 0)
> + bpf_log(log, " first exit to %d\n", loop->exit_to);
The BPF_LOG_LEVEL2 loop dump no longer prints one " exit from %d to %d"
line per exit, and it drops ", dominating exits at ...", ", too many exits"
and ", too many exits of a nested loop".
tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c is still
registered in prog_tests/verifier.c and still expects the old lines with
__msg_next():
loop_nested_siblings_common_ancestors:
__msg_next(" backedge from 10, latch at 10")
__msg_next(" exit from 10 to 11")
...
loop_outer_backedge_from_inner:
__msg_next(" backedge from 4{{$}}")
__msg_next(" exit from 5 to 7")
...
"exit from 5 to 7" is not a substring of "first exit to 7", so both
subtests should fail. loop_nested_siblings_common_ancestors also exists to
check that edge 6->7 is "logged only as an exit from loop at 5", which the
new log cannot show. progs/verifier_scev.c also matches "too many exits"
and "dominating exits at" lines that are removed here.
Do these selftests need to be updated or reworked in this commit? None of
the commits up to a50a112fb866 touches them.
[ ... ]
> diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c
> deleted file mode 100644
[ ... ]
Deleting scev.c removes the only code that prints the "scev at header N:",
" scev at latch N:" and expression-dump lines at log level 2.
tools/testing/selftests/bpf/progs/verifier_scev.c still has 39 __msg() and
__msg_next() expectations for them, and prog_tests/verifier.c still has:
void test_verifier_scev(void) { RUN(verifier_scev); }
For example:
__success
__log_level(2)
__msg_next("scev at header 1:")
__msg_next(" r0=(+ r0 1) / (linear r0 1)")
__naked void simple_loop1(void)
Won't test_progs -t verifier_scev fail on these? verifier_scev is not in
ci/vmtest/configs/DENYLIST*, and no commit through a50a112fb866 touches the
file. Should verifier_scev.c and its RUN() entry be removed or rewritten
in this commit, given that it deletes the feature they test?
[ ... ]
> Delete scev.c, its callers, and the helpers that only it uses.
Is this accurate for bpf_mark_reg_known_scalar() and scalar_union()?
Both are non-static in kernel/bpf/verifier.c, both have prototypes in
include/linux/bpf_verifier.h, and in the parent their only callers were in
scev.c:
bpf_mark_reg_known_scalar(&scev->zero_reg, 0); (bpf_init_scev())
scalar_union(&iters->iv_range, &tmp.iv_range);
Neither has a caller after this commit, and neither is used again through
a50a112fb866. The compiler does not warn because they are not static.
scalar_union() is also the only remaining user of gcd(), so the
#include <linux/gcd.h> in verifier.c appears to be left only for it.
scalar_intersect() and bpf_same_memory_origin() also lose their callers
here, but later patches in the series use them again.
Should bpf_mark_reg_known_scalar() and scalar_union() be deleted together
with their prototypes, or should the commit message be reworded?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (3 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 04/49] bpf: Delete scalar evolution Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:00 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 06/49] bpf: Record what each insn reads and writes Alexei Starovoitov
` (43 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The caps on states are gone. Something has to stop a load that takes
too long. A cap on one kind of work is a poor tool. It rejects a valid
program that needs much of that kind and little else.
Use two limits for the whole load, one on cost and one on memory:
weight work
1 one step of a scan
4 copy an xreg
16 clean, compare or join an xreg
128 check an insn
512 make, order and free a state
BPF_MAX_COST 1 << 33, BPF_MAX_MEM 2 GB, 1/32 of both without CAP_BPF
The cost counts work, not time. So the verdict does not depend on the
speed or the load of the machine. bpf_check_limits() calls
bpf_must_abort() too, and replaces it in the loops that add cost. A
thaw does not take an abort back, see env->aborted.
A state knows its part of env->mem, so copy and free take env.
The walks up a deep nest of loops are quadratic. Move the check of
LOOP_STACK_SIZE to bpf_compute_loops(), before these walks.
Stack liveness skips the registers and slots that hold no argument.
It copies the stack only for a call that passes a pointer to a stack.
Stack liveness has a cap on one kind of work too. It rejects a program
after 10000 calls of functions:
for 12000 times:
call small_func liveness analysis exceeded complexity limit
The program is valid and cheap. Remove the cap. Each call site has an
instance, and the hash table of instances has 256 buckets. So 12000
instances make long chains. Use an array with the call site as the
index. An entry has one instance for each depth:
by_callsite[insn] -> depth 2 -> depth 1 -> NULL
A list has all instances, for the loops over them.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf.h | 4 +-
include/linux/bpf_verifier.h | 49 ++++++++-
kernel/bpf/const_fold.c | 15 +--
kernel/bpf/flow.c | 39 +++++---
kernel/bpf/liveness.c | 187 +++++++++++++++++++----------------
kernel/bpf/loops.c | 42 +++++++-
kernel/bpf/states.c | 1 +
kernel/bpf/verifier.c | 62 ++++++++----
8 files changed, 267 insertions(+), 132 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a5ad08b7937..8b985485f57 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2631,8 +2631,8 @@ struct bpf_array {
int bpf_array_get_next_key(struct bpf_map *map, void *key, void *next_key);
/*
- * the longest program. The verifier limits the number of states, not of
- * processed insns, see BPF_FLOW_MAX_STATES.
+ * the longest program. The verifier limits the cost of a load, not the number
+ * of processed insns, see BPF_MAX_COST.
*/
#define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
#define MAX_TAIL_CALL_CNT 33
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 894b8890f85..ed271135fe5 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -513,6 +513,8 @@ struct bpf_verifier_state {
/* The insn that the state comes from, -1 if no instructions have been executed yet. */
u32 last_insn_idx;
+ /* its part of env->mem */
+ u32 bytes;
};
/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */
@@ -1125,6 +1127,11 @@ struct bpf_verifier_env {
bool strict_alignment; /* perform strict pointer alignment checks */
bool test_reg_invariants; /* fail verification on register invariants violations */
bool uses_arena_sp; /* the program has insns with BPF_REG_ARENA_SP */
+ u64 cost, mem; /* see bpf_check_limits() */
+ u64 max_cost, max_mem;
+ u64 pass_max_cost; /* see bpf_undo_precision() */
+ bool aborted; /* a thaw does not take it back */
+ bool over_limit; /* see bpf_check_limits() */
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_loop_gets_guards() */
u32 loop_entry_cnt; /* see loop_stack_entry.entry_id, callback_entry_id */
@@ -1422,7 +1429,34 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
struct bpf_verifier_state *cur);
+/*
+ * The two limits of the verifier, for the whole load. The unit of cost is one step of a scan,
+ * about a dozen insns of the CPU. Memory is the bytes of states and tables at a time.
+ */
+#define BPF_COST_SCAN 1 /* one step of a scan */
+#define BPF_COST_COPY 4 /* copy an xreg, or look at its type */
+#define BPF_COST_XREG 16 /* clean, compare or join an xreg */
+#define BPF_COST_INSN 128 /* check an insn */
+#define BPF_COST_STATE 512 /* make, order and free a state */
+#define BPF_MAX_COST (1ULL << 33)
+#define BPF_MAX_MEM (1ULL << 31)
+/* Without CAP_BPF programs are small, and their loads wait for each other */
+#define BPF_LIMITS_SHIFT_UNPRIV 5
+
/* flow.c */
+int bpf_check_limits(struct bpf_verifier_env *env);
+
+static inline void bpf_add_cost(struct bpf_verifier_env *env, u64 cnt, u32 weight)
+{
+ env->cost += cnt * weight;
+}
+
+static inline int bpf_add_cost_and_check(struct bpf_verifier_env *env, u64 cnt, u32 weight)
+{
+ bpf_add_cost(env, cnt, weight);
+ return bpf_check_limits(env);
+}
+
int bpf_flow_prepare(struct bpf_verifier_env *env);
void bpf_flow_reset(struct bpf_verifier_env *env);
void bpf_flow_free(struct bpf_verifier_env *env);
@@ -1439,9 +1473,11 @@ bool bpf_insn_is_iter_next(struct bpf_verifier_env *env, struct bpf_insn *insn);
bool bpf_insn_leaves_loop(struct bpf_verifier_env *env, int h, int n);
int bpf_insert_recursion_checks(struct bpf_verifier_env *env);
-int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
+int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *dst_state,
const struct bpf_verifier_state *src);
-void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);
+void bpf_charge_state(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+void bpf_free_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *state,
+ bool free_self);
void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
struct bpf_reg_state *reg);
void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm);
@@ -2084,4 +2120,13 @@ int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
+static inline u32 bpf_xreg_cnt(const struct bpf_verifier_state *st)
+{
+ u32 fr, nr = 0;
+
+ for (fr = 0; fr <= st->curframe; fr++)
+ nr += MAX_BPF_REG + bpf_stack_nr_slots(st->frame[fr]);
+ return nr;
+}
+
#endif /* _LINUX_BPF_VERIFIER_H */
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index bf81bd54232..50179edbf34 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -250,9 +250,10 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
struct const_arg_info ci_out[MAX_BPF_REG];
struct bpf_iarray *succ;
bool changed;
- int i, r, t;
+ int i, r, t, err;
/* kvzalloc zeroes memory, so all entries start as CONST_ARG_UNVISITED (0) */
+ env->mem += insn_cnt * sizeof(*ci_in);
ci_in = kvzalloc_objs(*ci_in, insn_cnt, BPF_VERIFIER_GFP);
if (!ci_in)
return -ENOMEM;
@@ -278,10 +279,9 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
}
redo:
- if (bpf_must_abort()) {
- kvfree(ci_in);
- return -EINTR;
- }
+ err = bpf_add_cost_and_check(env, env->cfg.cur_postorder * 2ULL, BPF_COST_XREG);
+ if (err)
+ goto out;
changed = false;
for (i = env->cfg.cur_postorder - 1; i >= 0; i--) {
int idx = env->cfg.insn_postorder[i];
@@ -334,9 +334,10 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
aux->const_reg_map_mask = map_mask;
aux->const_reg_subprog_mask = subprog_mask;
}
-
+out:
+ env->mem -= insn_cnt * sizeof(*ci_in);
kvfree(ci_in);
- return 0;
+ return err;
}
static int eval_const_branch(u8 opcode, u64 dst_val, u64 src_val)
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index ac5e7c26c2a..cbbc663acac 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -215,9 +215,11 @@ static bool flow_loop_is_bounded(struct bpf_verifier_env *env, int h)
if (loop->backedges_overflow || loop->irreducible)
return false;
for (i = 0; i < loop->backedges_cnt; i++) {
- for (n = loop->backedges[i].from; n >= 0 && n != h; n = env->idoms[n])
+ for (n = loop->backedges[i].from; n >= 0 && n != h; n = env->idoms[n]) {
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
if (flow_is_bound(env, h, n))
break;
+ }
if (n < 0 || !flow_is_bound(env, h, n))
return false;
}
@@ -264,7 +266,7 @@ bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env)
int bpf_flow_place_guards(struct bpf_verifier_env *env)
{
struct bpf_insn_aux_data *aux = env->insn_aux_data;
- int len = env->prog->len, h, i, depth, ret = 0;
+ int len = env->prog->len, h, i, ret = 0, err = 0;
struct bpf_loop *loop;
u32 *seen;
@@ -274,27 +276,18 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
for (i = 0; i < len; i++) {
aux[i].not_stopped = -1;
- depth = 0;
- for (h = bpf_loop_at_index(env, i); h >= 0; h = aux[h].loop_header) {
- /* the main pass doesn't get that deep */
- if (++depth > LOOP_STACK_SIZE) {
- verbose(env, "Too many nested loops at %d\n", i);
- kvfree(seen);
- return -E2BIG;
- }
+ for (h = bpf_loop_at_index(env, i); h >= 0; h = aux[h].loop_header)
aux[h].loop->nr_insns++;
- }
}
for (h = 0; h < len; h++) {
loop = aux[h].loop;
if (!loop)
continue;
- /* a deep nest of loops takes long */
- if (bpf_must_abort()) {
- ret = -EINTR;
+ err = bpf_add_cost_and_check(env, (u64)loop->nr_insns * loop->backedges_cnt,
+ BPF_COST_COPY);
+ if (err)
break;
- }
loop->rt_bounded = flow_loop_is_bounded(env, h);
/* also when it's bounded: the main pass may find that it's not */
flow_place_guards(env, h, seen);
@@ -302,5 +295,19 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
ret = 1;
}
kvfree(seen);
- return ret;
+ return err ?: ret;
+}
+
+int bpf_check_limits(struct bpf_verifier_env *env)
+{
+ env->aborted |= bpf_must_abort();
+ if (env->aborted)
+ return -EINTR;
+ if (env->cost <= min(env->max_cost, env->pass_max_cost) && env->mem <= env->max_mem &&
+ !env->over_limit)
+ return 0;
+ verbose(env, "BPF program is too complex: cost %llu of %llu, memory %llu of %llu\n",
+ env->cost, min(env->max_cost, env->pass_max_cost), env->mem, env->max_mem);
+ env->over_limit = true;
+ return -E2BIG;
}
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 57c37ebae00..4a58c5f4db8 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -3,8 +3,6 @@
#include <linux/bpf_verifier.h>
#include <linux/btf.h>
-#include <linux/hashtable.h>
-#include <linux/jhash.h>
#include <linux/slab.h>
#include <linux/sort.h>
@@ -54,7 +52,9 @@ struct frame_masks {
* Marks are tracked for each frame up to @depth.
*/
struct func_instance {
- struct hlist_node hl_node;
+ struct list_head node; /* all instances */
+ struct func_instance *next; /* same callsite, other depth */
+ struct bpf_verifier_env *env;
u32 callsite; /* call insn that invoked this subprog (subprog_start for depth 0) */
u32 depth; /* call depth (0 = entry subprog) */
u32 subprog; /* subprog index */
@@ -75,36 +75,25 @@ struct live_stack_query {
struct arg_track;
struct bpf_liveness {
- DECLARE_HASHTABLE(func_instances, 8); /* maps (depth, callsite) to func_instance */
- struct live_stack_query live_stack_query; /* cache to avoid repetitive ht lookups */
- u32 subprog_calls; /* analyze_subprog() invocations */
+ struct list_head func_instances;
+ struct live_stack_query live_stack_query; /* cache to avoid repetitive lookups */
/* MAX_AT_TRACK_REGS for every depth: analyze_subprog() calls itself, keep its frame small */
struct arg_track *callee_args;
+ struct func_instance *by_callsite[];
};
/*
- * Hash/compare key for func_instance: (depth, callsite).
* For depth == 0 (entry subprog), @callsite is the subprog start insn.
* For depth > 0, @callsite is the call instruction index that invoked the subprog.
*/
-static u32 instance_hash(u32 callsite, u32 depth)
-{
- u32 key[2] = { depth, callsite };
-
- return jhash2(key, 2, 0);
-}
-
static struct func_instance *find_instance(struct bpf_verifier_env *env,
u32 callsite, u32 depth)
{
- struct bpf_liveness *liveness = env->liveness;
- struct func_instance *f;
- u32 key = instance_hash(callsite, depth);
+ struct func_instance *f = env->liveness->by_callsite[callsite];
- hash_for_each_possible(liveness->func_instances, f, hl_node, key)
- if (f->depth == depth && f->callsite == callsite)
- return f;
- return NULL;
+ while (f && f->depth != depth)
+ f = f->next;
+ return f;
}
static struct func_instance *call_instance(struct bpf_verifier_env *env,
@@ -115,7 +104,6 @@ static struct func_instance *call_instance(struct bpf_verifier_env *env,
u32 subprog_start = env->subprog_info[subprog].start;
u32 lookup_key = depth > 0 ? callsite : subprog_start;
struct func_instance *f;
- u32 hash;
f = find_instance(env, lookup_key, depth);
if (f)
@@ -124,13 +112,16 @@ static struct func_instance *call_instance(struct bpf_verifier_env *env,
f = kvzalloc_obj(*f, BPF_VERIFIER_GFP);
if (!f)
return ERR_PTR(-ENOMEM);
+ f->env = env;
f->callsite = lookup_key;
f->depth = depth;
f->subprog = subprog;
f->subprog_start = subprog_start;
f->insn_cnt = (env->subprog_info + subprog + 1)->start - subprog_start;
- hash = instance_hash(lookup_key, depth);
- hash_add(env->liveness->func_instances, &f->hl_node, hash);
+ f->next = env->liveness->by_callsite[lookup_key];
+ env->liveness->by_callsite[lookup_key] = f;
+ list_add(&f->node, &env->liveness->func_instances);
+ env->mem += sizeof(*f);
return f;
}
@@ -155,22 +146,23 @@ static struct func_instance *lookup_instance(struct bpf_verifier_env *env,
int bpf_stack_liveness_init(struct bpf_verifier_env *env)
{
- env->liveness = kvzalloc_obj(*env->liveness, BPF_VERIFIER_GFP);
+ env->liveness = kvzalloc_flex(*env->liveness, by_callsite, env->prog->len,
+ BPF_VERIFIER_GFP);
if (!env->liveness)
return -ENOMEM;
- hash_init(env->liveness->func_instances);
+ INIT_LIST_HEAD(&env->liveness->func_instances);
+ env->mem += env->prog->len * sizeof(*env->liveness->by_callsite);
return 0;
}
void bpf_stack_liveness_free(struct bpf_verifier_env *env)
{
- struct func_instance *instance;
- struct hlist_node *tmp;
- int bkt, i;
+ struct func_instance *instance, *tmp;
+ int i;
if (!env->liveness)
return;
- hash_for_each_safe(env->liveness->func_instances, bkt, tmp, instance, hl_node) {
+ list_for_each_entry_safe(instance, tmp, &env->liveness->func_instances, node) {
for (i = 0; i <= instance->depth; i++)
kvfree(instance->frames[i]);
kvfree(instance);
@@ -213,12 +205,18 @@ static struct frame_masks *widen_frame_masks(struct func_instance *instance,
{
struct frame_masks *old = instance->frames[frame], *new;
u32 i, kind;
+ int err;
if (old && old->words >= words)
return old;
+ instance->env->mem += frame_mask_words(instance, words - (old ? old->words : 0)) *
+ sizeof(long);
+ err = bpf_check_limits(instance->env);
+ if (err)
+ return ERR_PTR(err);
new = kvzalloc_flex(*new, bits, frame_mask_words(instance, words), BPF_VERIFIER_GFP);
if (!new)
- return NULL;
+ return ERR_PTR(-ENOMEM);
new->words = words;
if (old) {
for (i = 0; i < instance->insn_cnt; i++)
@@ -249,8 +247,8 @@ static int mark_stack_range(struct func_instance *instance, u32 frame, u32 insn_
if (lo > hi)
return 0;
fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(hi + 1));
- if (!fm)
- return -ENOMEM;
+ if (IS_ERR(fm))
+ return PTR_ERR(fm);
bitmap_set(rel_mask(fm, relative_idx(instance, insn_idx), kind), lo, hi - lo + 1);
return 0;
}
@@ -314,8 +312,8 @@ static int mark_stack_mask(struct func_instance *instance, u32 frame, u32 insn_i
if (last == nbits)
return 0;
fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1));
- if (!fm)
- return -ENOMEM;
+ if (IS_ERR(fm))
+ return PTR_ERR(fm);
dst = rel_mask(fm, relative_idx(instance, insn_idx), kind);
/* @src has no bits set past @last, hence none past @fm->words either */
src_words = min(src_words, fm->words);
@@ -462,6 +460,7 @@ static int update_instance(struct bpf_verifier_env *env, struct func_instance *i
int *insn_postorder = env->cfg.insn_postorder;
struct bpf_subprog_info *subprog;
bool changed;
+ int err;
instance->must_write_initialized = true;
subprog = &env->subprog_info[instance->subprog];
@@ -469,12 +468,15 @@ static int update_instance(struct bpf_verifier_env *env, struct func_instance *i
po_end = (subprog + 1)->postorder_start;
/* repeat until fixed point is reached */
do {
- if (bpf_must_abort())
- return -EINTR;
changed = false;
for (frame = 0; frame <= instance->depth; frame++) {
if (!instance->frames[frame])
continue;
+ err = bpf_add_cost_and_check(env, (po_end - po_start) *
+ (4ULL + instance->frames[frame]->words),
+ BPF_COST_XREG);
+ if (err)
+ return err;
for (i = po_start; i < po_end; i++) {
changed |= update_insn(env, instance, frame, insn_postorder[i],
@@ -779,9 +781,9 @@ static void compute_may_write_masks(struct bpf_verifier_env *env)
struct func_instance *instance;
struct frame_masks *fm;
u32 nbits;
- int bkt, i;
+ int i;
- hash_for_each(liveness->func_instances, bkt, instance, hl_node) {
+ list_for_each_entry(instance, &liveness->func_instances, node) {
fm = instance->frames[instance->depth];
if (!fm)
continue;
@@ -800,16 +802,16 @@ static int print_instances(struct bpf_verifier_env *env)
{
struct func_instance *instance, **sorted_instances;
struct bpf_liveness *liveness = env->liveness;
- int i, bkt, cnt;
+ int i, cnt;
cnt = 0;
- hash_for_each(liveness->func_instances, bkt, instance, hl_node)
+ list_for_each_entry(instance, &liveness->func_instances, node)
cnt++;
sorted_instances = kvmalloc_objs(*sorted_instances, cnt, BPF_VERIFIER_GFP);
if (!sorted_instances)
return -ENOMEM;
cnt = 0;
- hash_for_each(liveness->func_instances, bkt, instance, hl_node)
+ list_for_each_entry(instance, &liveness->func_instances, node)
sorted_instances[cnt++] = instance;
sort(sorted_instances, cnt, sizeof(*sorted_instances), cmp_instances, NULL);
for (i = 0; i < cnt; i++)
@@ -1058,9 +1060,13 @@ static struct arg_track __arg_track_join(struct arg_track a, struct arg_track b)
static bool arg_track_join(struct bpf_verifier_env *env, int idx, int target, int r,
struct arg_track *in, struct arg_track out)
{
- struct arg_track old = *in;
- struct arg_track new_val = __arg_track_join(old, out);
+ struct arg_track old, new_val;
+ /* most registers and slots hold no argument */
+ if (out.frame == ARG_NONE && in->frame == ARG_NONE)
+ return false;
+ old = *in;
+ new_val = __arg_track_join(old, out);
if (arg_track_eq(&new_val, &old))
return false;
@@ -1090,7 +1096,7 @@ static bool arg_track_join_all(struct bpf_verifier_env *env, int idx, int target
const struct arg_track *out, const struct arg_track *stack_out,
u32 nslots)
{
- bool changed = false;
+ bool changed = false, same;
int r;
if (!arg_is_visited(&in[BPF_REG_FP])) {
@@ -2044,6 +2050,18 @@ static void record_stack_ptrs(struct bpf_verifier_env *env, int idx, struct arg_
env->insn_aux_data[idx].stack_ptrs |= mask;
}
+/* Return true if any of R1-R5 or stack args is derived from a frame pointer. */
+static bool has_fp_args(struct arg_track *args)
+{
+ for (int r = BPF_REG_1; r <= BPF_REG_5; r++)
+ if (arg_is_fp(&args[r]))
+ return true;
+ for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++)
+ if (arg_is_fp(&args[MAX_BPF_REG + r]))
+ return true;
+ return false;
+}
+
/*
* Compute arg tracking dataflow for a single subprog.
* Runs forward fixed-point with arg_track_xfer(), then records
@@ -2077,9 +2095,15 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS];
struct arg_track unvisited = { .frame = ARG_UNVISITED };
struct arg_track none = { .frame = ARG_NONE };
- int i, p, r, err = -ENOMEM;
+ size_t bytes = (size_t)len * nslots * sizeof(*at_stack_in);
unsigned long *changed = NULL;
+ int i, p, r, err;
+ env->mem += bytes;
+ err = bpf_check_limits(env);
+ if (err)
+ goto err_free;
+ err = -ENOMEM;
at_in = kvmalloc_objs(*at_in, len, BPF_VERIFIER_GFP);
if (!at_in)
goto err_free;
@@ -2125,6 +2149,8 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
/* Forward fixed-point iteration in reverse post order, over insns whose input changed */
__set_bit(0, changed);
+ /* the first visit of each insn, see arg_track_join_all() for the others */
+ bpf_add_cost(env, (u64)len * (MAX_AT_TRACK_REGS + nslots), BPF_COST_XREG);
redo:
for (p = po_end - 1; p >= po_start; p--) {
int idx = env->cfg.insn_postorder[p];
@@ -2133,8 +2159,8 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
struct bpf_iarray *succ;
int ti;
- err = -EINTR;
- if (bpf_must_abort())
+ err = bpf_add_cost_and_check(env, 1, BPF_COST_SCAN);
+ if (err)
goto err_free;
if (!__test_and_clear_bit(i, changed))
continue;
@@ -2187,16 +2213,18 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
goto err_free;
}
- if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) {
- struct spill_snapshot *snap;
+ if ((bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) &&
+ has_fp_args(at_in[i])) {
+ struct spill_snapshot *snap = env->callsite_at_stack[idx];
- kvfree(env->callsite_at_stack[idx]);
- env->callsite_at_stack[idx] = kvmalloc_flex(*snap, at, nslots, BPF_VERIFIER_GFP);
- snap = env->callsite_at_stack[idx];
if (!snap) {
- err = -ENOMEM;
- goto err_free;
+ snap = kmalloc_flex(*snap, at, nslots, BPF_VERIFIER_GFP);
+ env->callsite_at_stack[idx] = snap;
+ env->mem += ksize(snap);
}
+ err = snap ? bpf_check_limits(env) : -ENOMEM;
+ if (err)
+ goto err_free;
snap->slots = nslots;
memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at));
}
@@ -2211,6 +2239,7 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
err = 0;
err_free:
+ env->mem -= bytes;
kvfree(changed);
kvfree(at_stack_out);
kvfree(at_stack_in);
@@ -2218,18 +2247,6 @@ static noinline_for_stack int compute_subprog_args(struct bpf_verifier_env *env,
return err;
}
-/* Return true if any of R1-R5 or stack args is derived from a frame pointer. */
-static bool has_fp_args(struct arg_track *args)
-{
- for (int r = BPF_REG_1; r <= BPF_REG_5; r++)
- if (arg_is_fp(&args[r]))
- return true;
- for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++)
- if (arg_is_fp(&args[MAX_BPF_REG + r]))
- return true;
- return false;
-}
-
/*
* Merge a freshly analyzed instance into the original.
* may_read, may_fill: union (any pass might read the slot).
@@ -2267,8 +2284,8 @@ static int merge_instances(struct func_instance *dst, struct func_instance *src)
continue;
}
d = widen_frame_masks(dst, f, s->words);
- if (!d)
- return -ENOMEM;
+ if (IS_ERR(d))
+ return PTR_ERR(d);
for (i = 0; i < dst->insn_cnt; i++) {
unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ);
unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE);
@@ -2326,6 +2343,7 @@ static struct func_instance *fresh_instance(struct func_instance *src)
f = kvzalloc_obj(*f, BPF_VERIFIER_GFP);
if (!f)
return ERR_PTR(-ENOMEM);
+ f->env = src->env;
f->callsite = src->callsite;
f->depth = src->depth;
f->subprog = src->subprog;
@@ -2338,8 +2356,12 @@ static void free_instance(struct func_instance *instance)
{
int i;
- for (i = 0; i <= instance->depth; i++)
+ for (i = 0; i <= instance->depth; i++) {
+ if (instance->frames[i])
+ instance->env->mem -= sizeof(long) *
+ frame_mask_words(instance, instance->frames[i]->words);
kvfree(instance->frames[i]);
+ }
kvfree(instance);
}
@@ -2367,16 +2389,6 @@ static int analyze_subprog(struct bpf_verifier_env *env,
struct func_instance *prev_instance = NULL;
int j, err;
- /* a subprog on its own is analyzed once, chains of calls multiply */
- if (depth && ++env->liveness->subprog_calls > 10000) {
- verbose(env, "liveness analysis exceeded complexity limit (%d calls)\n",
- env->liveness->subprog_calls);
- return -E2BIG;
- }
-
- if (bpf_must_abort())
- return -EINTR;
-
/*
* When an instance is reused (must_write_initialized == true),
* record into a fresh instance and merge afterward. This avoids
@@ -2547,12 +2559,15 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env)
int insn_cnt = env->prog->len;
struct func_instance *instance;
struct subprog_at_info *info;
+ /* each subprog keeps its at_in[] to the end */
+ size_t bytes = insn_cnt * (sizeof(*info->at_in) + sizeof(*env->callsite_at_stack));
int k, err = 0;
info = kvzalloc_objs(*info, env->subprog_cnt, BPF_VERIFIER_GFP);
if (!info)
return -ENOMEM;
+ env->mem += bytes;
env->callsite_at_stack = kvzalloc_objs(*env->callsite_at_stack, insn_cnt,
BPF_VERIFIER_GFP);
env->liveness->callee_args = kvcalloc(MAX_CALL_FRAMES * MAX_AT_TRACK_REGS,
@@ -2595,8 +2610,11 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env)
err = print_instances(env);
out:
- for (k = 0; k < insn_cnt; k++)
- kvfree(env->callsite_at_stack[k]);
+ env->mem -= bytes;
+ for (k = 0; k < insn_cnt; k++) {
+ env->mem -= ksize(env->callsite_at_stack[k]);
+ kfree(env->callsite_at_stack[k]);
+ }
kvfree(env->callsite_at_stack);
env->callsite_at_stack = NULL;
kvfree(env->liveness->callee_args);
@@ -2890,10 +2908,9 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
changed = false;
/* a cycle of calls takes a round for each link */
- if (bpf_must_abort()) {
- err = -EINTR;
+ err = bpf_add_cost_and_check(env, env->cfg.cur_postorder, BPF_COST_XREG);
+ if (err)
goto out;
- }
for (i = 0; i < env->cfg.cur_postorder; ++i) {
int insn_idx = order[i];
struct insn_live_regs *live = &state[insn_idx];
diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
index 8a63bb1799a..e1f39629f60 100644
--- a/kernel/bpf/loops.c
+++ b/kernel/bpf/loops.c
@@ -88,9 +88,11 @@ static int idoms_intersect(struct bpf_verifier_env *env, int a, int b)
while (a != b) {
while (postorder_nums[a] < postorder_nums[b]) {
a = idoms[a];
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
}
while (postorder_nums[b] < postorder_nums[a]) {
b = idoms[b];
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
}
}
return a;
@@ -248,6 +250,7 @@ static int mark_as_header(struct bpf_verifier_env *env, int h)
aux[h].loop = kvzalloc_obj(struct bpf_loop, BPF_VERIFIER_GFP);
if (!aux[h].loop)
return -ENOMEM;
+ env->mem += sizeof(struct bpf_loop);
}
return 0;
}
@@ -268,6 +271,7 @@ static int assign_header(struct bpf_verifier_env *env, struct loops_dfs *dfs, in
/* Make sure that loop headers up the chain are sorted by dfs_pos. */
while (aux[n].loop_header != -1) {
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
nh = aux[n].loop_header;
if (nh == h)
return 0;
@@ -290,6 +294,20 @@ int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx)
return aux[idx].loop ? idx : aux[idx].loop_header;
}
+static int check_loop_depth(struct bpf_verifier_env *env, int idx)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ int h, depth = 0;
+
+ for (h = bpf_loop_at_index(env, idx); h >= 0; h = aux[h].loop_header) {
+ if (++depth <= LOOP_STACK_SIZE)
+ continue;
+ bpf_log(&env->log, "Too many nested loops at %d\n", idx);
+ return -E2BIG;
+ }
+ return 0;
+}
+
/*
* The backedge->from and its latch can reside in an inner loop of @h, e.g.:
*
@@ -331,8 +349,9 @@ static int compute_loops_in_subprog(struct bpf_verifier_env *env, struct loops_d
dfs_pos[start] = 1;
stack_sz = 1;
do {
- if (bpf_must_abort())
- return -EINTR;
+ err = bpf_add_cost_and_check(env, 1, BPF_COST_XREG);
+ if (err)
+ return err;
cur = stack[stack_sz - 1];
succ = bpf_insn_successors(env, cur);
@@ -351,6 +370,10 @@ static int compute_loops_in_subprog(struct bpf_verifier_env *env, struct loops_d
stack_sz++;
continue;
}
+ /* A nest only grows. The cases below walk up the nests of both insns. */
+ err = check_loop_depth(env, cur) ?: check_loop_depth(env, s);
+ if (err)
+ return err;
/* 's' is fully explored at this point */
if (dfs_pos[s]) {
/*
@@ -414,9 +437,11 @@ bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int oute
struct bpf_insn_aux_data *aux = env->insn_aux_data;
int idx;
- for (idx = inner_header; idx >= 0; idx = aux[idx].loop_header)
+ for (idx = inner_header; idx >= 0; idx = aux[idx].loop_header) {
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
if (aux[idx].loop_header == outer_header)
return true;
+ }
return false;
}
@@ -463,12 +488,20 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
if (err)
goto out;
}
+ for (i = 0; i < len; i++) {
+ err = check_loop_depth(env, i);
+ if (err)
+ goto out;
+ }
for (i = 0; i < len; i++) {
loop = aux[i].loop;
if (!loop)
continue;
loop->exit_to = -1;
for (j = 0; j < loop->backedges_cnt; j++) {
+ err = bpf_check_limits(env);
+ if (err)
+ goto out;
loop->backedges[j].latch = find_latch(env, i, loop->backedges[j].from);
}
}
@@ -477,6 +510,9 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
iloop = aux[i].loop ? i : aux[i].loop_header;
if (iloop < 0)
continue;
+ err = bpf_check_limits(env);
+ if (err)
+ goto out;
succ = bpf_insn_successors(env, i);
iarray_for_each(s, succ) {
sloop = bpf_loop_at_index(env, s);
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 378dd07dd21..333ae94d153 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -133,6 +133,7 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
{
int i, err;
+ bpf_add_cost(env, bpf_xreg_cnt(st), BPF_COST_XREG);
err = bpf_live_stack_query_init(env, st);
if (err)
return err;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f079435dd25..363ba494cb9 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1742,11 +1742,27 @@ static void free_func_state(struct bpf_func_state *state)
kfree(state);
}
-void bpf_free_verifier_state(struct bpf_verifier_state *state,
- bool free_self)
+/* @st is new or has grown */
+void bpf_charge_state(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
+{
+ /* with the struct that holds it */
+ u32 bytes = 2 * sizeof(*st) + ksize(st->refs), fr;
+ struct bpf_func_state *f;
+
+ for (fr = 0; fr <= st->curframe && (f = st->frame[fr]); fr++)
+ bytes += kmalloc_size_roundup(sizeof(*f)) + ksize(f->stack) +
+ ksize(f->stack_arg_regs) + ksize(f->loop_stack);
+ env->mem += bytes - (s64)st->bytes;
+ st->bytes = bytes;
+}
+
+void bpf_free_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *state,
+ bool free_self)
{
int i;
+ env->mem -= state->bytes;
+ state->bytes = 0;
for (i = 0; i <= state->curframe; i++) {
free_func_state(state->frame[i]);
state->frame[i] = NULL;
@@ -1772,8 +1788,8 @@ static int copy_func_state(struct bpf_func_state *dst,
return copy_stack_state(dst, src);
}
-int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
- const struct bpf_verifier_state *src)
+int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *dst_state,
+ const struct bpf_verifier_state *src)
{
struct bpf_func_state *dst;
int i, err;
@@ -1786,6 +1802,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
err = copy_reference_state(dst_state, src);
if (err)
return err;
+ bpf_add_cost(env, bpf_xreg_cnt(src) + src->acquired_refs, BPF_COST_COPY);
dst_state->speculative = src->speculative;
dst_state->in_sleepable = src->in_sleepable;
dst_state->callers_unknown = src->callers_unknown;
@@ -1807,6 +1824,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
if (err)
return err;
}
+ bpf_charge_state(env, dst_state);
return 0;
}
@@ -1838,9 +1856,9 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
elem->prev_insn_idx = prev_insn_idx;
elem->next = env->head;
env->head = elem;
- err = bpf_copy_verifier_state(&elem->st, cur);
+ err = bpf_copy_verifier_state(env, &elem->st, cur) ?: bpf_check_limits(env);
if (err)
- return ERR_PTR(-ENOMEM);
+ return ERR_PTR(err);
elem->st.speculative |= speculative;
return &elem->st;
}
@@ -6952,6 +6970,9 @@ static int check_stack_access_within_bounds(
return err;
}
+ /* the callers look at each byte */
+ bpf_add_cost(env, max_off - min_off, BPF_COST_COPY);
+
/*
* Every stack access passes through here, including buffers passed to
* helpers and kfuncs and accesses into a caller's frame, so check the
@@ -11458,7 +11479,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
callee->regs[i] = caller->regs[i];
err = grow_stack_arg_slots(env, callee, bpf_in_stack_arg_cnt(&env->subprog_info[subprog]));
if (err) {
- bpf_free_verifier_state(&entry, false);
+ bpf_free_verifier_state(env, &entry, false);
return err;
}
for (i = 0; i < callee->out_stack_arg_cnt; i++)
@@ -11466,7 +11487,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
err = copy_reference_state(&entry, state);
if (err) {
- bpf_free_verifier_state(&entry, false);
+ bpf_free_verifier_state(env, &entry, false);
return err;
}
for (i = 0; i < entry.acquired_refs; i++) {
@@ -11496,7 +11517,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
account_current_path(env);
env->subprog_info[subprog].on_its_own = true;
err = bpf_flow_call(env, state, &entry, subprog, insn_idx);
- bpf_free_verifier_state(&entry, false);
+ bpf_free_verifier_state(env, &entry, false);
/* the call waits, there is nothing to go on with */
return err ?: PROCESS_BPF_EXIT;
}
@@ -11926,13 +11947,13 @@ static int check_func_callx_set(struct bpf_verifier_env *env, int regno, int *in
copy = kzalloc_obj(*copy, BPF_VERIFIER_GFP);
if (!copy)
return -ENOMEM;
- err = bpf_copy_verifier_state(copy, cur);
+ err = bpf_copy_verifier_state(env, copy, cur);
if (!err) {
env->cur_state = copy;
err = check_func_callx_one(env, set->subprog[i], insn_idx);
env->cur_state = cur;
}
- bpf_free_verifier_state(copy, true);
+ bpf_free_verifier_state(env, copy, true);
if (err == PROCESS_BPF_EXIT)
err = 0;
else if (verifier_bug_if(!err, env, "callx: func#%d doesn't wait", set->subprog[i]))
@@ -12228,6 +12249,7 @@ static void account_processed_insn(struct bpf_verifier_env *env)
struct bpf_func_state *frame = cur_func(env);
env->insn_processed++;
+ bpf_add_cost(env, 1, BPF_COST_INSN);
frame->insns_subtotal++;
env->subprog_info[frame->subprogno].insns_self++;
}
@@ -20734,7 +20756,7 @@ static int flow_drain(struct bpf_verifier_env *env)
else if (!err)
err = bpf_flow_deliver(env, &elem->st, elem->insn_idx,
elem->prev_insn_idx);
- bpf_free_verifier_state(&elem->st, false);
+ bpf_free_verifier_state(env, &elem->st, false);
kfree(elem);
}
return err;
@@ -20768,14 +20790,14 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
struct bpf_insn_aux_data *insn_aux;
int err;
- if (bpf_must_abort())
- return -EINTR;
-
/* no insn makes as many ids as there are from here to BPF_INHERITED_ID */
if (env->id_gen >= BPF_MAX_ID) {
verbose(env, "too many ids\n");
return -E2BIG;
}
+ err = bpf_check_limits(env);
+ if (err)
+ return err;
env->prev_insn_idx = prev_insn_idx;
if (env->insn_idx >= insn_cnt) {
@@ -20912,7 +20934,7 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
process_bpf_exit:
account_current_path(env);
mark_verifier_state_scratched(env);
- bpf_free_verifier_state(state, false);
+ bpf_free_verifier_state(env, state, false);
memset(state, 0, sizeof(*state));
return flow_drain(env);
}
@@ -22013,7 +22035,7 @@ static void free_states(struct bpf_verifier_env *env)
{
struct bpf_verifier_stack_elem *elem;
- bpf_free_verifier_state(env->cur_state, true);
+ bpf_free_verifier_state(env, env->cur_state, true);
env->cur_state = NULL;
while ((elem = env->head)) {
env->head = elem->next;
@@ -24122,6 +24144,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
+ env->max_cost = BPF_MAX_COST >> (is_priv ? 0 : BPF_LIMITS_SHIFT_UNPRIV);
+ env->pass_max_cost = U64_MAX;
+ env->max_mem = BPF_MAX_MEM >> (is_priv ? 0 : BPF_LIMITS_SHIFT_UNPRIV);
env->signature = attr->signature;
/* user could have requested verbose verifier output
@@ -24170,6 +24195,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
ret = -ENOMEM;
if (!env->insn_aux_data)
goto skip_full_check;
+ env->mem = len * sizeof(struct bpf_insn_aux_data);
+ /* the analyses that look at an insn once */
+ bpf_add_cost(env, len, BPF_COST_INSN);
for (i = 0; i < len; i++)
env->insn_aux_data[i].orig_idx = i;
env->succ = bpf_iarray_realloc(NULL, 2);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load
2026-11-20 7:41 ` [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load Alexei Starovoitov
@ 2026-11-20 8:00 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:00 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [High] bpf: missing env argument in bpf_free_verifier_state() call in
free_states()
--
commit ca42d04453901ed3278a751f03b001046243fce1
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Limit the cost and the memory of a load
This commit replaces per-pass verifier complexity caps with load-wide cost
and memory limits, and switches stack liveness function instances from a
hash table to a per-callsite array.
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index f079435dd25c5..363ba494cb9a3 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -1742,11 +1742,27 @@ static void free_func_state(struct bpf_func_state *state)
> kfree(state);
> }
>
> -void bpf_free_verifier_state(struct bpf_verifier_state *state,
> - bool free_self)
> +/* @st is new or has grown */
> +void bpf_charge_state(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
> +{
> + /* with the struct that holds it */
> + u32 bytes = 2 * sizeof(*st) + ksize(st->refs), fr;
> + struct bpf_func_state *f;
> +
> + for (fr = 0; fr <= st->curframe && (f = st->frame[fr]); fr++)
> + bytes += kmalloc_size_roundup(sizeof(*f)) + ksize(f->stack) +
> + ksize(f->stack_arg_regs) + ksize(f->loop_stack);
> + env->mem += bytes - (s64)st->bytes;
> + st->bytes = bytes;
> +}
> +
> +void bpf_free_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *state,
> + bool free_self)
> {
> int i;
>
> + env->mem -= state->bytes;
> + state->bytes = 0;
> for (i = 0; i <= state->curframe; i++) {
> free_func_state(state->frame[i]);
> state->frame[i] = NULL;
[ ... ]
> @@ -22013,7 +22035,7 @@ static void free_states(struct bpf_verifier_env *env)
> {
> struct bpf_verifier_stack_elem *elem;
>
> - bpf_free_verifier_state(env->cur_state, true);
> + bpf_free_verifier_state(env, env->cur_state, true);
> env->cur_state = NULL;
> while ((elem = env->head)) {
> env->head = elem->next;
[Severity: High]
Does the second bpf_free_verifier_state() call in free_states() also need
env passed as its first argument? Leaving it at two arguments breaks the
build until commit "bpf: Put a barrier where no mask fits" replaces the
loop later in the series:
kernel/bpf/verifier.c:free_states() {
bpf_free_verifier_state(env, env->cur_state, true);
env->cur_state = NULL;
while ((elem = env->head)) {
env->head = elem->next;
bpf_free_verifier_state(&elem->st, false);
kfree(elem);
}
bpf_flow_reset(env);
}
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=5
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load
2026-11-20 7:41 ` [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load Alexei Starovoitov
2026-11-20 8:00 ` sashiko-bot
@ 2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 3205 bytes --]
> bpf: Limit the cost and the memory of a load
TL;DR:
- free_states() loop still calls bpf_free_verifier_state() with the old
arguments, likely a build break and unreleased memory charge
- verifier_liveness_exp selftest may now fail: it expects the removed
"complexity limit" message
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -22010,9 +22032,9 @@ static void sanitize_dead_code(struct bpf_verifier_env *env)
> static void free_states(struct bpf_verifier_env *env)
> {
> struct bpf_verifier_stack_elem *elem;
>
> - bpf_free_verifier_state(env->cur_state, true);
> + bpf_free_verifier_state(env, env->cur_state, true);
> env->cur_state = NULL;
> while ((elem = env->head)) {
> env->head = elem->next;
> bpf_free_verifier_state(&elem->st, false);
This call in the loop still uses the old two-argument form, while the
prototype now takes env as the first argument. Won't this fail to build
with "too few arguments to function 'bpf_free_verifier_state'"?
Even once the argument count is fixed, these pushed states were charged to
env->mem by push_stack() through bpf_copy_verifier_state() ->
bpf_charge_state(), so they need the env-aware free to give those bytes back.
The later commit 931721673c10 ("bpf: Put a barrier where no mask fits")
moves this loop into free_pushed_states() and passes env, so every commit
from this one up to that one is affected. Should the loop be fixed here,
where the prototype changes, like flow_drain() already does in this patch?
> diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
> --- a/kernel/bpf/liveness.c
> +++ b/kernel/bpf/liveness.c
[ ... ]
> @@ -2366,18 +2388,8 @@ static int analyze_subprog(struct bpf_verifier_env *env,
> int po_end = env->subprog_info[subprog + 1].postorder_start;
> struct func_instance *prev_instance = NULL;
> int j, err;
>
> - /* a subprog on its own is analyzed once, chains of calls multiply */
> - if (depth && ++env->liveness->subprog_calls > 10000) {
> - verbose(env, "liveness analysis exceeded complexity limit (%d calls)\n",
> - env->liveness->subprog_calls);
> - return -E2BIG;
> - }
> -
> - if (bpf_must_abort())
> - return -EINTR;
> -
Removing this message leaves the selftest
tools/testing/selftests/bpf/progs/verifier_liveness_exp.c expecting it:
SEC("?raw_tp")
__failure __log_level(2)
__msg("liveness analysis exceeded complexity limit")
__naked int liveness_exponential_complexity(void)
The program still fails to load, but now through the cost limit, where
bpf_check_limits() logs "BPF program is too complex: cost %llu of %llu,
memory %llu of %llu" and returns -E2BIG. Nothing under kernel/bpf prints
the old text any more, and no later commit in the series touches the
selftests.
Will test_progs -t verifier_liveness_exp now fail? Should the expectation
be updated, for example to __msg("BPF program is too complex")?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 06/49] bpf: Record what each insn reads and writes
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (4 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 05/49] bpf: Limit the cost and the memory of a load Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 07/49] bpf: Name an id after the xreg that has it Alexei Starovoitov
` (42 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Liveness knows what an insn reads and writes, and drops most of that
when it is done. A later patch walks back from an unproven insn and
needs the same data. Keep it in bpf_insn_aux_data:
r2 += r3 use_regs: r2 r3, def_regs: r2
r2 = *(u64 *)(r10 - 8) may_read_mask: fp-8
*(u64 *)(r1 + 0) = r2 writes_outer, if r1 points to the frame
of a caller
Give registers and stack slots one number, the xreg:
0 .. 10 11 .. 13 14 ..
r0 .. r10 hidden registers fp-8, fp-16, ...
Two sets of reads were too large:
- LD_ABS and LD_IND read all registers and wrote none. They read r6 and
the source register, and they clobber r0-r5.
- bpf_timer_set_callback() does not read r3. The verifier sets that
argument.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 33 ++++++++++++++++++++++++++++
kernel/bpf/liveness.c | 42 ++++++++++++++++++++++++------------
kernel/bpf/verifier.c | 1 +
3 files changed, 62 insertions(+), 14 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index ed271135fe5..ac20aa50835 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -382,6 +382,30 @@ struct bpf_retval_range {
bool return_32bit;
};
+/*
+ * An xreg is a register or a stack slot of a frame:
+ *
+ * 0 .. 10 11 .. 13 MAX_BPF_EXT_REG ..
+ * r0 .. r10 hidden registers fp-8, fp-16, ...
+ *
+ * A state has no value for a hidden register. No mask of xregs has its bit.
+ */
+#define MAX_BPF_XREG (MAX_BPF_EXT_REG + MAX_BPF_STACK_SLOTS)
+
+static inline bool bpf_xreg_is_slot(u32 xregno)
+{
+ return xregno >= MAX_BPF_EXT_REG;
+}
+
+static inline u32 bpf_spi_to_xreg(u32 spi)
+{
+ return MAX_BPF_EXT_REG + spi;
+}
+
+static inline u32 bpf_xreg_to_spi(u32 xregno)
+{
+ return xregno - MAX_BPF_EXT_REG;
+}
#define LOOP_STACK_SIZE 16
@@ -811,10 +835,19 @@ struct bpf_insn_aux_data {
u64 non_stack_access:1; /* instruction can access non-stack memory */
/* true if some jump or call instruction targets this instruction */
u64 jump_target:1;
+ /* it can read or write the frame of a caller, in a call with a known callee */
+ u32 reads_outer:1;
+ u32 writes_outer:1;
+ DECLARE_BITMAP(may_read_mask, MAX_BPF_STACK_SLOTS);
unsigned int orig_idx; /* original instruction index, initialized once */
/* registers alive before this instruction. */
u16 live_regs_before;
+ /*
+ * Registers that it reads and writes. The arguments of a static function and of callx
+ * are not in @use_regs. The bits above r10 are not valid.
+ */
+ u16 use_regs, def_regs;
/*
* Bitmask of R0-R9 that hold known values at this instruction.
* const_reg_mask: scalar constants that fit in 32 bits.
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 4a58c5f4db8..6e6b2c6d88c 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -770,29 +770,40 @@ static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask,
}
/*
- * Precompute, for each instruction, the OR of may_write and live_before masks
+ * Precompute, for each instruction, the OR of may_read, may_write and live_before masks
* over its top frame across all func_instances reaching it, stash them in the
- * insn_aux_data.
+ * insn_aux_data. The other frames set reads_outer and writes_outer.
*/
static void compute_may_write_masks(struct bpf_verifier_env *env)
{
- struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct bpf_insn_aux_data *aux = env->insn_aux_data, *a;
struct bpf_liveness *liveness = env->liveness;
struct func_instance *instance;
+ unsigned long *read, *write;
struct frame_masks *fm;
- u32 nbits;
+ u32 nbits, f;
int i;
list_for_each_entry(instance, &liveness->func_instances, node) {
- fm = instance->frames[instance->depth];
- if (!fm)
- continue;
- nbits = frame_mask_bits(fm);
- for (i = 0; i < instance->insn_cnt; i++) {
- half_spis_to_slots(aux[instance->subprog_start + i].may_write_mask,
- rel_mask(fm, i, FM_MAY_WRITE), nbits);
- half_spis_to_slots(aux[instance->subprog_start + i].live_stack_before,
- rel_mask(fm, i, FM_LIVE_BEFORE), nbits);
+ for (f = 0; f <= instance->depth; f++) {
+ fm = instance->frames[f];
+ if (!fm)
+ continue;
+ nbits = frame_mask_bits(fm);
+ for (i = 0; i < instance->insn_cnt; i++) {
+ a = &aux[instance->subprog_start + i];
+ read = rel_mask(fm, i, FM_MAY_READ);
+ write = rel_mask(fm, i, FM_MAY_WRITE);
+ if (f < instance->depth) {
+ a->reads_outer |= !bitmap_empty(read, nbits);
+ a->writes_outer |= !bitmap_empty(write, nbits);
+ continue;
+ }
+ half_spis_to_slots(a->may_read_mask, read, nbits);
+ half_spis_to_slots(a->may_write_mask, write, nbits);
+ half_spis_to_slots(a->live_stack_before,
+ rel_mask(fm, i, FM_LIVE_BEFORE), nbits);
+ }
}
}
}
@@ -2680,7 +2691,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
break;
case BPF_ABS:
case BPF_IND:
- /* stick with defaults */
+ def = mask_widen(ALL_CALLER_SAVED_REGS);
+ use = reg64_mask(BPF_REG_6) | (mode == BPF_IND ? src : 0);
break;
}
break;
@@ -2953,6 +2965,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
u32 in = state[i].in;
insn_aux[i].live_regs_before = mask_lo(in) | mask_hi(in);
+ insn_aux[i].use_regs = mask_lo(state[i].use) | mask_hi(state[i].use);
+ insn_aux[i].def_regs = mask_lo(state[i].def);
/*
* On architectures where 32-bit operations do not reset upper halves
* of the registers, the verifier needs to zero extend a destination
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 363ba494cb9..491b4ecd5f9 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19834,6 +19834,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
if (fn->arg_type[i] == ARG_UNUSED)
break;
cs->arg_slot_cnt++;
+ cs->implicit_slot_cnt += fn->arg_type[i] == ARG_PTR_TO_PROG_AUX;
}
return true;
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 06/49] bpf: Record what each insn reads and writes
2026-11-20 7:41 ` [PATCH bpf-next 06/49] bpf: Record what each insn reads and writes Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2000 bytes --]
> bpf: Record what each insn reads and writes
TL;DR: The new LD_ABS/LD_IND liveness rules likely change the logged register
masks, so the compute_live_registers ldabs selftest probably fails unless its
expectations and comment are updated.
> diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
> --- a/kernel/bpf/liveness.c
> +++ b/kernel/bpf/liveness.c
[ ... ]
> @@ -2680,7 +2691,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
> break;
> case BPF_ABS:
> case BPF_IND:
> - /* stick with defaults */
> + def = mask_widen(ALL_CALLER_SAVED_REGS);
> + use = reg64_mask(BPF_REG_6) | (mode == BPF_IND ? src : 0);
> break;
> }
> break;
Does this change break the ldabs test in
tools/testing/selftests/bpf/progs/compute_live_registers.c?
That test still carries the comment "No rules specific for LD_ABS/LD_IND,
default behaviour kicks in" and expects the old live_regs_before output:
__msg("2: 0123456789 (30) r0 = *(u8 *)skb[42]")
__msg("3: 012.456789 (0f) r7 += r0")
__msg("4: 012.456789 (b7) r3 = 42")
__msg("5: 0123456789 (50) r0 = *(u8 *)skb[r3 + 0]")
__msg("6: 0......7.. (0f) r7 += r0")
With def = r0-r5 and use = r6 (plus src for LD_IND), the masks computed by
bpf_compute_live_registers() for that program become:
insn 5: ...3..67.. (expected 0123456789)
insn 4: ......67.. (expected 012.456789)
insn 3: 0.....67.. (expected 012.456789)
insn 2: ......67.. (expected 0123456789)
log_program() prints these masks and test_loader matches __msg() literally
and in order, so test_progs -t compute_live_registers should fail on the
first of these messages.
Should the test expectations and the comment be updated in this patch, or
in a selftest patch placed right after it?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 07/49] bpf: Name an id after the xreg that has it
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (5 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 06/49] bpf: Record what each insn reads and writes Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values Alexei Starovoitov
` (41 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
An id stands for the set of xregs that have it. Its number depends on
the path. Two states with the same sets have other numbers:
state 1: r1=ptr(id=17) r6=ptr(id=17) r7=ptr(id=4)
state 2: r1=ptr(id=3) r6=ptr(id=3) r7=ptr(id=9)
So the compare, the join and the return from a function each had a map
from one set of ids to another.
bpf_renumber_ids() names each id after the first xreg that has it:
both: r1=ptr(id=r1) r6=ptr(id=r1) r7=ptr(id=r7)
Equal sets have equal ids now, and reg_object_equal() is a memcmp().
If r1 = r6 in one state and r1 is not set in the other, r6 has two
names and counts as two objects. The maps saw one object.
The ids are renumbered at the end of a basic block and before the call
of a function. The low bits tell the origin:
bpf_new_id() an insn of this basic block made it
BPF_REF_ID() it is held, the number is the index in refs[]
BPF_XREG_ID() the number is an xreg
BPF_RETURNED_ID() a function returned it
A caller and its callee have the same refs[], so they have the same
names. inherited_id(), fresh_id() and BPF_MAX_ID go away.
The id of a __ref argument is not constant any more. refs[].ctx_arg
tells which argument a reference came with. The report of a leak finds
the acquire by refs[].diag_id, which is not renumbered. A reference
that leaks has no release, so the events of a release go away.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf.h | 4 +-
include/linux/bpf_verifier.h | 56 +++++---
kernel/bpf/btf.c | 2 +-
kernel/bpf/diagnostics.c | 56 +++-----
kernel/bpf/diagnostics.h | 6 +-
kernel/bpf/states.c | 79 +++++++++-
kernel/bpf/verifier.c | 271 ++++++++++++-----------------------
7 files changed, 229 insertions(+), 245 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 8b985485f57..861b6ec4a17 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1188,7 +1188,7 @@ struct bpf_insn_access_aux {
struct {
struct btf *btf;
u32 btf_id;
- u32 ref_id;
+ u32 ref_arg; /* see struct bpf_ctx_arg_aux */
};
};
u32 mem_size;
@@ -1831,7 +1831,7 @@ struct bpf_ctx_arg_aux {
enum bpf_reg_type reg_type;
struct btf *btf;
u32 btf_id;
- u32 ref_id;
+ u32 ref_arg; /* a number for a __ref argument, else 0 */
u32 mem_size;
bool refcounted;
};
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index ac20aa50835..5886cc48653 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -241,6 +241,12 @@ static inline bool reg_fixed_part_equal(const struct bpf_reg_state *ra,
return ra->type == rb->type && !memcmp(&ra->raw, &rb->raw, sizeof(ra->raw));
}
+/* That and the ids, see bpf_renumber_ids() */
+static inline bool reg_object_equal(const struct bpf_reg_state *ra, const struct bpf_reg_state *rb)
+{
+ return !memcmp(ra, rb, offsetof(struct bpf_reg_state, var_off));
+}
+
/* All but the ids */
static inline bool reg_value_equal(const struct bpf_reg_state *ra, const struct bpf_reg_state *rb)
{
@@ -366,9 +372,13 @@ struct bpf_reference_state {
* is used purely to inform the user of a reference leak.
*/
int insn_idx;
+ u32 diag_id; /* finds the acquire in the report of a leak, it is not renumbered */
union {
/* For REF_TYPE_PTR */
- int parent_id;
+ struct {
+ int parent_id;
+ u32 ctx_arg; /* it came with that __ref argument, the first is 1 */
+ };
/* Use to keep track of the source object of a lock, to ensure
* it matches on unlock.
*/
@@ -510,15 +520,17 @@ struct bpf_verifier_state {
u32 insn_idx;
u32 curframe;
+ struct_group(held,
+ void *active_lock_ptr;
u32 acquired_refs;
u32 active_locks;
u32 active_preempt_locks;
+ u32 active_rcu_locks;
u32 active_irq_id;
u32 active_lock_id;
- void *active_lock_ptr;
- u32 active_rcu_locks;
bool speculative;
+ );
bool in_sleepable;
/* a global function or an extension, and what it calls: exit of frame 0 is not the end */
bool callers_unknown;
@@ -1048,21 +1060,28 @@ struct bpf_id_pair {
};
/*
- * A function that is verified on its own, see func_is_on_its_own(), has no ids of its caller.
- * It has names for the ids of what is held: BPF_INHERITED_ID + i is the id of refs[i] at the call.
- * - A name depends only on the place. Each id in the state at the entry is replaced with its name.
- * The ids of the caller can be names that it got from its own caller: they are replaced too.
- * - A name stands for the same thing in all states of the function. It is equal only to itself:
- * check_ids() and jid_note() do not rename it. So two calls share a state at the entry, and what
- * comes out for it, only if the same places have the same names.
- * - bpf_flow_return() puts the ids of the call that waits back. All other ids get new values.
- * - What comes out has the ranges of all calls that share the state at the entry. So a name is good
- * for what holds whatever the ranges are: the same object. Other ids have no name: such an
- * argument keeps the frames of the caller, see arg_needs_caller().
- * - env->id_gen stays below BPF_MAX_ID. So no id is a name.
+ * At a join point each id gets the number of the xreg that has it, see bpf_renumber_ids().
+ * The low bits give the origin of the id:
+ *
+ * bpf_new_id() an insn made it. It is renumbered at the end of the basic block.
+ * BPF_REF_ID() it is held. The number is the index in refs[].
+ * BPF_XREG_ID() the number is the first xreg that has it.
+ * BPF_RETURNED_ID() a function on its own returned it. No xreg of the caller has it.
+ *
+ * A function on its own, see func_is_on_its_own(), and its waiting call have the same ids for
+ * what is held. An exit state has the ranges of all calls of the instance. So such id shows
+ * only what holds for all ranges: the same object. An argument with another id keeps the
+ * frames of the caller, see arg_needs_caller().
*/
-#define BPF_INHERITED_ID 0x20000000
-#define BPF_MAX_ID (BPF_INHERITED_ID / 2)
+#define bpf_new_id(env) (((env)->id_gen += 4) - 3)
+#define BPF_REF_ID(i) (((i) + 1) << 2)
+#define BPF_XREG_ID(i) ((i) << 2 | 2)
+#define BPF_RETURNED_ID(id) ((id) | 1)
+
+static inline bool bpf_id_is_xreg(u32 id)
+{
+ return id & 2;
+}
struct bpf_idmap {
u32 cnt;
@@ -1462,6 +1481,8 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
struct bpf_verifier_state *cur);
+int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+
/*
* The two limits of the verifier, for the whole load. The unit of cost is one step of a scan,
* about a dozen insns of the CPU. Memory is the bytes of states and tables at a time.
@@ -1499,6 +1520,7 @@ int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, i
void bpf_flow_next_is_no_bound(struct bpf_verifier_env *env, int insn_idx, int h);
bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env);
int bpf_flow_place_guards(struct bpf_verifier_env *env);
+#define BPF_EXIT_NEEDS_CALLER 1 /* verify the call in the frames of the caller */
int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
const struct bpf_verifier_state *exit, int call_idx);
int bpf_insert_loop_guards(struct bpf_verifier_env *env);
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index e95ca39f990..400aa2f226e 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -7645,7 +7645,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
info->reg_type = ctx_arg_info->reg_type;
info->btf = ctx_arg_info->btf ? : btf_vmlinux;
info->btf_id = ctx_arg_info->btf_id;
- info->ref_id = ctx_arg_info->ref_id;
+ info->ref_arg = ctx_arg_info->ref_arg;
return true;
}
}
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c
index 5783fa20f56..25d7982327d 100644
--- a/kernel/bpf/diagnostics.c
+++ b/kernel/bpf/diagnostics.c
@@ -114,7 +114,6 @@ enum bpf_diag_history_kind {
BPF_DIAG_HISTORY_BRANCH,
BPF_DIAG_HISTORY_MOD,
BPF_DIAG_HISTORY_REF_ACQUIRE,
- BPF_DIAG_HISTORY_REF_RELEASE,
BPF_DIAG_HISTORY_CONTEXT,
};
@@ -134,7 +133,7 @@ struct bpf_diag_history_event {
bool origin_valid;
} mod;
struct {
- u32 ref_id;
+ u32 diag_id;
} ref;
struct {
u32 depth;
@@ -157,7 +156,7 @@ struct bpf_diag_history_opts {
u32 frameno;
int regno;
int stack_arg_slot;
- u32 ref_id;
+ const struct bpf_reference_state *ref;
enum bpf_diag_context_kind ctx_kind;
u32 ctx_depth;
};
@@ -220,6 +219,7 @@ struct bpf_diag {
struct list_head fmt_chunks;
struct bpf_diag_mod_scope mod;
u32 frame_id_gen;
+ u32 ref_id_gen;
};
bool bpf_diag_enabled(const struct bpf_verifier_env *env)
@@ -1693,29 +1693,19 @@ void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env,
reason, NULL, NULL, NULL);
}
-static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id)
+void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, struct bpf_reference_state *ref)
{
struct bpf_diag_history_event event = {
- .insn_idx = insn_idx,
- .kind = kind,
- .ref = {
- .ref_id = ref_id,
- },
+ .insn_idx = ref->insn_idx,
+ .kind = BPF_DIAG_HISTORY_REF_ACQUIRE,
};
+ if (!env->diag)
+ return;
+ event.ref.diag_id = ref->diag_id = ++env->diag->ref_id_gen;
diag_append_history(env, &event);
}
-void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id)
-{
- diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id);
-}
-
-void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id)
-{
- diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id);
-}
-
void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx,
enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth)
{
@@ -1864,7 +1854,7 @@ static int diag_history_start_idx(const struct bpf_diag_log *log,
event = &log->events[log_pos(log, i - 1)];
if (event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE &&
- event->ref.ref_id == opts->ref_id)
+ event->ref.diag_id == opts->ref->diag_id)
return i - 1;
}
@@ -1882,9 +1872,8 @@ static bool diag_history_event_visible(const struct bpf_diag_history_event *even
case BPF_DIAG_HISTORY_MOD:
return filter->lineage_valid && event->in_lineage;
case BPF_DIAG_HISTORY_REF_ACQUIRE:
- case BPF_DIAG_HISTORY_REF_RELEASE:
return opts->scope == BPF_DIAG_HISTORY_SCOPE_REF &&
- event->ref.ref_id == opts->ref_id;
+ event->ref.diag_id == opts->ref->diag_id;
case BPF_DIAG_HISTORY_CONTEXT:
return opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT &&
event->ctx.kind == opts->ctx_kind;
@@ -2071,21 +2060,22 @@ void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *proble
diag_suggestion(env, "%s", suggestion);
}
-void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn)
+void bpf_diag_leak(struct bpf_verifier_env *env, const struct bpf_reference_state *ref,
+ u32 fail_insn)
{
struct bpf_diag_history_opts opts = {
.scope = BPF_DIAG_HISTORY_SCOPE_REF,
- .ref_id = ref_id,
+ .ref = ref,
};
bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, "unreleased resource");
diag_reason(
env, "Owned resource (id=%u) was acquired at instruction %u and still needs to be released before this exit path.",
- ref_id, alloc_insn);
+ ref->id, ref->insn_idx);
diag_section(env, "At");
bpf_diag_source(env, fail_insn, "error",
- "owned resource (id=%u) still needs release", ref_id);
+ "owned resource (id=%u) still needs release", ref->id);
diag_print_history(env, &opts);
@@ -2258,16 +2248,6 @@ static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_h
old, new);
}
-static void diag_print_ref_event(struct bpf_verifier_env *env,
- const struct bpf_diag_history_event *event)
-{
- const char *label;
-
- label = event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE ? "acquired" : "released";
- bpf_diag_source(env, event->insn_idx, label, "owned resource (id=%u)",
- event->ref.ref_id);
-}
-
static const char *diag_context_name(enum bpf_diag_context_kind kind)
{
switch (kind) {
@@ -2362,8 +2342,8 @@ static void diag_print_history(struct bpf_verifier_env *env,
diag_print_mod(env, event);
break;
case BPF_DIAG_HISTORY_REF_ACQUIRE:
- case BPF_DIAG_HISTORY_REF_RELEASE:
- diag_print_ref_event(env, event);
+ bpf_diag_source(env, event->insn_idx, "acquired", "owned resource (id=%u)",
+ opts->ref->id);
break;
case BPF_DIAG_HISTORY_CONTEXT:
diag_print_context_event(env, event);
diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h
index e155d2588a5..f6380137e9a 100644
--- a/kernel/bpf/diagnostics.h
+++ b/kernel/bpf/diagnostics.h
@@ -77,7 +77,8 @@ void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *probl
const struct bpf_reference_state *active_lock);
void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem,
const char *reason, const char *suggestion, u32 depth);
-void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn);
+void bpf_diag_leak(struct bpf_verifier_env *env, const struct bpf_reference_state *ref,
+ u32 fail_insn);
void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno,
int stack_arg_slot, const char *call_name, const char *arg_name,
const char *reason, const char *suggestion);
@@ -104,8 +105,7 @@ void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_st
void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env,
const struct bpf_func_state *state, s16 min_off, s16 max_off,
enum bpf_diag_mod_reason reason);
-void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id);
-void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id);
+void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, struct bpf_reference_state *ref);
void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx,
enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth);
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 333ae94d153..1e264757be4 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -35,6 +35,83 @@ static bool range_within(const struct bpf_reg_state *old,
(cur->base - old->base) % old->step == 0;
}
+static void renumber_id(struct bpf_verifier_env *env, u32 *id, u32 new_id)
+{
+ struct bpf_idmap *map = &env->idmap_scratch;
+ u32 i;
+
+ if (!*id)
+ return;
+ for (i = 0; i < map->cnt && map->map[i].old != *id; i++)
+ ;
+ bpf_add_cost(env, 2 * i, BPF_COST_SCAN);
+ if (i == BPF_ID_MAP_SIZE)
+ return;
+ if (i == map->cnt)
+ map->map[map->cnt++] = (struct bpf_id_pair){ *id, new_id };
+ *id = map->map[i].cur;
+}
+
+static void renumber_reg_ids(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 xreg)
+{
+ renumber_id(env, ®->id, BPF_XREG_ID(3 * xreg));
+ renumber_id(env, ®->parent_id, BPF_XREG_ID(3 * xreg + 1));
+ renumber_id(env, ®->map_uid, BPF_XREG_ID(3 * xreg + 2));
+}
+
+/*
+ * An id stands for the set of xregs that have it. Name it after the first one. What is held goes
+ * first. Then states with the same sets have the same ids, and ids are compared as they are.
+ * The name is the xreg, not a number in a sequence. So an xreg that is not set does not change
+ * the names of other sets. But if only one state has r1 = r6, r6 has two names.
+ * Visit an xreg once: a name can be the old id of another xreg.
+ */
+int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
+{
+ u32 mask[] = { BIT(STACK_SPILL),
+ BIT(STACK_DYNPTR) | BIT(STACK_ITER) | BIT(STACK_IRQ_FLAG) };
+ struct bpf_idmap *map = &env->idmap_scratch;
+ struct bpf_reg_state *reg;
+ struct bpf_func_state *f;
+ u32 fr, i, xreg;
+ int held;
+
+ map->cnt = 0;
+ for (i = 0; i < st->acquired_refs && i < BPF_ID_MAP_SIZE; i++) {
+ renumber_id(env, &st->refs[i].id, BPF_REF_ID(2 * i));
+ if (st->refs[i].type == REF_TYPE_PTR)
+ renumber_id(env, &st->refs[i].parent_id, BPF_REF_ID(2 * i + 1));
+ }
+ /* they are among these */
+ renumber_id(env, &st->active_irq_id, 0);
+ renumber_id(env, &st->active_lock_id, 0);
+ for (held = 1; held >= 0; held--) {
+ for (fr = 0; fr <= st->curframe; fr++) {
+ f = st->frame[fr];
+ /* registers, the frame, slots, arguments */
+ xreg = fr * (BPF_ID_MAP_SIZE / MAX_CALL_FRAMES + 1) + MAX_BPF_REG;
+ if (held)
+ renumber_id(env, &f->frame_id, BPF_XREG_ID(3 * xreg));
+ for (i = 0; i < bpf_stack_nr_slots(f); i++) {
+ reg = bpf_get_spilled_reg(i, f, mask[held]);
+ if (reg)
+ renumber_reg_ids(env, reg, xreg + 1 + i);
+ }
+ if (held)
+ continue;
+ for (i = 0; i < MAX_BPF_REG; i++)
+ renumber_reg_ids(env, &f->regs[i], xreg - MAX_BPF_REG + i);
+ for (i = 0; i < f->out_stack_arg_cnt; i++)
+ renumber_reg_ids(env, &f->stack_arg_regs[i],
+ xreg + 1 + MAX_BPF_STACK_SLOTS + i);
+ }
+ }
+ if (map->cnt < BPF_ID_MAP_SIZE && st->acquired_refs < BPF_ID_MAP_SIZE)
+ return 0;
+ bpf_log(&env->log, "too many ids\n");
+ return -E2BIG;
+}
+
static void __clean_func_state(struct bpf_verifier_env *env,
struct bpf_func_state *st,
u16 live_regs, int frame)
@@ -143,7 +220,7 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
__clean_func_state(env, st->frame[i], live_regs, i);
}
- return 0;
+ return bpf_renumber_ids(env, st);
}
static struct bpf_reg_state unbound_reg;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 491b4ecd5f9..030263d01e3 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -205,8 +205,7 @@ struct bpf_verifier_stack_elem {
#define BPF_PRIV_STACK_MIN_SIZE 64
static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id);
-static int __release_reference_nomark(struct bpf_verifier_state *state, int id);
-static int release_reference_nomark(struct bpf_verifier_env *env, int id);
+static int release_reference_nomark(struct bpf_verifier_state *state, int id);
static int release_reference(struct bpf_verifier_env *env, int id);
static int invalidate_reference(struct bpf_verifier_env *env, int id);
static u32 reg_lifetime_id(struct bpf_verifier_env *env,
@@ -717,7 +716,7 @@ static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
struct bpf_reg_state *sreg2,
enum bpf_dynptr_type type, int parent_id)
{
- int id = ++env->id_gen;
+ int id = bpf_new_id(env);
__mark_dynptr_reg(sreg1, type, true, id, parent_id);
__mark_dynptr_reg(sreg2, type, false, id, parent_id);
@@ -727,7 +726,7 @@ static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
struct bpf_reg_state *reg,
enum bpf_dynptr_type type)
{
- __mark_dynptr_reg(reg, type, true, ++env->id_gen, 0);
+ __mark_dynptr_reg(reg, type, true, bpf_new_id(env), 0);
}
static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
@@ -1439,13 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf
if (!dst->refs)
return -ENOMEM;
- dst->acquired_refs = src->acquired_refs;
- dst->active_locks = src->active_locks;
- dst->active_preempt_locks = src->active_preempt_locks;
- dst->active_rcu_locks = src->active_rcu_locks;
- dst->active_irq_id = src->active_irq_id;
- dst->active_lock_id = src->active_lock_id;
- dst->active_lock_ptr = src->active_lock_ptr;
+ dst->held = src->held;
return 0;
}
@@ -1556,9 +1549,9 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par
if (!s)
return -ENOMEM;
s->type = REF_TYPE_PTR;
- s->id = ++env->id_gen;
+ s->id = bpf_new_id(env);
s->parent_id = parent_id;
- bpf_diag_record_ref_acquire(env, insn_idx, s->id);
+ bpf_diag_record_ref_acquire(env, s);
return s->id;
}
@@ -1592,7 +1585,7 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)
if (!s)
return -ENOMEM;
s->type = REF_TYPE_IRQ;
- s->id = ++env->id_gen;
+ s->id = bpf_new_id(env);
state->active_irq_id = s->id;
bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true,
@@ -2504,7 +2497,7 @@ static int mark_btf_ld_reg(struct bpf_verifier_env *env,
regs[regno].btf = btf;
regs[regno].btf_id = btf_id;
if (type_may_be_null(flag))
- regs[regno].id = ++env->id_gen;
+ regs[regno].id = bpf_new_id(env);
return 0;
case PTR_TO_MEM:
mark_reg_known_zero(env, regs, regno);
@@ -4758,7 +4751,7 @@ static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
reg = reg_state(env, regno);
reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
reg->mem_size = t->size;
- reg->id = ++env->id_gen;
+ reg->id = bpf_new_id(env);
return 0;
}
@@ -4974,6 +4967,17 @@ static bool is_var_ctx_off_allowed(struct bpf_prog *prog)
return resolve_prog_type(prog) == BPF_PROG_TYPE_SYSCALL;
}
+/* The id of the reference that came with __ref argument @arg. 0 if there is none. */
+static u32 ctx_arg_ref_id(const struct bpf_verifier_state *st, u32 arg)
+{
+ u32 i;
+
+ for (i = 0; arg && i < st->acquired_refs; i++)
+ if (st->refs[i].type == REF_TYPE_PTR && st->refs[i].ctx_arg == arg)
+ return st->refs[i].id;
+ return 0;
+}
+
/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
enum bpf_access_type t, struct bpf_insn_access_aux *info)
@@ -4988,8 +4992,7 @@ static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int of
* type of narrower access.
*/
if (base_type(info->reg_type) == PTR_TO_BTF_ID) {
- if (info->ref_id &&
- !find_reference_state(env->cur_state, info->ref_id)) {
+ if (info->ref_arg && !ctx_arg_ref_id(env->cur_state, info->ref_arg)) {
verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",
off);
return -EACCES;
@@ -7362,12 +7365,13 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
if (base_type(info.reg_type) == PTR_TO_BTF_ID) {
regs[value_regno].btf = info.btf;
regs[value_regno].btf_id = info.btf_id;
- regs[value_regno].id = info.ref_id;
+ regs[value_regno].id =
+ ctx_arg_ref_id(env->cur_state, info.ref_arg);
} else if (base_type(info.reg_type) == PTR_TO_MEM) {
regs[value_regno].mem_size = info.mem_size;
}
if (type_may_be_null(info.reg_type) && !regs[value_regno].id)
- regs[value_regno].id = ++env->id_gen;
+ regs[value_regno].id = bpf_new_id(env);
}
regs[value_regno].type = info.reg_type;
}
@@ -10869,7 +10873,7 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range
reg->range = AT_PKT_END;
}
-static int __release_reference_nomark(struct bpf_verifier_state *state, int id)
+static int release_reference_nomark(struct bpf_verifier_state *state, int id)
{
int i;
@@ -10884,16 +10888,6 @@ static int __release_reference_nomark(struct bpf_verifier_state *state, int id)
return -EINVAL;
}
-static int release_reference_nomark(struct bpf_verifier_env *env, int id)
-{
- int err;
-
- err = __release_reference_nomark(env->cur_state, id);
- if (!err)
- bpf_diag_record_ref_release(env, env->insn_idx, id);
- return err;
-}
-
static int idstack_push(struct bpf_idmap *idmap, u32 id)
{
int i;
@@ -10990,7 +10984,7 @@ static int invalidate_reference(struct bpf_verifier_env *env, int id)
static int release_reference(struct bpf_verifier_env *env, int id)
{
if (find_reference_state(env->cur_state, id))
- WARN_ON_ONCE(release_reference_nomark(env, id));
+ WARN_ON_ONCE(release_reference_nomark(env->cur_state, id));
return invalidate_reference(env, id);
}
@@ -11031,7 +11025,7 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id)
struct bpf_reg_state *reg;
int err;
- err = release_reference_nomark(env, id);
+ err = release_reference_nomark(env->cur_state, id);
if (err)
return err;
@@ -11370,40 +11364,23 @@ static void account_current_path(struct bpf_verifier_env *env);
* callers have: nothing points to their stack.
*
* What the callers hold, references, locks and so on, is a part of that state:
- * it matters for what the function may do. Calls that differ in ids only
- * have to get to the same state, so the ids of what is held are replaced with
- * BPF_INHERITED_ID + the position for the function, and put back at the return.
+ * it matters for what the function can do.
*/
-/* 0 if it is not held */
-static u32 inherited_id(const struct bpf_verifier_state *state, u32 id)
-{
- int i;
-
- /* the first one: a lock in an object that the caller owns has the id of the reference */
- for (i = 0; id && i < state->acquired_refs; i++)
- if (state->refs[i].id == id)
- return BPF_INHERITED_ID + i;
- return 0;
-}
-
/* What func_is_on_its_own() asks of an argument, be it in a register or on the stack */
static bool arg_needs_caller(struct bpf_verifier_env *env, const struct bpf_verifier_state *state,
const struct bpf_subprog_info *sub, const struct bpf_reg_state *reg)
{
- /* the id of a __ref argument is kept in ctx_arg_info[], see __check_ctx_access() */
+ /* it can load a __ref argument from the context and release it */
if (reg->type == PTR_TO_CTX && state->acquired_refs &&
env->prog->type == BPF_PROG_TYPE_STRUCT_OPS)
return true;
- /* only the ids of what is held have names */
- if (reg->type != SCALAR_VALUE &&
- ((reg->id && !inherited_id(state, reg->id)) ||
- (reg->parent_id && !inherited_id(state, reg->parent_id)) ||
- (reg->map_uid && !inherited_id(state, reg->map_uid))))
+ /* only the ids of what is held mean something without the caller */
+ if (bpf_id_is_xreg(reg->id | reg->parent_id | reg->map_uid))
return true;
/*
- * Pointers to the packet that share an id have ranges of the same size, see
- * find_good_pkt_pointers(). What comes out of the function has the range of all calls.
+ * Packet pointers that share an id have ranges of the same size, see
+ * find_good_pkt_pointers(). The result has the range of all calls of the instance.
*/
if (reg_is_pkt_pointer(reg) && reg->id)
return true;
@@ -11434,9 +11411,6 @@ static bool func_is_on_its_own(struct bpf_verifier_env *env, struct bpf_verifier
/* what must not be done in that callback is refused by looking at the frames */
if (in_rbtree_lock_required_cb(env))
return false;
- /* there are that many names for what is held, and bpf_flow_return() has room for them */
- if (state->acquired_refs > BPF_ID_MAP_SIZE / 2)
- return false;
for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
if (!(live & BIT(i)))
continue;
@@ -11490,29 +11464,9 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
bpf_free_verifier_state(env, &entry, false);
return err;
}
- for (i = 0; i < entry.acquired_refs; i++) {
- /* the id of a lock is the id of the pointer to it, which may have none */
- entry.refs[i].id = inherited_id(state, entry.refs[i].id);
- if (entry.refs[i].type == REF_TYPE_PTR)
- entry.refs[i].parent_id = inherited_id(state, entry.refs[i].parent_id);
- }
- entry.active_irq_id = inherited_id(state, entry.active_irq_id);
- entry.active_lock_id = inherited_id(state, entry.active_lock_id);
callee->entry_refs = entry.acquired_refs;
callee->entry_rcu_locks = entry.active_rcu_locks;
callee->entry_preempt_locks = entry.active_preempt_locks;
- for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
- callee->regs[i].id = inherited_id(state, callee->regs[i].id);
- callee->regs[i].parent_id = inherited_id(state, callee->regs[i].parent_id);
- callee->regs[i].map_uid = inherited_id(state, callee->regs[i].map_uid);
- }
- for (i = 0; i < callee->out_stack_arg_cnt; i++) {
- callee->stack_arg_regs[i].id = inherited_id(state, callee->stack_arg_regs[i].id);
- callee->stack_arg_regs[i].parent_id =
- inherited_id(state, callee->stack_arg_regs[i].parent_id);
- callee->stack_arg_regs[i].map_uid =
- inherited_id(state, callee->stack_arg_regs[i].map_uid);
- }
account_current_path(env);
env->subprog_info[subprog].on_its_own = true;
@@ -11550,39 +11504,39 @@ static int exit_func_on_its_own(struct bpf_verifier_env *env)
return err ?: PROCESS_BPF_EXIT;
}
-static u32 fresh_id(struct bpf_verifier_env *env, struct bpf_idmap *idmap, u32 id)
+static u32 returned_id(u32 id)
{
- u32 i;
-
- if (!id)
- return 0;
- for (i = 0; i < idmap->cnt; i++)
- if (idmap->map[i].old == id)
- return idmap->map[i].cur;
- if (idmap->cnt == BPF_ID_MAP_SIZE)
- return 0;
- idmap->map[idmap->cnt].old = id;
- idmap->map[idmap->cnt].cur = ++env->id_gen;
- return idmap->map[idmap->cnt++].cur;
+ return bpf_id_is_xreg(id) ? BPF_RETURNED_ID(id) : id;
}
/*
- * @caller waits at the call at @call_idx and @exit is what came out of the
- * function. Make @caller what it is after the call.
- * Returns 1 if the call has to be verified in the context of the caller.
+ * @caller waits at the call at @call_idx. @exit is the state at the exit of the
+ * function. Make @caller the state after the call.
*/
int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
const struct bpf_verifier_state *exit, int call_idx)
{
struct bpf_func_state *frame = caller->frame[caller->curframe];
const struct bpf_func_state *callee = exit->frame[0];
- struct bpf_idmap *idmap = &env->idmap_scratch;
- u32 caller_rcu = caller->active_rcu_locks, caller_preempt = caller->active_preempt_locks;
- struct bpf_reference_state *held;
+ u32 i, nregs, nheld = caller->acquired_refs;
struct bpf_reg_state *reg;
- u32 i, j, nregs, nheld, kept = 0, id;
int err;
+ /*
+ * Keep only what the function held all the time. The state at its exit looks the same if
+ * the function released a lock and acquired it again. Else verify the function in the
+ * instance of the caller.
+ */
+ if (callee->entry_refs < nheld || callee->entry_rcu_locks < caller->active_rcu_locks ||
+ callee->entry_preempt_locks < caller->active_preempt_locks) {
+ if (exit->speculative)
+ return -EACCES;
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "func#%d releases what its caller holds at insn %d\n",
+ callee->subprogno, call_idx);
+ env->subprog_info[callee->subprogno].releases_inherited = true;
+ return BPF_EXIT_NEEDS_CALLER;
+ }
if (exit->unwound && !bpf_is_unwind_goto(&env->prog->insnsi[call_idx + 1])) {
verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n",
call_idx);
@@ -11597,7 +11551,7 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
*/
if (exit->changed_pkt_data) {
env->subprog_info[callee->subprogno].changes_pkt_data = true;
- return 1;
+ return BPF_EXIT_NEEDS_CALLER;
}
for (i = 0; i < CALLER_SAVED_REGS; i++)
@@ -11606,70 +11560,20 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
bpf_mark_reg_not_init(env, &frame->stack_arg_regs[i]);
/* what is held now is what was held at the call and what the function took */
- held = caller->refs;
- nheld = caller->acquired_refs;
- caller->refs = NULL;
- caller->acquired_refs = 0;
- err = copy_reference_state(caller, exit);
- if (err) {
- kfree(held);
+ err = resize_reference_state(caller, exit->acquired_refs);
+ if (err)
return err;
- }
+ for (i = nheld; i < exit->acquired_refs; i++)
+ caller->refs[i] = exit->refs[i];
+ caller->held = exit->held;
- /*
- * The same state comes out for every call. Ids in it have to be new
- * every time, or two calls in a row return the same object.
- */
- idmap->cnt = 0;
- for (i = 0; i < nheld; i++) {
- idmap->map[idmap->cnt].old = BPF_INHERITED_ID + i;
- idmap->map[idmap->cnt++].cur = held[i].id;
- }
- /*
- * What is taken is added at the end. What is released leaves no hole. Only what the
- * function held all the time is kept: the state at its exit looks the same when it
- * released a lock and took it again, or took another lock that has no id either.
- */
- for (i = 0; i < nheld && i < callee->entry_refs; i++) {
- /* see inherited_id() */
- for (j = 0; held[j].id != held[i].id; j++)
- ;
- id = held[i].id ? BPF_INHERITED_ID + j : 0;
- if (caller->refs[i].id != id || caller->refs[i].type != held[i].type)
- break;
- caller->refs[i] = held[i];
- kept++;
- }
- kfree(held);
- /* What it means for the caller is found out in the context of the caller. */
- if (kept != nheld || callee->entry_rcu_locks < caller_rcu ||
- callee->entry_preempt_locks < caller_preempt) {
- if (env->log.level & BPF_LOG_LEVEL2)
- verbose(env, "func#%d releases what its caller holds at insn %d\n",
- callee->subprogno, call_idx);
- env->subprog_info[callee->subprogno].releases_inherited = true;
- return 1;
- }
nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1;
for (i = 0; i < nregs; i++) {
reg = &frame->regs[ret_regs[i]];
*reg = callee->regs[ret_regs[i]];
- reg->id = fresh_id(env, idmap, reg->id);
- reg->parent_id = fresh_id(env, idmap, reg->parent_id);
- reg->map_uid = fresh_id(env, idmap, reg->map_uid);
- }
- /* those that were held are the first */
- for (i = nheld; i < caller->acquired_refs; i++) {
- caller->refs[i].id = fresh_id(env, idmap, caller->refs[i].id);
- if (caller->refs[i].type == REF_TYPE_PTR)
- caller->refs[i].parent_id = fresh_id(env, idmap, caller->refs[i].parent_id);
- }
- caller->active_irq_id = fresh_id(env, idmap, caller->active_irq_id);
- caller->active_lock_id = fresh_id(env, idmap, caller->active_lock_id);
- /* fresh_id() had no room for an id, or there are no ids left */
- if (idmap->cnt == BPF_ID_MAP_SIZE || env->id_gen >= BPF_MAX_ID) {
- verbose(env, "too many ids at the call at insn %d\n", call_idx);
- return -E2BIG;
+ reg->id = returned_id(reg->id);
+ reg->parent_id = returned_id(reg->parent_id);
+ reg->map_uid = returned_id(reg->map_uid);
}
return 0;
}
@@ -12029,7 +11933,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
mark_map_callback_reg(&callee->regs[BPF_REG_2], &caller->regs[BPF_REG_1], PTR_TO_MAP_KEY);
mark_map_callback_reg(&callee->regs[BPF_REG_3], &caller->regs[BPF_REG_1], PTR_TO_MAP_VALUE);
- callee->regs[BPF_REG_3].id = ++env->id_gen;
+ callee->regs[BPF_REG_3].id = bpf_new_id(env);
/* pointer to stack or null */
callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
@@ -12041,7 +11945,7 @@ void mark_frame_scoped_arg(struct bpf_verifier_env *env,
struct bpf_func_state *callee, u32 regno)
{
if (!callee->frame_id)
- callee->frame_id = ++env->id_gen;
+ callee->frame_id = bpf_new_id(env);
callee->regs[regno].id = callee->frame_id;
callee->regs[regno].parent_id = callee->frame_id;
}
@@ -12127,7 +12031,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
mark_map_callback_reg(&callee->regs[BPF_REG_1], &caller->regs[BPF_REG_1], CONST_PTR_TO_MAP);
mark_map_callback_reg(&callee->regs[BPF_REG_2], &caller->regs[BPF_REG_1], PTR_TO_MAP_KEY);
mark_map_callback_reg(&callee->regs[BPF_REG_3], &caller->regs[BPF_REG_1], PTR_TO_MAP_VALUE);
- callee->regs[BPF_REG_3].id = ++env->id_gen;
+ callee->regs[BPF_REG_3].id = bpf_new_id(env);
callee->in_async_callback_fn = true;
return 0;
@@ -12218,7 +12122,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
mark_map_callback_reg(&callee->regs[BPF_REG_1], &caller->regs[BPF_REG_3], CONST_PTR_TO_MAP);
mark_map_callback_reg(&callee->regs[BPF_REG_2], &caller->regs[BPF_REG_3], PTR_TO_MAP_KEY);
mark_map_callback_reg(&callee->regs[BPF_REG_3], &caller->regs[BPF_REG_3], PTR_TO_MAP_VALUE);
- callee->regs[BPF_REG_3].id = ++env->id_gen;
+ callee->regs[BPF_REG_3].id = bpf_new_id(env);
callee->in_async_callback_fn = true;
callee->callback_ret_range = retval_range(S32_MIN, S32_MAX);
@@ -12365,7 +12269,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
continue;
verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
state->refs[i].id, state->refs[i].insn_idx);
- bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, *insn_idx);
+ bpf_diag_leak(env, &state->refs[i], *insn_idx);
return -EINVAL;
}
@@ -12586,7 +12490,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi
continue;
verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
state->refs[i].id, state->refs[i].insn_idx);
- bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx);
+ bpf_diag_leak(env, &state->refs[i], env->insn_idx);
refs_lingering = true;
}
return refs_lingering ? -EINVAL : 0;
@@ -13094,7 +12998,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
if (type_may_be_null(ret_flag) ||
btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) {
- regs[BPF_REG_0].id = ++env->id_gen;
+ regs[BPF_REG_0].id = bpf_new_id(env);
}
/* requires regs[BPF_REG_0].id to be set because of the map-in-map case */
refine_map_lookup_value(®s[BPF_REG_0]);
@@ -13196,7 +13100,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
}
if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id)
- regs[BPF_REG_0].id = ++env->id_gen;
+ regs[BPF_REG_0].id = bpf_new_id(env);
if (is_ptr_cast_function(func_id) &&
find_reference_state(env->cur_state, meta.ref_obj.id)) {
@@ -14666,7 +14570,7 @@ static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id)
struct bpf_reg_state *reg;
int err;
- err = release_reference_nomark(env, id);
+ err = release_reference_nomark(env->cur_state, id);
WARN_ON_ONCE(err);
bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
@@ -16044,7 +15948,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
if (is_kfunc_ret_null(&meta)) {
regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
/* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
- regs[BPF_REG_0].id = ++env->id_gen;
+ regs[BPF_REG_0].id = bpf_new_id(env);
}
if (is_kfunc_acquire(&meta)) {
id = acquire_reference(env, insn_idx, 0);
@@ -16056,7 +15960,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}
if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id)
- regs[BPF_REG_0].id = ++env->id_gen;
+ regs[BPF_REG_0].id = bpf_new_id(env);
} else if (btf_type_is_void(t)) {
if (meta.btf == btf_vmlinux) {
if (is_bpf_obj_drop_kfunc(meta.func_id) ||
@@ -16670,7 +16574,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
dst_reg->raw = ptr_reg->raw;
if (reg_is_pkt_pointer(ptr_reg)) {
if (!known)
- dst_reg->id = ++env->id_gen;
+ dst_reg->id = bpf_new_id(env);
/*
* The range is what umax of the pointer may get to, see
* check_packet_access(). umax of the addend was added to it
@@ -16721,7 +16625,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
dst_reg->raw = ptr_reg->raw;
if (reg_is_pkt_pointer(ptr_reg)) {
if (!known)
- dst_reg->id = ++env->id_gen;
+ dst_reg->id = bpf_new_id(env);
/*
* Clear range if the subtrahend may be negative since
* pkt pointer could move past its bounds. A positive
@@ -18831,7 +18735,7 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
* No one could have freed the reference state before
* doing the NULL check.
*/
- WARN_ON_ONCE(__release_reference_nomark(vstate, id));
+ WARN_ON_ONCE(release_reference_nomark(vstate, id));
bpf_for_each_reg_in_vstate(vstate, state, reg, ({
mark_ptr_or_null_reg(state, reg, id, is_null);
@@ -20486,10 +20390,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
mark_reg_scratched(env, BPF_REG_0);
if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno]))
end_stack_arg_loads(env, cur_func(env));
+ /* func_is_on_its_own() uses the names of the ids */
if (bpf_is_callx(insn))
- return check_func_callx(env, insn, &env->insn_idx);
+ return bpf_renumber_ids(env, env->cur_state) ?:
+ check_func_callx(env, insn, &env->insn_idx);
if (insn->src_reg == BPF_PSEUDO_CALL)
- return check_func_call(env, insn, &env->insn_idx);
+ return bpf_renumber_ids(env, env->cur_state) ?:
+ check_func_call(env, insn, &env->insn_idx);
/* a callback is one function */
for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
err = split_func_set(env, i);
@@ -20791,11 +20698,6 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
struct bpf_insn_aux_data *insn_aux;
int err;
- /* no insn makes as many ids as there are from here to BPF_INHERITED_ID */
- if (env->id_gen >= BPF_MAX_ID) {
- verbose(env, "too many ids\n");
- return -E2BIG;
- }
err = bpf_check_limits(env);
if (err)
return err;
@@ -22131,7 +22033,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
* Take it for that of an skb, so that a write
* through it and its slices are treated with care.
*/
- __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0);
+ __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true,
+ bpf_new_id(env), 0);
} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
reg->type = PTR_TO_MEM;
reg->type |= arg->arg_type &
@@ -22139,7 +22042,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
mark_reg_known_zero(env, regs, i);
reg->mem_size = arg->mem_size;
if (arg->arg_type & PTR_MAYBE_NULL)
- reg->id = ++env->id_gen;
+ reg->id = bpf_new_id(env);
} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
reg->type = PTR_TO_BTF_ID;
if (arg->arg_type & PTR_MAYBE_NULL)
@@ -22151,7 +22054,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
mark_reg_known_zero(env, regs, i);
reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */
reg->btf_id = arg->btf_id;
- reg->id = ++env->id_gen;
+ reg->id = bpf_new_id(env);
} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
/* caller can pass either PTR_TO_ARENA or SCALAR */
mark_reg_unknown(env, regs, i);
@@ -22195,7 +22098,9 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
if (ret < 0)
goto out;
- aux->ctx_arg_info[i].ref_id = ret;
+ aux->ctx_arg_info[i].ref_arg = ret ? i + 1 : 0;
+ if (ret)
+ state->refs[state->acquired_refs - 1].ctx_arg = i + 1;
}
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 07/49] bpf: Name an id after the xreg that has it
2026-11-20 7:41 ` [PATCH bpf-next 07/49] bpf: Name an id after the xreg that has it Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 5487 bytes --]
> bpf: Name an id after the xreg that has it
TL;DR:
- Verifier log ids change (e.g. "Unreleased reference id=2" can't appear now),
so several selftest __msg() patterns likely fail; none updated
- Id naming comment doesn't match bpf_renumber_ids(), and bpf_id_is_xreg() is
true for returned ids
- reg_object_equal() is unused
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -241,6 +241,12 @@ static inline bool reg_fixed_part_equal(const struct bpf_reg_state *ra,
> return ra->type == rb->type && !memcmp(&ra->raw, &rb->raw, sizeof(ra->raw));
> }
>
> +/* That and the ids, see bpf_renumber_ids() */
> +static inline bool reg_object_equal(const struct bpf_reg_state *ra, const struct bpf_reg_state *rb)
> +{
> + return !memcmp(ra, rb, offsetof(struct bpf_reg_state, var_off));
> +}
The commit message says "Equal sets have equal ids now, and
reg_object_equal() is a memcmp()." Nothing calls reg_object_equal() at
this commit, or at the end of the series (a50a112fb866 "bpf: Delete the
equations that say nothing"). refsafe(), regsafe() and the join code that
would use it are not defined in this tree.
Should the helper be added together with its first user, or should the
changelog say that it is groundwork that is not used yet?
[ ... ]
> @@ -1048,21 +1060,28 @@ struct bpf_id_pair {
> };
>
> /*
> - * A function that is verified on its own, see func_is_on_its_own(), has no ids of its caller.
[ ... ]
> + * At a join point each id gets the number of the xreg that has it, see bpf_renumber_ids().
> + * The low bits give the origin of the id:
> + *
> + * bpf_new_id() an insn made it. It is renumbered at the end of the basic block.
> + * BPF_REF_ID() it is held. The number is the index in refs[].
> + * BPF_XREG_ID() the number is the first xreg that has it.
> + * BPF_RETURNED_ID() a function on its own returned it. No xreg of the caller has it.
[ ... ]
> + */
> +#define bpf_new_id(env) (((env)->id_gen += 4) - 3)
> +#define BPF_REF_ID(i) (((i) + 1) << 2)
> +#define BPF_XREG_ID(i) ((i) << 2 | 2)
> +#define BPF_RETURNED_ID(id) ((id) | 1)
> +
> +static inline bool bpf_id_is_xreg(u32 id)
> +{
> + return id & 2;
> +}
This comment says BPF_REF_ID() takes the index in refs[] and BPF_XREG_ID()
takes the first xreg, but bpf_renumber_ids() in kernel/bpf/states.c passes
other numbers:
renumber_id(env, &st->refs[i].id, BPF_REF_ID(2 * i));
renumber_id(env, &st->refs[i].parent_id, BPF_REF_ID(2 * i + 1));
so BPF_REF_ID(1) is the name of refs[0].parent_id, not of refs[1].id.
Likewise BPF_XREG_ID() gets 3 * xreg, 3 * xreg + 1 and 3 * xreg + 2 for
id, parent_id and map_uid, and the xreg itself is numbered by
fr * (BPF_ID_MAP_SIZE / MAX_CALL_FRAMES + 1) + ..., which differs from the
hidden registers at 11..13 and fp-8 at MAX_BPF_EXT_REG defined earlier in
the same header.
Can the refs index or the xreg be read back from a printed id the way the
comment and the commit message describe? The ids appear in "Unreleased
reference id=%d" and in the register dumps.
Also, bpf_id_is_xreg() returns true for BPF_RETURNED_ID() values (bit 1 is
set in 3 mod 4), while the comment says no xreg of the caller has those
ids. The current callers of arg_needs_caller() only become more
conservative because of it, but is that the intended meaning of the name?
[ ... ]
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -12586,7 +12490,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi
> continue;
> verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
> state->refs[i].id, state->refs[i].insn_idx);
> - bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx);
> + bpf_diag_leak(env, &state->refs[i], env->insn_idx);
> refs_lingering = true;
The id printed here has different values now. A reference id comes from
bpf_new_id(), which gives 1, 5, 9, ..., or from a BPF_REF_ID() name given
by bpf_renumber_ids(), which gives 4, 12, 20, ... So the values 2 and 3
can never be printed, but several selftests match exactly those values:
tools/testing/selftests/bpf/progs/verifier_sock.c
__msg("Unreleased reference id=2 alloc_insn=6")
tools/testing/selftests/bpf/progs/iters_state_safety.c
__msg("Unreleased reference id=2 alloc_insn=7")
tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
__msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}")
tools/testing/selftests/bpf/progs/rbtree_fail.c
__msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}")
In the sockmap case check_helper_call() gives R0 the id bpf_new_id() == 1
for PTR_MAYBE_NULL, and then acquire_reference() gives it bpf_new_id() ==
5, so the log would say "Unreleased reference id=5 alloc_insn=6" where it
used to say id=2. Other __msg() patterns in 13 progs/ files match numeric
ids such as "(id=1," and depend on the same numbering.
No commit in b96f99ad6132..a50a112fb866 touches tools/testing/selftests/bpf,
and the commit message does not mention that the log output changes. Do
these expected messages need updating?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (6 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 07/49] bpf: Name an id after the xreg that has it Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 8:39 ` sashiko-bot
2026-11-20 7:41 ` [PATCH bpf-next 09/49] bpf: Check fixups in the main pass and keep a snapshot of them Alexei Starovoitov
` (40 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
More precision must not make the result worse. If an insn is proven in
a state, it has to be proven in a state with fewer values too. Else it
is proven after a join and unproven without the join. Three checks do
not work that way.
1. Pointer arithmetic looks at the smallest offset:
r1 = 1 << 30 r1 = 0 .. 1 << 30
r0 += r1 fails r0 += r1 ok
Allow the add. bpf_blur_far_offset() extends the range of the
offset down to BPF_MAX_VAR_OFF - 1. So the offset is not a constant
that overflows an int, and an access fails. The same holds far below
the object. The number still needs a lower bound.
2. A jump can cut a range in two:
r8 = -4..3 r8 = any number
if r8 < 3 goto L -4..-1, 3 -> -4..3 3..U64_MAX
if r8 s< 0 goto L 0..3 3..S64_MAX
The small state has 0..2, the large state does not. cut_range() does
the jump two times: for -4..-1 and for 0..3. It does that only if
an error depends on the register, see bpf_xreg_in_deps().
'x != 0' is 'x u> 0', so it cuts a range too.
3. A load takes a part of a spilled pointer:
*(u64 *)(r10 - 16) = ptr
r1 = r10 - 16 + (0..7) r1 = r10 - 16 + 3
r0 = *(u8 *)(r1 + 0) ok r0 = *(u8 *)(r1 + 0) fails
With allow_ptr_leaks the constant offset takes the path of the
variable offset: the result is an unknown number. An atomic
read-modify-write still needs the whole slot.
do_check() has only a range for a far offset. So const_fold.c does not
follow a far pointer to a map value.
A far pointer to the packet can wrap around:
- It gets a new id and loses its range.
bpf_blur_far_offset() looks at the source and the result of the add.
- A compare with the end of the packet leaks the address. Without
allow_ptr_leaks, compare only offsets from -MAX_PACKET_OFF to
2 * MAX_PACKET_OFF. A header and a 16-bit length fit.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 25 ++++++
kernel/bpf/const_fold.c | 3 +
kernel/bpf/liveness.c | 5 +-
kernel/bpf/verifier.c | 167 ++++++++++++++++++++++++-----------
4 files changed, 146 insertions(+), 54 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 5886cc48653..333ab842333 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1179,10 +1179,13 @@ struct bpf_verifier_env {
bool strict_alignment; /* perform strict pointer alignment checks */
bool test_reg_invariants; /* fail verification on register invariants violations */
bool uses_arena_sp; /* the program has insns with BPF_REG_ARENA_SP */
+ bool path_sensitive; /* no joins, see run_path_sensitive() */
u64 cost, mem; /* see bpf_check_limits() */
u64 max_cost, max_mem;
u64 pass_max_cost; /* see bpf_undo_precision() */
bool aborted; /* a thaw does not take it back */
+ int err; /* the first error */
+ bool imprecise; /* a state before the first error is imprecise */
bool over_limit; /* see bpf_check_limits() */
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_loop_gets_guards() */
@@ -1741,6 +1744,27 @@ static inline bool type_may_be_null(u32 type)
return type & PTR_MAYBE_NULL;
}
+/* A number, or a pointer that stands for the offsets in its range */
+static inline bool bpf_reg_has_range(const struct bpf_reg_state *reg)
+{
+ switch (base_type(reg->type)) {
+ case SCALAR_VALUE:
+ case PTR_TO_MAP_KEY:
+ case PTR_TO_MAP_VALUE:
+ case PTR_TO_STACK:
+ case PTR_TO_PACKET:
+ case PTR_TO_PACKET_META:
+ case PTR_TO_MEM:
+ case PTR_TO_BUF:
+ case PTR_TO_TP_BUFFER:
+ case PTR_TO_BTF_ID:
+ case PTR_TO_INSN:
+ return !type_may_be_null(reg->type);
+ default:
+ return false;
+ }
+}
+
static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
{
env->scratched_regs |= 1U << regno;
@@ -2024,6 +2048,7 @@ bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env,
int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env);
bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a,
const struct bpf_reg_state *reg_b);
+void bpf_blur_far_offset(struct bpf_verifier_env *env, struct bpf_reg_state *reg);
void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax);
void scalar_set_unknown(struct scalar *s);
void scalar_set_const(struct scalar *s, u64 imm);
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index 50179edbf34..61e22636ff8 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -214,6 +214,9 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
ci_out[r] = unknown;
break;
}
+ /* do_check() has only a range for a far offset */
+ if (ci_is_map_value(dst) && abs((s64)dst->val) >= BPF_MAX_VAR_OFF)
+ *dst = unknown;
}
/* Join function: merge output state into a successor's input state. */
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 6e6b2c6d88c..e488ab97131 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -596,9 +596,8 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp
* fill a pointer, and a helper, a kfunc or a global subprog that gets a pointer to the stack reads
* memory. Such a read keeps the bytes of the slot alive, not what the verifier knows about them.
*
- * A short load at an offset that is known here counts as a fill: the main pass rejects it when it
- * finds a pointer. If only the main pass knows the offset, the load doesn't count. It is rejected
- * too, or it reads unknown bytes if the state was cleaned between the spill and the load.
+ * A short load at an offset that is known here counts as a fill: it gets a part of a number.
+ * If only the main pass knows the offset, the load doesn't count.
*/
bool bpf_stack_slot_filled(struct bpf_verifier_env *env, u32 frameno, u32 half_spi)
{
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 030263d01e3..ef8e45a399c 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -197,6 +197,7 @@ struct bpf_verifier_stack_elem {
int prev_insn_idx;
/* it goes there only if may_goto in front of prev_insn_idx does, see check_loop_guard() */
enum bpf_guard_edge guard_edge;
+ bool again; /* see bpf_flow_deliver() */
struct bpf_verifier_stack_elem *next;
};
@@ -346,6 +347,11 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env,
verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);
}
+static bool offset_is_far(const struct bpf_reg_state *reg)
+{
+ return reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF;
+}
+
static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
{
enum bpf_reg_type type;
@@ -358,7 +364,7 @@ static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_stat
* The types below guarantee a non-NULL base, an unbounded offset can
* still wrap base + offset to zero.
*/
- if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF)
+ if (offset_is_far(reg))
return false;
type = base_type(type);
@@ -4131,6 +4137,7 @@ static int check_stack_read(struct bpf_verifier_env *env,
int dst_regno)
{
struct bpf_func_state *state = bpf_func(env, reg);
+ struct bpf_stack_state *ss;
int err;
/* Some accesses are only permitted with a static offset. */
bool var_off = !tnum_is_const(reg->var_off);
@@ -4164,6 +4171,11 @@ static int check_stack_read(struct bpf_verifier_env *env,
* just checking it here would be insufficient as speculative stack
* writes could still lead to unsafe speculative behaviour.
*/
+ if (!var_off && dst_regno >= 0 && size != BPF_REG_SIZE && env->allow_ptr_leaks) {
+ ss = bpf_stack_slot(state, (-off - reg->var_off.value - 1) / BPF_REG_SIZE);
+ /* a part of a pointer is a number, as with a variable offset */
+ var_off = bpf_is_spilled_reg(ss) && !bpf_is_spilled_scalar_reg(ss);
+ }
if (!var_off) {
off += reg->var_off.value;
err = check_stack_read_fixed_off(env, state, off, size,
@@ -6940,8 +6952,7 @@ static int check_stack_access_within_bounds(
min_off = (s64)reg->var_off.value + off;
max_off = min_off + access_size;
} else {
- if (reg_smax(reg) >= BPF_MAX_VAR_OFF ||
- reg_smin(reg) <= -BPF_MAX_VAR_OFF) {
+ if (offset_is_far(reg)) {
verbose(env, "invalid unbounded variable-offset%s stack %s\n",
err_extra, reg_arg_name(env, argno));
return -EACCES;
@@ -16026,7 +16037,7 @@ static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
s64 val = reg->var_off.value;
s64 smin = reg_smin(reg);
- if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
+ if (known && val <= -BPF_MAX_VAR_OFF) {
verbose(env, "math between %s pointer and %lld is not allowed\n",
reg_type_str(env, type), val);
return false;
@@ -16038,7 +16049,7 @@ static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
return false;
}
- if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
+ if (smin <= -BPF_MAX_VAR_OFF) {
verbose(env, "value %lld makes %s pointer be out of bounds\n",
smin, reg_type_str(env, type));
return false;
@@ -16047,29 +16058,6 @@ static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
return true;
}
-static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env,
- const struct bpf_reg_state *reg,
- enum bpf_reg_type type)
-{
- bool known = tnum_is_const(reg->var_off);
- s64 val = reg->var_off.value;
- s64 smin = reg_smin(reg);
-
- if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
- verbose(env, "%s pointer offset %lld is not allowed\n",
- reg_type_str(env, type), val);
- return false;
- }
-
- if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
- verbose(env, "%s pointer offset %lld is not allowed\n",
- reg_type_str(env, type), smin);
- return false;
- }
-
- return true;
-}
-
enum {
REASON_BOUNDS = -1,
REASON_TYPE = -2,
@@ -16078,6 +16066,27 @@ enum {
REASON_STACK = -5,
};
+/*
+ * A far pointer is no error: a state with near offsets gets here, so this one does too.
+ * An access through it is unproven. But a constant offset has to fit in an int.
+ */
+void bpf_blur_far_offset(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
+{
+ s64 smin = reg_smin(reg), smax = reg_smax(reg);
+
+ /* an id and a range need an offset that does not wrap around */
+ if (reg_is_pkt_pointer(reg) && offset_is_far(reg)) {
+ reg->id = bpf_new_id(env);
+ reg->range = 0;
+ }
+ if (reg->type == SCALAR_VALUE || (smin < BPF_MAX_VAR_OFF && smax > -BPF_MAX_VAR_OFF))
+ return;
+ scalar_set_unknown(scalar_of(reg));
+ reg->r64 = cnum64_from_srange(min(smin, BPF_MAX_VAR_OFF - 1),
+ max(smax, 1 - BPF_MAX_VAR_OFF));
+ reg_bounds_sync(reg);
+}
+
static int retrieve_ptr_limit(const struct bpf_verifier_env *env,
const struct bpf_reg_state *ptr_reg,
u32 *alu_limit, bool mask_to_left)
@@ -16517,6 +16526,8 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
*/
if (dst_reg != ptr_reg)
*dst_reg = *ptr_reg;
+ /* a far pointer that comes back leaves its copies */
+ bpf_blur_far_offset(env, dst_reg);
/*
* Accesses to untrusted PTR_TO_MEM are done through probe
@@ -16527,23 +16538,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) {
reason = bpf_diag_fmt(
- env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].",
- ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF);
+ env, "The scalar offset used with R%d can be -%u or less.",
+ ptr_regno, BPF_MAX_VAR_OFF);
bpf_diag_register_type(
env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason,
"Clamp or bounds-check the scalar offset before applying it to the pointer.");
return -EINVAL;
}
- if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) {
- reason = bpf_diag_fmt(
- env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.",
- ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF,
- bpf_diag_reg_type_plain(env, ptr_reg->type));
- bpf_diag_register_type(
- env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason,
- "Keep the base pointer within the verifier's allowed offset range before applying more arithmetic.");
- return -EINVAL;
- }
if (sanitize_needed(opcode)) {
ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
@@ -16666,16 +16667,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
return -EACCES;
}
- if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) {
- reason = bpf_diag_fmt(
- env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.",
- dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF,
- bpf_diag_reg_type_plain(env, ptr_reg->type));
- bpf_diag_register_type(
- env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason,
- "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range.");
- return -EINVAL;
- }
/*
* A packet pointer that keeps its id or range is checked against a
* range set from the checked pointer's umax, so var_off must not tighten
@@ -16692,6 +16683,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
reg_bounds_sync(dst_reg);
bpf_trim_pkt_range(dst_reg, smax);
}
+ bpf_blur_far_offset(env, dst_reg);
bounds_ret = sanitize_check_bounds(env, insn, dst_reg);
if (bounds_ret == -EACCES)
return bounds_ret;
@@ -18930,9 +18922,70 @@ static int refine_live_branch(struct bpf_verifier_env *env, struct bpf_insn *ins
return 0;
}
+/* How many values of @reg are before the end of the order of @insn? 0 if all of them are. */
+static u64 values_to_end(const struct bpf_insn *insn, const struct bpf_reg_state *reg)
+{
+ u8 op = BPF_OP(insn->code);
+ bool sign = op == BPF_JSGT || op == BPF_JSGE || op == BPF_JSLT || op == BPF_JSLE;
+ u64 first = sign ? S64_MIN : 0;
+ u32 cnt32 = (first >> 32) - reg->r32.base;
+ u64 cnt = first - reg->r64.base;
+
+ if (BPF_CLASS(insn->code) == BPF_JMP32)
+ return cnt32 <= reg->r32.size && reg->r32.size != U32_MAX ? cnt32 : 0;
+ return cnt <= reg->r64.size && reg->r64.size != U64_MAX ? cnt : 0;
+}
+
+/*
+ * A jump can cut a range in two: 'x u>= 3' leaves -4..-1 and 3 of -4..3. One range has both and
+ * all values between them. So a state that knows less is proven where this one is not.
+ * Do the jump for the values before the end of the order, then again for the values after it.
+ * Returns 1 if the current state has fewer values now.
+ */
+static int cut_range(struct bpf_verifier_env *env, const struct bpf_insn *insn, u32 regno,
+ const struct bpf_reg_state *taken, const struct bpf_reg_state *not_taken)
+{
+ struct bpf_verifier_state *st = env->cur_state, *other;
+ struct bpf_reg_state *reg = &cur_regs(env)[regno], front = *reg, back = *reg;
+ u64 cnt = values_to_end(insn, reg);
+ u8 op = BPF_OP(insn->code);
+ const struct bpf_reg_state *equal = op == BPF_JEQ ? taken : not_taken;
+
+ /* 'x != 0' is 'x u> 0' */
+ if ((op == BPF_JEQ || op == BPF_JNE) && !tnum_equals_const(tnum_subreg(equal->var_off), 0))
+ return 0;
+ if (op == BPF_JSET || reg->type != SCALAR_VALUE || !cnt ||
+ !(values_to_end(insn, taken) || values_to_end(insn, not_taken)))
+ return 0;
+ env->imprecise |= !env->err;
+ if (!env->path_sensitive && !bpf_xreg_in_deps(env, st, st->curframe, regno))
+ return 0;
+ if (BPF_CLASS(insn->code) == BPF_JMP32) {
+ front.r32.size = cnt - 1;
+ back.r32.base += cnt;
+ back.r32.size -= cnt;
+ } else {
+ front.r64.size = cnt - 1;
+ back.r64.base += cnt;
+ back.r64.size -= cnt;
+ }
+ reg_bounds_sync(&front);
+ reg_bounds_sync(&back);
+ if (reg_is_empty(&front) || reg_is_empty(&back))
+ return 0;
+ other = push_stack(env, env->insn_idx, env->insn_idx, false);
+ if (IS_ERR(other))
+ return PTR_ERR(other);
+ env->head->again = true;
+ other->frame[other->curframe]->regs[regno] = back;
+ *reg = front;
+ return 1;
+}
+
static int check_cond_jmp_op(struct bpf_verifier_env *env,
struct bpf_insn *insn, int *insn_idx)
{
+ struct cnum64 near = cnum64_from_srange(-MAX_PACKET_OFF, 2 * MAX_PACKET_OFF);
struct bpf_verifier_state *this_branch = env->cur_state;
struct bpf_verifier_state *other_branch;
struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
@@ -18992,7 +19045,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
return err;
src_reg = ®s[insn->src_reg];
- if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
+ /* A header and a 16-bit length fit. If a pointer wraps, it leaks the address. */
+ if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg) &&
+ cnum64_is_subset(near, dst_reg->r64) &&
+ cnum64_is_subset(near, src_reg->r64)) &&
is_pointer_value(env, insn->src_reg)) {
verbose(env, "R%d pointer comparison prohibited\n",
insn->src_reg);
@@ -19007,6 +19063,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
}
is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
+again:
env->false_reg1 = *dst_reg;
env->false_reg2 = *src_reg;
env->true_reg1 = *dst_reg;
@@ -19048,6 +19105,14 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
return 0;
}
+ err = cut_range(env, insn, insn->dst_reg, &env->true_reg1, &env->false_reg1);
+ if (!err && BPF_SRC(insn->code) == BPF_X)
+ err = cut_range(env, insn, insn->src_reg, &env->true_reg2, &env->false_reg2);
+ if (err > 0)
+ goto again;
+ if (err)
+ return err;
+
other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false);
if (IS_ERR(other_branch))
return PTR_ERR(other_branch);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values
2026-11-20 7:41 ` [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 8:39 ` sashiko-bot
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 10349 bytes --]
> bpf: Do not fail a check because a state has fewer values
TL;DR:
- cut_range() calls undefined bpf_xreg_in_deps(), so the tree does not build
- the unread 'again' flag may let the split-off range be dropped, leaving
unchecked values
- new env fields are never set or read; two comment nits
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -1179,10 +1179,13 @@ struct bpf_verifier_env {
> bool uses_arena_sp; /* the program has insns with BPF_REG_ARENA_SP */
> + bool path_sensitive; /* no joins, see run_path_sensitive() */
> u64 cost, mem; /* see bpf_check_limits() */
> u64 max_cost, max_mem;
> u64 pass_max_cost; /* see bpf_undo_precision() */
> bool aborted; /* a thaw does not take it back */
> + int err; /* the first error */
> + bool imprecise; /* a state before the first error is imprecise */
Are these three new fields wired up anywhere?
path_sensitive is never written, and no file defines run_path_sensitive().
err is never written either, and its only use is the read in cut_range():
env->imprecise |= !env->err;
imprecise is written there, but nothing reads it. The same is true at the
end of the series (a50a112fb866). env comes from kvzalloc_obj(), so
path_sensitive is always false and err is always 0.
The commit message relies on bpf_xreg_in_deps() ("It does that only if an
error depends on the register, see bpf_xreg_in_deps()"), but git grep -w
bpf_xreg_in_deps finds only its call in cut_range(), at this commit and at
a50a112fb866. The parent commit already calls an undefined
bpf_flow_deliver(), so the tree did not build before this commit either.
Should the code that sets path_sensitive and err, reads imprecise, and
defines run_path_sensitive() and bpf_xreg_in_deps() be part of this commit,
or should the fields and the gate wait for the commit that adds it?
[ ... ]
> diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
> --- a/kernel/bpf/liveness.c
> +++ b/kernel/bpf/liveness.c
> @@ -596,9 +596,8 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp
> * fill a pointer, and a helper, a kfunc or a global subprog that gets a pointer to the stack reads
> * memory. Such a read keeps the bytes of the slot alive, not what the verifier knows about them.
> *
> - * A short load at an offset that is known here counts as a fill: the main pass rejects it when it
> - * finds a pointer. If only the main pass knows the offset, the load doesn't count. It is rejected
> - * too, or it reads unknown bytes if the state was cleaned between the spill and the load.
> + * A short load at an offset that is known here counts as a fill: it gets a part of a number.
> + * If only the main pass knows the offset, the load doesn't count.
> */
This isn't a bug, but the rewritten comment no longer gives a reason for the
rule it describes.
bpf_stack_slot_filled() has one caller, the pointer-cleaning branch of
__clean_func_state(), which runs only for:
env->allow_ptr_leaks && bpf_is_spilled_reg(slot) &&
slot->spilled_ptr.type != SCALAR_VALUE
With this commit, check_stack_read() sends every privileged short load of a
spilled non-scalar slot down the variable-offset path. There,
check_stack_range_initialized() accepts the STACK_SPILL bytes and
mark_reg_stack_read() produces an unknown scalar, which is the same result
the load gets after the slot has been cleaned to STACK_MISC. A short load
can no longer tell a kept pointer from a cleaned one, whether or not
liveness knows the offset.
The old reason was removed, and the new one ("it gets a part of a number")
describes a spilled scalar, which this query never covers. Meanwhile
liveness still makes the distinction:
info.bytes_only = class == BPF_LDX && info.size < BPF_REG_SIZE &&
!ptr->off_cnt;
A short load at a known offset still marks the slot as filled, which stops
a spilled pointer from being cleaned and costs pruning.
Should "&& !ptr->off_cnt" be dropped, so that the comment can say a short
load never fills, or does the comment need a different reason?
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -197,6 +197,7 @@ struct bpf_verifier_stack_elem {
> int prev_insn_idx;
> /* it goes there only if may_goto in front of prev_insn_idx does, see check_loop_guard() */
> enum bpf_guard_edge guard_edge;
> + bool again; /* see bpf_flow_deliver() */
> struct bpf_verifier_stack_elem *next;
> };
Is this 'again' field ever read? It is set in cut_range():
env->head->again = true;
The only place that pops these elements is flow_drain(). It passes
guard_edge on explicitly but leaves 'again' out:
if (!err && elem->guard_edge)
err = bpf_flow_guard_edge(env, &elem->st, elem->insn_idx,
elem->prev_insn_idx, elem->guard_edge);
else if (!err)
err = bpf_flow_deliver(env, &elem->st, elem->insn_idx,
elem->prev_insn_idx);
bpf_flow_deliver() cannot recover the flag with container_of(), since
do_check_block() also calls it with env->cur_state, which is not inside a
stack element. A grep of kernel/bpf and include/linux at this commit and
at a50a112fb866 finds no other reference to the field.
So the second half of a split range reaches bpf_flow_deliver() looking like
an ordinary state on an edge from the jump to itself (insn_idx ==
prev_insn_idx). The bpf_flow_deliver() removed by 32e8f52de3c8 merged an
arriving state into the states already recorded at insn_idx, and
flow_apply_exit() avoided calling it for a state that "was at the insn
before, it may be there still". A cut state is in exactly that situation,
so that could undo the cut or absorb the 'back' half.
This tree has no definition of bpf_flow_deliver(), so the effect cannot be
confirmed here. Should flow_drain() pass elem->again to the flow layer?
If insn_idx == prev_insn_idx is meant to be the signal, the field and its
comment can go.
[ ... ]
> @@ -4164,6 +4171,11 @@ static int check_stack_read(struct bpf_verifier_env *env,
> * writes could still lead to unsafe speculative behaviour.
> */
> + if (!var_off && dst_regno >= 0 && size != BPF_REG_SIZE && env->allow_ptr_leaks) {
> + ss = bpf_stack_slot(state, (-off - reg->var_off.value - 1) / BPF_REG_SIZE);
> + /* a part of a pointer is a number, as with a variable offset */
> + var_off = bpf_is_spilled_reg(ss) && !bpf_is_spilled_scalar_reg(ss);
> + }
> if (!var_off) {
This isn't a bug, but the header comment of check_stack_read_var_off() now
says the opposite of what this routing does:
* the offset is not fixed). We conservatively mark 'dst_regno' as containing
* SCALAR_VALUE. That's why we assert that the 'reg' has a variable
* offset; for a fixed offset check_stack_read_fixed_off should be used
* instead.
The new routing itself works, since check_stack_range_initialized() and
mark_reg_stack_read() handle a constant var_off. But a later reader could
add a real tnum_is_const() assertion there, or "fix" the caller back to
check_stack_read_fixed_off(), which would bring back the "invalid size of
register fill" rejection this patch removes.
Should the comment say that a constant offset into a spilled non-scalar
also comes here?
[ ... ]
> +static int cut_range(struct bpf_verifier_env *env, const struct bpf_insn *insn, u32 regno,
> + const struct bpf_reg_state *taken, const struct bpf_reg_state *not_taken)
> +{
[ ... ]
> + env->imprecise |= !env->err;
> + if (!env->path_sensitive && !bpf_xreg_in_deps(env, st, st->curframe, regno))
> + return 0;
Does kernel/bpf/verifier.c build with this? bpf_xreg_in_deps() is not
defined or declared anywhere in the tree, at this commit or at
a50a112fb866, and no commit in 8f72de1c4064..a50a112fb866 adds it.
scripts/Makefile.warn adds -Werror=implicit-function-declaration, so this
call is a build error.
The fields read on the same lines have the same problem. Nothing assigns
env->path_sensitive or env->err, so !env->path_sensitive is always true and
every split depends on the missing function. env->imprecise is written here
but never read.
The parent commit already lacks a definition of bpf_flow_deliver(), so this
may be a gap in how the series was extracted, but this commit still adds a
new call to an undefined function.
[ ... ]
> + other = push_stack(env, env->insn_idx, env->insn_idx, false);
> + if (IS_ERR(other))
> + return PTR_ERR(other);
> + env->head->again = true;
> + other->frame[other->curframe]->regs[regno] = back;
> + *reg = front;
> + return 1;
Can the 'back' half actually be explored here? cut_range() narrows the
current state to 'front' and pushes 'back' to the same conditional jump with
prev_insn_idx == insn_idx, relying on env->head->again. Nothing reads
'again', and flow_drain() delivers the pushed state like any other:
else if (!err)
err = bpf_flow_deliver(env, &elem->st, elem->insn_idx,
elem->prev_insn_idx);
bpf_flow_deliver() takes only (env, st, insn_idx, prev_insn_idx), and
struct bpf_verifier_stack_elem is private to verifier.c.
In the last bpf_flow_deliver() in history (5395aa9bbeb5, kernel/bpf/flow.c),
a delivered state goes through flow_merge(). That function frees the
arriving state when a member of the set at that insn covers it
(flow_within() -> return 1), or joins it with members. When the jump is a
flow point, the set there holds the state that arrived before the split, and
that state covers 'back'.
If delivery still works this way, 'back' is dropped or joined back into the
wide range, so values in 'back' (for example 0..3 in the commit message
example) are never checked past the jump. Or the join recreates the
crossing range and the split repeats. The self-edge prev_insn_idx ==
insn_idx also reaches the loop enter/leave logic in delivery without any
marker.
No commit in 8f72de1c4064..a50a112fb866 reads elem->again or changes
flow_drain().
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values
2026-11-20 7:41 ` [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
@ 2026-11-20 8:39 ` sashiko-bot
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:39 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [High] bpf: missing far-offset check in check_ptr_to_btf_access()
after blur
It also noticed 1 related pre-existing issue this patch does not
introduce:
- [Critical] bpf: unaligned memory access in bpf_map_direct_read() [1]
--
commit 22e172eae518ce6c856dc6c4380ea2788200c435
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Do not fail a check because a state has fewer values
This commit avoids rejecting pointer arithmetic with large positive
offsets, range-splitting conditional jumps, and narrow loads from spilled
pointers when a verifier state has a narrower set of values than a joined
state would.
> diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
> index 50179edbf348c..61e22636ff890 100644
> --- a/kernel/bpf/const_fold.c
> +++ b/kernel/bpf/const_fold.c
> @@ -214,6 +214,9 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
> ci_out[r] = unknown;
> break;
> }
> + /* do_check() has only a range for a far offset */
> + if (ci_is_map_value(dst) && abs((s64)dst->val) >= BPF_MAX_VAR_OFF)
> + *dst = unknown;
> }
[Severity: Critical]
This problem wasn't introduced by this patch, but can the BPF_LDX case in
const_reg_xfer() trigger an unaligned kernel memory access in
bpf_map_direct_read() [1]?
bpf_compute_const_regs() runs before the main verifier enforces alignment
checks. When const_reg_xfer() folds a BPF_LDX from a read-only map, it
computes off = src->val + insn->off and passes it directly to
bpf_map_direct_read(), which dereferences ptr with a direct C pointer cast
such as *(u32 *)ptr. On strict-alignment architectures, an unprivileged
user loading a program with an unaligned read offset from a read-only map
triggers an unhandled alignment fault and panics the kernel.
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 030263d01e3e2..ef8e45a399ca5 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -16078,6 +16066,27 @@ enum {
> REASON_STACK = -5,
> };
>
> +/*
> + * A far pointer is no error: a state with near offsets gets here, so this one does too.
> + * An access through it is unproven. But a constant offset has to fit in an int.
> + */
> +void bpf_blur_far_offset(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
> +{
> + s64 smin = reg_smin(reg), smax = reg_smax(reg);
> +
> + /* an id and a range need an offset that does not wrap around */
> + if (reg_is_pkt_pointer(reg) && offset_is_far(reg)) {
> + reg->id = bpf_new_id(env);
> + reg->range = 0;
> + }
> + if (reg->type == SCALAR_VALUE || (smin < BPF_MAX_VAR_OFF && smax > -BPF_MAX_VAR_OFF))
> + return;
> + scalar_set_unknown(scalar_of(reg));
> + reg->r64 = cnum64_from_srange(min(smin, BPF_MAX_VAR_OFF - 1),
> + max(smax, 1 - BPF_MAX_VAR_OFF));
> + reg_bounds_sync(reg);
> +}
[Severity: High]
Can a far PTR_TO_BTF_ID pointer pass check_ptr_to_btf_access() and
btf_struct_access() after bpf_blur_far_offset() lowers its smin to
BPF_MAX_VAR_OFF - 1?
Unlike check_stack_access_within_bounds() and check_mem_region_access(),
neither check_ptr_to_btf_access() nor btf_struct_access() checks
offset_is_far(reg). Because bpf_blur_far_offset() lowers reg_smin(reg) to
BPF_MAX_VAR_OFF - 1 with step = 1, min_off in btf_struct_access() fits in a
32-bit int even when a huge positive offset was added, and a trailing
1-byte flexible array member (such as data[] in struct
bpf_local_storage_data or ifalias[] in struct dev_ifalias) skips the
max_off upper-bound check:
kernel/bpf/btf.c:btf_struct_access() {
if (check_add_overflow(reg_smin(reg), (s64)_off, &min_off) ||
check_add_overflow(reg_smax(reg), (s64)_off, &max_off) ||
min_off != (int)min_off) {
...
return -EINVAL;
}
...
array_start = min_off - arrays[i].off;
/* An array within the flexible tail has no upper bound to exceed */
if (array_start >= flex_off && btf_type_is_array(t) &&
btf_type_array(t)->nelems == 0)
return ret;
}
For a PTR_TRUSTED or MEM_RCU pointer, bpf_may_fault_on_deref() is false, so
the load is not rewritten to BPF_PROBE_MEM and can read arbitrary
out-of-bounds kernel memory or fault in kernel mode. Checking
offset_is_far(reg) in check_ptr_to_btf_access() would reject far BTF
pointer accesses.
[1] https://sashiko.dev/#/bug/linux-ad386ec5-9f86-4c83-a85e-b56e4d8748cb
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=8
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 09/49] bpf: Check fixups in the main pass and keep a snapshot of them
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (7 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 08/49] bpf: Do not fail a check because a state has fewer values Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 10/49] bpf: Reject a program with a call that needs two rewrites Alexei Starovoitov
` (39 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The main pass writes fixups for the code that rewrites the program: the
type of a pointer at a load, the mask of an add, the depth of a stack.
The main pass will run more than once, and a join can make a path that
the program does not have:
r7, r8 = 0, 0 or 1, 5 join: r7 = 0..1, r8 = 0..5
if r7 != 0 goto out
if r8 == 0 goto out
call f only the join gets here
The next pass must not see what this path wrote. Put what a pass
writes in groups: bpf_insn_fixups and bpf_env_per_pass. save_snapshot()
copies them, with subprog_info, func_info_aux, the prog and fields of
prog->aux. Each pass starts from the snapshot.
Some checks ran after the main pass. That is too late for another pass.
Move them:
to the insn a tail call with two maps, a load of an arena
member and of something else, a missing kfunc,
bpf_override_return() and a map of programs
check_after_pass() the depth of the stack, the prologue of an
extension, r12 in a callback that is called too,
the slots of stack arguments
The check for room for the arguments of an arena stack entry goes away.
The frame is empty at the entry, so R1-R5 wait there.
An insn has one fixup for all states. A second one is an error, see
fixups_differ(). pass_cnt counts the calls of do_check(): rename it to
check_cnt.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 60 +++++----
kernel/bpf/fixups.c | 104 ++--------------
kernel/bpf/verifier.c | 227 ++++++++++++++++++++++++++++-------
3 files changed, 230 insertions(+), 161 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 333ab842333..e40e4be6cf8 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -764,6 +764,8 @@ struct bpf_loop {
};
struct bpf_insn_aux_data {
+ /* What a pass writes for the rewrite of the insn. Each pass starts from the snapshot. */
+ struct_group_tagged(bpf_insn_fixups, fixups,
union {
enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
struct bpf_map_ptr_state map_ptr_state;
@@ -795,15 +797,8 @@ struct bpf_insn_aux_data {
};
struct btf_struct_meta *kptr_struct_meta;
u64 map_key_state; /* constant (32 bit) key tracking for maps */
- /*
- * Per-instruction summary of stack slots in the current frame
- * that this instruction may write to.
- */
- DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS);
- DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS);
- u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */
int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
- u32 seen; /* this insn was processed by the verifier at env->pass_cnt */
+ u32 seen; /* this insn was processed by the verifier at env->check_cnt */
u64 nospec:1; /* do not execute this instruction speculatively */
u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */
u64 zext_dst:1; /* this insn zero extends dst reg */
@@ -813,12 +808,7 @@ struct bpf_insn_aux_data {
u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */
u64 cast_arena:1; /* addr_space_cast of an address in arena */
u64 cast_mov:1; /* addr_space_cast of a pointer that is not in arena, it's a move */
- u64 bb_end:1; /* last instruction of a basic block */
u64 loop_guard:1; /* may_goto is added in front of this instruction */
- u64 guard_place:1; /* it's the place for may_goto if the loop needs one */
- u64 guard_at_header:1; /* there is no better place than the head of the loop */
- u64 guard_no_unwind:1; /* it cannot unwind from here, at least sometimes */
- u64 guard_unset_reg:1; /* where it goes a register is read that is not set here */
u64 alu_state:5; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill
* pattern for a bpf_fastcall call.
@@ -837,20 +827,35 @@ struct bpf_insn_aux_data {
u64 arena_member:1;
u64 arena_member_nullable:1;
u64 non_arena_load:1;
+ u64 indirect_target:1; /* if it is an indirect jump target */
+ u64 non_stack_access:1; /* instruction can access non-stack memory */
+ );
+
+ /* what one pass gives to the next pass, see bpf_flow_new_pass() */
+ u32 guard_no_unwind:1; /* it cannot unwind from here, at least sometimes */
+ u32 guard_unset_reg:1; /* where it goes a register is read that is not set here */
/* below flags are initialized once */
+ u32 bb_end:1; /* last instruction of a basic block */
+ u32 guard_place:1; /* it is the place for may_goto if the loop needs one */
+ u32 guard_at_header:1; /* there is no better place than the header of the loop */
/* true if instruction is a call to a helper function that
* accepts callback function as a parameter.
*/
- u64 calls_callback:1;
- u64 indirect_target:1; /* if it is an indirect jump target */
- u64 non_stack_access:1; /* instruction can access non-stack memory */
+ u32 calls_callback:1;
/* true if some jump or call instruction targets this instruction */
- u64 jump_target:1;
+ u32 jump_target:1;
/* it can read or write the frame of a caller, in a call with a known callee */
u32 reads_outer:1;
u32 writes_outer:1;
+ /*
+ * Per-instruction summary of stack slots in the current frame
+ * that this instruction may write, may read, and that are live before it.
+ */
+ DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS);
DECLARE_BITMAP(may_read_mask, MAX_BPF_STACK_SLOTS);
+ DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS);
+ u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */
unsigned int orig_idx; /* original instruction index, initialized once */
/* registers alive before this instruction. */
@@ -1184,13 +1189,23 @@ struct bpf_verifier_env {
u64 max_cost, max_mem;
u64 pass_max_cost; /* see bpf_undo_precision() */
bool aborted; /* a thaw does not take it back */
+ /* What a pass writes. Each pass starts from the snapshot. */
+ struct_group_tagged(bpf_env_per_pass, per_pass,
int err; /* the first error */
bool imprecise; /* a state before the first error is imprecise */
bool over_limit; /* see bpf_check_limits() */
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_loop_gets_guards() */
- u32 loop_entry_cnt; /* see loop_stack_entry.entry_id, callback_entry_id */
+ bool seen_direct_write;
+ /* a register may point to the stack and is not PTR_TO_STACK, see bpf_reg_state::buf */
+ bool seen_stack_slice;
+ bool seen_exception;
+ bool sleepable_async_cb;
+ u32 id_gen; /* used to generate unique reg IDs */
+ u32 max_stack_depth; /* maximum combined stack depth */
u32 max_arena_stack_depth; /* of all call chains */
+ u32 loop_entry_cnt; /* see loop_stack_entry.entry_id */
+ );
struct bpf_verifier_state *cur_state; /* current verifier state */
struct bpf_map **used_maps; /* array of map's used by eBPF program */
struct btf_mod_pair *used_btfs; /* array of BTF's used by BPF program */
@@ -1202,7 +1217,6 @@ struct bpf_verifier_env {
u32 used_map_cnt; /* number of used maps */
u32 used_btf_cnt; /* number of used BTF objects */
u32 insn_array_map_cnt; /* number of used maps of type BPF_MAP_TYPE_INSN_ARRAY */
- u32 id_gen; /* used to generate unique reg IDs */
u32 hidden_subprog_cnt; /* number of hidden subprogs */
int exception_callback_subprog;
bool explore_alu_limits;
@@ -1215,12 +1229,8 @@ struct bpf_verifier_env {
bool bpf_capable;
bool bypass_spec_v1;
bool bypass_spec_v4;
- bool seen_direct_write;
- /* a register may point to the stack and is not PTR_TO_STACK, see bpf_reg_state::buf */
- bool seen_stack_slice;
/* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */
bool no_rcu_read_lock;
- bool seen_exception;
bool signature;
u32 insn_aux_data_len;
struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
@@ -1267,12 +1277,10 @@ struct bpf_verifier_env {
} cfg;
/* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */
struct spill_snapshot **callsite_at_stack;
- u32 pass_cnt; /* number of times do_check() was called */
+ u32 check_cnt; /* number of times do_check() was called */
u32 subprog_cnt;
/* number of instructions analyzed by the verifier */
u32 insn_processed;
- /* maximum combined stack depth */
- u32 max_stack_depth;
/* stack budget of the program, see bpf_prog_stack_limit() */
u32 stack_limit;
/* total verification time */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index bd47a38c7bc..f8f439b3e41 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -841,7 +841,7 @@ static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env)
continue;
memcpy(insn + t, &trap, sizeof(trap));
aux_data[t].zext_dst = false;
- aux_data[t].seen = env->pass_cnt;
+ aux_data[t].seen = env->check_cnt;
}
}
@@ -963,17 +963,11 @@ static int insert_arena_stack_get(struct bpf_verifier_env *env, int s)
int size = s ? BPF_REG_8 : BPF_REG_4;
struct bpf_subprog_info *sub = &env->subprog_info[s];
struct bpf_insn *buf = env->insn_buf, *patch;
- int jfree, jdone, jslow, slow, free, done, i, spill = 0;
+ int jfree, jdone, jslow, slow, free, done, i;
- if (s) {
- spill = round_up(sub->stack_depth, BPF_REG_SIZE);
- sub->stack_depth = spill + MAX_BPF_FUNC_REG_ARGS * BPF_REG_SIZE;
- if (sub->stack_depth > env->stack_limit) {
- verbose(env, "no room in the stack of %s() for its arguments\n",
- bpf_subprog_name(env, s));
- return -EACCES;
- }
- }
+ /* the arguments wait in the frame, nothing is there yet */
+ if (s)
+ sub->stack_depth = max(sub->stack_depth, MAX_BPF_FUNC_REG_ARGS * BPF_REG_SIZE);
patch = emit_arena_stack_percpu(env, buf, ptr);
*patch++ = BPF_MOV32_IMM(cnt, 1);
@@ -984,7 +978,7 @@ static int insert_arena_stack_get(struct bpf_verifier_env *env, int s)
if (!s)
*patch++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
for (i = BPF_REG_1; s && i <= BPF_REG_5; i++)
- *patch++ = BPF_STX_MEM(BPF_DW, BPF_REG_FP, i, -(spill + i * BPF_REG_SIZE));
+ *patch++ = BPF_STX_MEM(BPF_DW, BPF_REG_FP, i, -i * BPF_REG_SIZE);
*patch++ = ld_aux[0];
*patch++ = ld_aux[1];
*patch++ = BPF_MOV64_REG(BPF_REG_2, cnt);
@@ -993,7 +987,7 @@ static int insert_arena_stack_get(struct bpf_verifier_env *env, int s)
if (!s)
*patch++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
for (i = BPF_REG_1; s && i <= BPF_REG_5; i++)
- *patch++ = BPF_LDX_MEM(BPF_DW, i, BPF_REG_FP, -(spill + i * BPF_REG_SIZE));
+ *patch++ = BPF_LDX_MEM(BPF_DW, i, BPF_REG_FP, -i * BPF_REG_SIZE);
jdone = patch++ - buf;
*patch++ = BPF_MOV64_IMM(BPF_REG_0, 0);
*patch++ = BPF_EXIT_INSN();
@@ -1025,15 +1019,6 @@ int bpf_insert_arena_stack(struct bpf_verifier_env *env)
if (!env->uses_arena_sp)
return 0;
- /* the caller would get r12 of the callee back */
- for (i = 0; i < env->subprog_cnt; i++) {
- if ((i && !subprogs[i].arena_stack_entry) || !subprogs[i].is_called)
- continue;
- verbose(env, "%s() uses R%d and is both called and a callback\n",
- bpf_subprog_name(env, i), BPF_REG_ARENA_SP);
- return -EINVAL;
- }
-
/*
* With spare bytes r12 is neither the start nor the end of the stack,
* which may be at 0, so 'rX = r12' is not NULL.
@@ -1217,25 +1202,6 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
return bpf_patch_list_commit(env);
}
-/*
- * The prologue takes the context from R1, and it can clobber R2-R5. An
- * extension gets the arguments of the function that it replaces.
- */
-static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env)
-{
- const struct bpf_subprog_info *sub = &env->subprog_info[0];
- int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS);
-
- if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX)
- return false;
-
- for (i = 1; i < cnt; i++)
- if (sub->args[i].arg_type != ARG_SCALAR)
- return false;
-
- return true;
-}
-
/* convert load instructions that access fields of a context type into a
* sequence of instructions that access fields of the underlying structure:
* struct __sk_buff -> struct sk_buff
@@ -1248,7 +1214,7 @@ static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env)
* __arena arguments. A __nullable member keeps NULL as NULL, tested on the
* full kernel address, and so does a probed load, which reads as 0 when it
* faults. Returns the number of insns in @buf, 0 when the insn loads no such
- * member, or an error when it does so on some paths only.
+ * member.
*/
static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn,
struct bpf_insn *buf)
@@ -1259,10 +1225,6 @@ static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_
if (!aux->arena_member)
return 0;
- if (aux->non_arena_load) {
- verbose(env, "an insn loads an arena pointer member on one path and something else on another\n");
- return -EINVAL;
- }
base = bpf_arena_get_kern_vm_start(env->prog->aux->arena);
buf[cnt++] = *insn;
@@ -1318,17 +1280,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
verifier_bug(env, "prologue is too long");
return -EFAULT;
} else if (cnt) {
- if (env->prog->type == BPF_PROG_TYPE_EXT &&
- !ext_prog_takes_prologue(env)) {
- verbose(env,
- "Extension writing packet data must take ctx and scalars only\n");
- return -EINVAL;
- }
- if (env->prog->type == BPF_PROG_TYPE_EXT &&
- bpf_check_ext_prologue(&env->log,
- env->prog->aux->dst_prog,
- env->prog->aux->attach_btf_id))
- return -EINVAL;
env->prog->aux->has_prologue = true;
/* the prologue ends with the first insn of the program */
if (verifier_bug_if(memcmp(&insn_buf[cnt - 1], env->prog->insnsi,
@@ -1412,8 +1363,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
env->insn_aux_data[i].arena_member) {
/* an acquire load of an __arena member is rewritten like a plain load */
cnt = arena_member_fixup(env, i, insn, insn_buf);
- if (cnt < 0)
- return cnt;
insn = bpf_patch_list_add(env, i, insn_buf, cnt);
if (IS_ERR(insn))
return PTR_ERR(insn);
@@ -1511,8 +1460,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
}
cnt = arena_member_fixup(env, i, insn, insn_buf);
- if (cnt < 0)
- return cnt;
if (!cnt)
continue;
insn = bpf_patch_list_add(env, i, insn_buf, cnt);
@@ -1987,21 +1934,9 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
struct bpf_prog *prog = env->prog;
struct bpf_insn *insn = prog->insnsi;
- int depth;
+ int i, depth;
#endif
- int i, err = 0;
-
- for (i = 0; i < env->subprog_cnt; i++) {
- struct bpf_subprog_info *subprog = &env->subprog_info[i];
- u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog);
-
- if (subprog->max_out_stack_arg_cnt > outgoing) {
- verbose(env,
- "func#%d writes %u stack arg slots, but calls only require %u\n",
- i, subprog->max_out_stack_arg_cnt, outgoing);
- return -EINVAL;
- }
- }
+ int err = 0;
if (env->prog->jit_requested &&
!bpf_prog_is_offloaded(env->prog->aux)) {
@@ -2736,11 +2671,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
* index &= array->index_mask;
* to avoid out-of-bounds cpu speculation
*/
- if (bpf_map_ptr_poisoned(aux)) {
- verbose(env, "tail_call abusing map_ptr\n");
- return -EINVAL;
- }
-
map_ptr = aux->map_ptr_state.map_ptr;
insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
map_ptr->max_entries, 2);
@@ -3355,17 +3285,16 @@ int bpf_optimize_bpf_loop(struct bpf_verifier_env *env)
return bpf_patch_list_commit(env);
}
-/* Remove unnecessary spill/fill pairs, members of fastcall pattern,
- * adjust subprograms stack depth when possible.
+/*
+ * Remove unnecessary spill/fill pairs, members of fastcall pattern.
+ * check_max_stack_depth() has adjusted the stack depth of subprograms.
*/
int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env)
{
- struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn_aux_data *aux = env->insn_aux_data;
struct bpf_insn *insn = env->prog->insnsi;
int insn_cnt = env->prog->len;
u32 spills_num;
- bool modified = false;
int i, j;
for (i = 0; i < insn_cnt; i++, insn++) {
@@ -3376,13 +3305,6 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env)
*(insn - j) = NOP;
*(insn + j) = NOP;
}
- modified = true;
- }
- if ((subprog + 1)->start == i + 1) {
- if (modified && !subprog->keep_fastcall_stack)
- subprog->stack_depth = -subprog->fastcall_stack_off;
- subprog++;
- modified = false;
}
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ef8e45a399c..e43a3dd9a39 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6019,6 +6019,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
int s, ret;
find_subprog_properties(env, true);
+ /* see bpf_remove_fastcall_spills_fills() */
+ for (s = 0; s < env->subprog_cnt; s++)
+ for (int i = si[s].start; i < si[s + 1].start && !si[s].keep_fastcall_stack; i++)
+ if (env->insn_aux_data[i].fastcall_spills_num)
+ si[s].stack_depth = -si[s].fastcall_stack_off;
+
for (int i = 0; i < env->subprog_cnt; i++) {
if (si[i].has_tail_call) {
/* the stack in arena wouldn't be returned */
@@ -7501,6 +7507,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
bool allow_trust_mismatch);
+static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env);
+static int fixups_differ(struct bpf_verifier_env *env, const char *msg);
+
static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
bool strict_alignment_once, bool is_ldsx,
bool allow_trust_mismatch, const char *ctx)
@@ -7545,12 +7554,13 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
* may share the insn, see save_aux_ptr_type(). Record a load that did
* not produce an arena pointer from an __arena member, the only load
* through a kernel struct pointer that does, next to the one
- * check_arena_member_load() records, so that arena_member_fixup() can
- * tell when the paths disagree.
+ * check_arena_member_load() records. Fail if the insn has both.
*/
if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID &&
regs[insn->dst_reg].type == PTR_TO_ARENA))
- env->insn_aux_data[env->insn_idx].non_arena_load = true;
+ cur_aux(env)->non_arena_load = true;
+ if (!err && cur_aux(env)->arena_member && cur_aux(env)->non_arena_load)
+ return fixups_differ(env, "an insn loads an arena pointer member on one path and something else on another");
err = err ?: save_aux_ptr_type(env, src_reg_type,
allow_trust_mismatch);
err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx);
@@ -8959,7 +8969,6 @@ static bool is_bpf_obj_drop_kfunc(u32 func_id);
static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id);
static bool is_bpf_rbtree_add_kfunc(u32 func_id);
static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta);
-static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env);
static int process_irq_flag(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
struct bpf_call_arg_meta *meta);
@@ -11316,7 +11325,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
if (IS_ERR(async_cb))
return PTR_ERR(async_cb);
if (is_async_cb_sleepable(env, insn))
- mark_sleepable_async_cb(env);
+ env->sleepable_async_cb = true;
callee = async_cb->frame[0];
callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1;
@@ -11611,7 +11620,7 @@ static int check_static_func_call(struct bpf_verifier_env *env, int subprog,
* This is needed so the JIT knows how much stack arg space to allocate.
* The callee can load less than its prototype has, and the caller can still store
* all of it. Count the slots of the prototype only as far as the caller has stored
- * them: compilers store in front of the call. bpf_fixup_call_args() rejects the rest.
+ * them: compilers store before the call. check_after_pass() fails for the rest.
*/
caller_info = &env->subprog_info[caller->subprogno];
callee_incoming = bpf_in_stack_arg_cnt(callee_info);
@@ -12406,6 +12415,7 @@ record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
{
struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
struct bpf_map *map = meta->map.ptr;
+ bool other;
if (func_id != BPF_FUNC_tail_call &&
func_id != BPF_FUNC_map_lookup_elem &&
@@ -12437,12 +12447,13 @@ record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
return -EACCES;
}
- if (!aux->map_ptr_state.map_ptr)
- bpf_map_ptr_store(aux, meta->map.ptr,
- !meta->map.ptr->bypass_spec_v1, false);
- else if (aux->map_ptr_state.map_ptr != meta->map.ptr)
- bpf_map_ptr_store(aux, meta->map.ptr,
- !meta->map.ptr->bypass_spec_v1, true);
+ other = aux->map_ptr_state.map_ptr && aux->map_ptr_state.map_ptr != map;
+ /* the index is masked for one map */
+ if (func_id == BPF_FUNC_tail_call && other &&
+ (bpf_map_ptr_unpriv(aux) || !map->bypass_spec_v1))
+ return fixups_differ(env, "tail_call abusing map_ptr");
+ if (other || !aux->map_ptr_state.map_ptr)
+ bpf_map_ptr_store(aux, map, !map->bypass_spec_v1, other);
return 0;
}
@@ -12613,6 +12624,14 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
return &env->insn_aux_data[env->insn_idx];
}
+/* An insn has one fixup for all states. The state that was here first can be wrong too. */
+static int fixups_differ(struct bpf_verifier_env *env, const char *msg)
+{
+ verbose(env, "%s\n", msg);
+ bpf_note_fixups_differ(env);
+ return -EINVAL;
+}
+
static bool loop_flag_is_zero(struct bpf_verifier_env *env)
{
return bpf_register_is_null(reg_state(env, BPF_REG_4));
@@ -12940,6 +12959,15 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
break;
}
+ case BPF_FUNC_override_return:
+ /* see bpf_check_tail_call() */
+ for (i = 0; i < env->used_map_cnt; i++) {
+ if (!map_type_contains_progs(env->used_maps[i]))
+ continue;
+ verbose(env, "bpf_override_return() and a map of programs are used\n");
+ return -EINVAL;
+ }
+ break;
case BPF_FUNC_per_cpu_ptr:
case BPF_FUNC_this_cpu_ptr:
{
@@ -15523,9 +15551,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct btf *desc_btf;
int id;
- /* skip for now, but return error when we find this in fixup_kfunc_call */
- if (!insn->imm)
- return 0;
+ if (!insn->imm) {
+ verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
+ return -EINVAL;
+ }
err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);
if (err == -EACCES && meta.func_name) {
@@ -16290,7 +16319,7 @@ static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
* rewrite/sanitize them.
*/
if (!vstate->speculative)
- env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
+ env->insn_aux_data[env->insn_idx].seen = env->check_cnt;
}
static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason)
@@ -17642,10 +17671,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
* prevent zext from being set if the instruction has
* been previously called with non-arena registers.
*/
- if (aux->prevent_zext) {
- verbose(env, "same insn cannot be used with and without arena pointer\n");
- return -EINVAL;
- }
+ if (aux->prevent_zext)
+ return fixups_differ(env, "same insn cannot be used with and without arena pointer");
/*
* 32-bit operations zero upper bits automatically.
@@ -17661,10 +17688,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
/* Prevent the instruction from being used with arena pointers (see above). */
if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) {
- if (aux->needs_zext) {
- verbose(env, "same insn cannot be used with and without arena pointer\n");
- return -EINVAL;
- }
+ if (aux->needs_zext)
+ return fixups_differ(env, "same insn cannot be used with and without arena pointer");
aux->prevent_zext = true;
}
@@ -17937,10 +17962,8 @@ static int check_addr_space_cast(struct bpf_verifier_env *env, struct bpf_insn *
}
/* it's either wX = wY or rX = rY, see bpf_lower_cast_mov() */
- if (aux->cast_mov && aux->cast_arena) {
- verbose(env, "same addr_space_cast insn cannot be used with a number and with a pointer\n");
- return -EINVAL;
- }
+ if (aux->cast_mov && aux->cast_arena)
+ return fixups_differ(env, "same addr_space_cast insn cannot be used with a number and with a pointer");
return 0;
}
@@ -20129,8 +20152,7 @@ static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type typ
*/
*prev_type = merge_ptr_types(type, *prev_type);
} else {
- verbose(env, "same insn cannot be used with different pointers\n");
- return -EINVAL;
+ return fixups_differ(env, "same insn cannot be used with different pointers");
}
}
@@ -22026,7 +22048,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
int ret, i;
env->prev_linfo = NULL;
- env->pass_cnt++;
+ env->check_cnt++;
state = kzalloc_obj(struct bpf_verifier_state, BPF_VERIFIER_GFP);
if (!state)
@@ -22186,7 +22208,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
* Accumulate their total counts as total counts of the main or
* global subprog hosting the async call.
* Start from the saved total of earlier contexts: adding to the current
- * total would count this pass's synchronous paths twice.
+ * total would count the synchronous paths of this check twice.
*/
sub->insns_total = old_insns_total + (env->insn_processed - insn_processed);
return ret;
@@ -22277,6 +22299,134 @@ static int do_check_main(struct bpf_verifier_env *env)
return ret;
}
+/* What a pass writes outside of the states */
+struct bpf_snapshot {
+ struct bpf_env_per_pass env;
+ struct bpf_prog_aux aux;
+ struct bpf_insn_fixups *insns;
+ struct bpf_subprog_info *subprogs;
+ struct bpf_func_info_aux *funcs;
+ struct bpf_prog prog;
+};
+
+static void copy_snapshot(struct bpf_verifier_env *env, struct bpf_snapshot *snapshot, bool save)
+{
+ bool has_callchain_buf = env->prog->has_callchain_buf;
+ struct bpf_prog_aux *aux = env->prog->aux;
+ struct bpf_subprog_info *sub;
+ int i;
+
+#define COPY(kept, live) (save ? ((kept) = (live)) : ((live) = (kept)))
+#define COPY_AUX(field) COPY(snapshot->aux.field, aux->field)
+ COPY(snapshot->env, env->per_pass);
+ COPY(snapshot->prog, *env->prog);
+ /* it is taken once */
+ env->prog->has_callchain_buf = has_callchain_buf;
+ COPY_AUX(max_ctx_offset);
+ COPY_AUX(max_pkt_offset);
+ COPY_AUX(max_tp_access);
+ COPY_AUX(max_rdonly_access);
+ COPY_AUX(max_rdwr_access);
+ COPY_AUX(min_rdwr_write);
+ COPY_AUX(max_rdwr_write);
+ COPY_AUX(stack_depth);
+ COPY_AUX(tail_call_reachable);
+ COPY_AUX(jits_use_priv_stack);
+ COPY_AUX(kprobe_write_ctx);
+ for (i = 0; i < env->prog->len; i++)
+ COPY(snapshot->insns[i], env->insn_aux_data[i].fixups);
+ for (i = 0, sub = env->subprog_info; i < env->subprog_cnt; i++, sub++) {
+ /* the sum over all passes */
+ snapshot->subprogs[i].insns_total = sub->insns_total;
+ snapshot->subprogs[i].insns_self = sub->insns_self;
+ COPY(snapshot->subprogs[i], *sub);
+ }
+ for (i = 0; aux->func_info_aux && i < aux->func_info_cnt; i++)
+ COPY(snapshot->funcs[i], aux->func_info_aux[i]);
+#undef COPY_AUX
+#undef COPY
+ bpf_add_cost(env, env->prog->len, BPF_COST_XREG);
+}
+
+static struct bpf_snapshot *save_snapshot(struct bpf_verifier_env *env,
+ struct bpf_snapshot *snapshot)
+{
+ size_t insns = array_size(env->prog->len, sizeof(*snapshot->insns));
+ size_t subprogs = env->subprog_cnt * sizeof(*snapshot->subprogs);
+
+ if (!snapshot) {
+ snapshot = kvmalloc(sizeof(*snapshot) + insns + subprogs +
+ env->prog->aux->func_info_cnt * sizeof(*snapshot->funcs),
+ BPF_VERIFIER_GFP);
+ if (!snapshot)
+ return NULL;
+ env->mem += insns + subprogs;
+ snapshot->insns = (void *)(snapshot + 1);
+ snapshot->subprogs = (void *)snapshot->insns + insns;
+ snapshot->funcs = (void *)snapshot->subprogs + subprogs;
+ }
+ copy_snapshot(env, snapshot, true);
+ return snapshot;
+}
+
+static void restore_snapshot(struct bpf_verifier_env *env, struct bpf_snapshot *snapshot)
+{
+ copy_snapshot(env, snapshot, false);
+}
+
+/*
+ * The prologue takes the context from R1, and it can clobber R2-R5. An
+ * extension gets the arguments of the function that it replaces.
+ */
+static int check_ext_takes_prologue(struct bpf_verifier_env *env)
+{
+ const struct bpf_subprog_info *sub = &env->subprog_info[0];
+ int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS);
+ struct bpf_prog *prog = env->prog;
+
+ if (prog->type != BPF_PROG_TYPE_EXT || !env->ops->gen_prologue ||
+ !env->ops->gen_prologue(env->insn_buf, env->seen_direct_write, prog))
+ return 0;
+ for (i = 0; i < cnt; i++)
+ if (sub->args[i].arg_type != (i ? ARG_SCALAR : ARG_PTR_TO_CTX))
+ break;
+ if (!cnt || i < cnt) {
+ verbose(env, "Extension writing packet data must take ctx and scalars only\n");
+ return -EINVAL;
+ }
+ if (bpf_check_ext_prologue(&env->log, prog->aux->dst_prog, prog->aux->attach_btf_id))
+ return -EINVAL;
+ return 0;
+}
+
+/* What the pass found about functions and about the program has to be consistent */
+static int check_after_pass(struct bpf_verifier_env *env)
+{
+ struct bpf_subprog_info *sub = env->subprog_info;
+ u16 outgoing;
+ int i, err;
+
+ err = check_max_stack_depth(env) ?: check_ext_takes_prologue(env);
+ if (err)
+ return err;
+ for (i = 0; i < env->subprog_cnt; i++, sub++) {
+ /* the caller would get r12 of the callee back */
+ if (env->uses_arena_sp && (!i || sub->arena_stack_entry) && sub->is_called) {
+ verbose(env, "%s() uses R%d and is both called and a callback\n",
+ bpf_subprog_name(env, i), BPF_REG_ARENA_SP);
+ return -EINVAL;
+ }
+ outgoing = sub->stack_arg_cnt - bpf_in_stack_arg_cnt(sub);
+ if (sub->max_out_stack_arg_cnt > outgoing) {
+ verbose(env,
+ "func#%d writes %u stack arg slots, but calls only require %u\n",
+ i, sub->max_out_stack_arg_cnt, outgoing);
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
/*
* What it took to verify a static subprog on its own is not in insns_total of its callers.
* Add it, once per caller. The main prog and global subprogs have all of it already.
@@ -23656,11 +23806,6 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int err, desc_idx;
bool near_call;
- if (!insn->imm) {
- verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
- return -EINVAL;
- }
-
*cnt = 0;
/* insn->imm has the btf func_id. Replace it with an offset relative to
@@ -24336,16 +24481,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
ret = bpf_insert_loop_guards(env);
skip_full_check:
- /* might decrease stack depth, keep it before passes that
- * allocate additional slots.
- */
+ /* instruction rewrites happen after this point */
if (ret == 0)
ret = bpf_remove_fastcall_spills_fills(env);
- if (ret == 0)
- ret = check_max_stack_depth(env);
-
- /* instruction rewrites happen after this point */
if (ret == 0)
ret = bpf_optimize_bpf_loop(env);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 09/49] bpf: Check fixups in the main pass and keep a snapshot of them
2026-11-20 7:41 ` [PATCH bpf-next 09/49] bpf: Check fixups in the main pass and keep a snapshot of them Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 7964 bytes --]
> bpf: Check fixups in the main pass and keep a snapshot of them
TL;DR:
- Possible unsafe program free: sleepable_async_cb flag never read, so wq
callback may outlive program
- Snapshot restore may drop callx edges, letting stack depth check pass
wrongly
- check_after_pass() never called, so checks vanish; undefined function breaks
build
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
[ ... ]
> @@ -1184,13 +1189,23 @@ struct bpf_verifier_env {
> u64 max_cost, max_mem;
[ ... ]
> + bool seen_exception;
> + bool sleepable_async_cb;
> + u32 id_gen; /* used to generate unique reg IDs */
[ ... ]
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -11316,7 +11325,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
> if (IS_ERR(async_cb))
> return PTR_ERR(async_cb);
> if (is_async_cb_sleepable(env, insn))
> - mark_sleepable_async_cb(env);
> + env->sleepable_async_cb = true;
> callee = async_cb->frame[0];
Does anything read env->sleepable_async_cb? mark_sleepable_async_cb() was
the code that set prog->aux->sleepable_async_cb and ran:
for (i = 0; i < env->used_map_cnt; i++)
atomic64_inc(&env->used_maps[i]->sleepable_refcnt);
It now has no caller, and neither copy_snapshot() nor check_after_pass()
looks at the new flag.
With that, bpf_prog_runs_sleepable() returns false for a non-sleepable
program that sets a bpf_wq or bpf_task_work callback. Such a program is
freed after a plain RCU grace period, and its maps do not count it as a
sleepable user (bpf_map_fd_put_ptr(), bpf_local_storage).
bpf_wq_work() runs the callback under rcu_read_lock_trace() only, so can
the program or its JIT image be freed while the callback runs? That would
bring back the problem fixed by 1e28a5e52d32 ("bpf: Keep a program until
its bpf_wq callback has ended").
The refcounts presumably cannot be taken during a pass because a restored
snapshot cannot undo them. If so, the call to mark_sleepable_async_cb()
after the last pass when env->sleepable_async_cb is set is not documented
in the commit message, and no commit in this series range does it. Maybe
the later "bpf: Verify in passes" patch adds it?
[ ... ]
> @@ -12613,6 +12624,14 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
> return &env->insn_aux_data[env->insn_idx];
> }
>
> +/* An insn has one fixup for all states. The state that was here first can be wrong too. */
> +static int fixups_differ(struct bpf_verifier_env *env, const char *msg)
> +{
> + verbose(env, "%s\n", msg);
> + bpf_note_fixups_differ(env);
> + return -EINVAL;
> +}
bpf_note_fixups_differ() is not declared or defined anywhere in the tree,
at this commit or at a50a112fb866. Doesn't
-Werror=implicit-function-declaration make verifier.c fail to build here?
The same goes for the rest of the new code. check_after_pass(),
save_snapshot() and restore_snapshot() have no callers in this commit or
in the range, and nothing uses struct bpf_snapshot. The tree has not built
since 32e8f52de3c8 ("bpf: Delete the old main pass of the verifier"), which
says "bpf: Verify in passes" restores the build, and that patch is outside
this range. Should that patch define bpf_note_fixups_differ() and call
check_after_pass() and the snapshot helpers, or can this be arranged so
that each patch builds?
[ ... ]
> +static void copy_snapshot(struct bpf_verifier_env *env, struct bpf_snapshot *snapshot, bool save)
> +{
[ ... ]
> + for (i = 0, sub = env->subprog_info; i < env->subprog_cnt; i++, sub++) {
> + /* the sum over all passes */
> + snapshot->subprogs[i].insns_total = sub->insns_total;
> + snapshot->subprogs[i].insns_self = sub->insns_self;
> + COPY(snapshot->subprogs[i], *sub);
> + }
Is it correct to snapshot all of struct bpf_subprog_info here? That
includes callx_edge, the head of a per-caller list whose storage,
env->callx_edges, and dedup index, env->callx_index, are not snapshotted
and never reset.
record_callx_edge() returns early when the edge is already indexed:
if (bpf_index_find(&env->callx_index, key) >= 0)
return 0;
After restore_snapshot(), subprog_info[caller].callx_edge goes back to its
saved value (0 when the snapshot predates the first pass). When the next
pass reaches the same call, record_callx_edge() finds the key and does
nothing, so the edge is never linked again.
build_call_graph(), used by check_max_stack_depth() from check_after_pass(),
takes callbacks and callx targets only from si[s].callx_edge. Then the
callback is not marked CG_LIVE through its caller, check_call_depth() does
not add its frame to the caller's chain, and check_recursion() and the
tail-call marks miss paths through it. For example, a main program using
400 bytes of stack that calls bpf_loop() with a 400 byte callback could
pass the combined depth check.
The reverse also looks wrong: an edge recorded only on a path that the
snapshot is meant to forget stays in callx_index.
[ ... ]
> +/* What the pass found about functions and about the program has to be consistent */
> +static int check_after_pass(struct bpf_verifier_env *env)
> +{
> + struct bpf_subprog_info *sub = env->subprog_info;
> + u16 outgoing;
> + int i, err;
> +
> + err = check_max_stack_depth(env) ?: check_ext_takes_prologue(env);
[ ... ]
> @@ -24334,16 +24479,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
> - if (ret == 0)
> - ret = check_max_stack_depth(env);
> -
Nothing calls check_after_pass(), and this hunk drops the only other call
to check_max_stack_depth(). Doesn't that mean the following checks run
nowhere at this commit?
- the combined stack depth, check_call_depth(), check_tail_calls(),
check_callbacks(), check_exception_cb() and the private stack mode
- the removed BPF_PROG_TYPE_EXT checks in bpf_convert_ctx_accesses()
("Extension writing packet data must take ctx and scalars only" and
bpf_check_ext_prologue())
- the "func#%d writes %u stack arg slots, but calls only require %u" check
in bpf_fixup_call_args()
- the "uses R%d and is both called and a callback" check in
bpf_insert_arena_stack()
bpf_convert_ctx_accesses() still applies the prologue when gen_prologue()
returns cnt > 0. An extension replacing "int f(int x, struct __sk_buff
*skb)" that writes packet data would get a prologue that reads R1 as the
context while R1 holds a scalar.
mark_arena_stack_entries() and max_arena_stack_depth, which
bpf_insert_arena_stack() depends on, are also only reached through
check_max_stack_depth(). The stack depth shrink in
bpf_remove_fastcall_spills_fills() moved there as well, so the new comment
there, "check_max_stack_depth() has adjusted the stack depth of
subprograms", is not true at this commit.
[ ... ]
> diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
> --- a/kernel/bpf/fixups.c
> +++ b/kernel/bpf/fixups.c
> @@ -840,9 +840,9 @@ static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env)
> if (aux_data[t].seen)
> continue;
> memcpy(insn + t, &trap, sizeof(trap));
> aux_data[t].zext_dst = false;
> - aux_data[t].seen = env->pass_cnt;
> + aux_data[t].seen = env->check_cnt;
> }
> }
pass_cnt is renamed to check_cnt, but bpf_opt_hard_wire_dead_code_branches()
still has:
aux_data[i + 1].seen = env->pass_cnt;
No pass_cnt member is left in struct bpf_verifier_env, so doesn't fixups.c
fail to build here? 931721673c10 ("bpf: Put a barrier where no mask fits")
happens to rewrite that block later, so this commit and de9dd94c5b76 break
bisection.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 10/49] bpf: Reject a program with a call that needs two rewrites
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (8 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 09/49] bpf: Check fixups in the main pass and keep a snapshot of them Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 11/49] bpf: Put a barrier where no mask fits Alexei Starovoitov
` (38 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Some calls are rewritten with a number and with the special fields of
a type: bpf_obj_new(), bpf_obj_drop(), bpf_refcount_acquire(), list
push, rbtree add, per_cpu_ptr(). One state sets them for all states.
That is wrong if another state has another type:
if (...) p = a; else p = b; a has a kptr, b has none
bpf_obj_drop(p); one btf_struct_meta for both
With the fields of b, the drop of a leaks the kptr. With the fields of
a, the drop of b frees a kptr that is not there.
record_call_fixup() notes the number and the fields of the first state.
It fails if another state has other ones. Two types with equal fields
can share a call, see btf_record_equal(). bpf_refcount_acquire() reads
only the offset of the refcount, so only that has to be equal.
This rejects programs that load today. A compiler can make one call of
two calls in the source. The fix is __attribute__((nomerge)) on the
declaration of the function. A static function can get two types from
its callers. The fix is to inline it. The log says so.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 5 ++-
kernel/bpf/verifier.c | 73 ++++++++++++++++++++++++++----------
2 files changed, 58 insertions(+), 20 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index e40e4be6cf8..8d20ed2a5fd 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -794,6 +794,9 @@ struct bpf_insn_aux_data {
u64 obj_new_size;
/* remember the offset of node field within type to rewrite */
u64 insert_off;
+ /* {this,per}_cpu_ptr() with prog percpu alloc */
+ u64 call_with_percpu_alloc_ptr;
+ u64 call_num; /* any of these, or the offset of a refcount */
};
struct btf_struct_meta *kptr_struct_meta;
u64 map_key_state; /* constant (32 bit) key tracking for maps */
@@ -805,10 +808,10 @@ struct bpf_insn_aux_data {
u64 needs_zext:1; /* alu op needs to clear upper bits */
u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */
u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */
- u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */
u64 cast_arena:1; /* addr_space_cast of an address in arena */
u64 cast_mov:1; /* addr_space_cast of a pointer that is not in arena, it's a move */
u64 loop_guard:1; /* may_goto is added in front of this instruction */
+ u64 call_recorded:1; /* see record_call_fixup() */
u64 alu_state:5; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill
* pattern for a bpf_fastcall call.
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e43a3dd9a39..8de65e5a3a6 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -12409,6 +12409,36 @@ static int do_refine_retval_range(struct bpf_verifier_env *env,
return reg_bounds_sanity_check(env, ret_reg, "retval");
}
+/*
+ * The call is rewritten with @num and @meta. One call has one rewrite:
+ *
+ * if (...) p = type1; else p = type2;
+ * bpf_obj_drop(p); <- the check fails if the special fields differ
+ */
+static int record_call_fixup(struct bpf_verifier_env *env, u64 num, struct btf_struct_meta *meta)
+{
+ struct bpf_insn_aux_data *aux = cur_aux(env);
+ struct btf_struct_meta *first = aux->kptr_struct_meta;
+
+ if (!aux->call_recorded) {
+ aux->call_recorded = true;
+ aux->call_num = num;
+ aux->kptr_struct_meta = meta;
+ return 0;
+ }
+ if (aux->call_num == num &&
+ btf_record_equal(first ? first->record : NULL, meta ? meta->record : NULL))
+ return 0;
+ bpf_diag_program_structure(
+ env, env->insn_idx, "one call for two types",
+ "Use one call for each type. "
+ "If the call is in a static function, inline that function. "
+ "If a compiler merged two calls, mark the callee with __attribute__((nomerge)).",
+ "This call gets objects with different sizes, special fields or offsets, "
+ "or two kinds of percpu pointers.");
+ return fixups_differ(env, "same call cannot be used with different types");
+}
+
static int
record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
int func_id, int insn_idx)
@@ -12981,8 +13011,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
return -EFAULT;
}
returns_cpu_specific_alloc_ptr = true;
- env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;
}
+ err = record_call_fixup(env, returns_cpu_specific_alloc_ptr, NULL);
break;
}
case BPF_FUNC_user_ringbuf_drain:
@@ -15361,9 +15391,10 @@ bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_i
* < 0 - not fall-through to 'else' branch, return error
*/
static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
- struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux,
- const struct btf_type *ptr_type, struct btf *desc_btf)
+ struct bpf_reg_state *regs, const struct btf_type *ptr_type,
+ struct btf *desc_btf)
{
+ struct btf_struct_meta *struct_meta;
const struct btf_type *ret_t;
int err = 0;
@@ -15371,7 +15402,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
return 0;
if (is_bpf_obj_new_kfunc(meta->func_id) || is_bpf_percpu_obj_new_kfunc(meta->func_id)) {
- struct btf_struct_meta *struct_meta;
struct btf *ret_btf;
u32 ret_btf_id;
@@ -15447,17 +15477,21 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
if (is_bpf_percpu_obj_new_kfunc(meta->func_id))
regs[BPF_REG_0].type |= MEM_PERCPU;
- insn_aux->obj_new_size = ret_t->size;
- insn_aux->kptr_struct_meta = struct_meta;
+ err = record_call_fixup(env, ret_t->size, struct_meta);
+ if (err)
+ return err;
} else if (is_bpf_refcount_acquire_kfunc(meta->func_id)) {
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
regs[BPF_REG_0].btf = meta->arg_btf;
regs[BPF_REG_0].btf_id = meta->arg_btf_id;
- insn_aux->kptr_struct_meta =
- btf_find_struct_meta(meta->arg_btf,
- meta->arg_btf_id);
+ /* The call reads only the offset of the refcount. The other fields can differ. */
+ struct_meta = btf_find_struct_meta(meta->arg_btf, meta->arg_btf_id);
+ err = record_call_fixup(env, struct_meta->record->refcount_off,
+ cur_aux(env)->kptr_struct_meta ?: struct_meta);
+ if (err)
+ return err;
} else if (is_list_node_type(ptr_type)) {
struct btf_field *field = meta->arg_list_head.field;
@@ -15544,6 +15578,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
const struct btf_type *t, *ptr_type;
struct bpf_call_arg_meta meta;
struct bpf_insn_aux_data *insn_aux;
+ struct btf_struct_meta *struct_meta;
const char *operation;
int err, insn_idx = *insn_idx_p;
u32 i, proto_slots, ptr_type_id, ret_nregs = 1;
@@ -15661,8 +15696,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
/* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */
(prog_type == BPF_PROG_TYPE_TRACING && env->prog->expected_attach_type != BPF_TRACE_ITER
&& !env->prog->sleepable))) {
- struct btf_struct_meta *struct_meta;
-
struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
if (struct_meta && btf_record_has_nmi_unsafe_fields(struct_meta->record)) {
verbose(env, "%s cannot be used in tracing programs on types with NMI unsafe fields\n",
@@ -15778,10 +15811,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}
if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) {
- id = regs[BPF_REG_2].id;
- insn_aux->insert_off = regs[BPF_REG_2].var_off.value;
- insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
- ref_convert_owning_non_owning(env, id);
+ ref_convert_owning_non_owning(env, regs[BPF_REG_2].id);
+ struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
+ err = record_call_fixup(env, regs[BPF_REG_2].var_off.value, struct_meta);
+ if (err)
+ return err;
}
if ((is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice]) ||
@@ -15900,7 +15934,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
} else if (btf_type_is_ptr(t)) {
ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
- err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);
+ err = check_special_kfunc(env, &meta, regs, ptr_type, desc_btf);
if (err) {
if (err < 0)
return err;
@@ -16005,9 +16039,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
if (meta.btf == btf_vmlinux) {
if (is_bpf_obj_drop_kfunc(meta.func_id) ||
is_bpf_percpu_obj_drop_kfunc(meta.func_id)) {
- insn_aux->kptr_struct_meta =
- btf_find_struct_meta(meta.arg_btf,
- meta.arg_btf_id);
+ struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
+ err = record_call_fixup(env, 0, struct_meta);
+ if (err)
+ return err;
}
}
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 11/49] bpf: Put a barrier where no mask fits
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (9 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 10/49] bpf: Reject a program with a call that needs two rewrites Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number Alexei Starovoitov
` (37 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Without bypass_spec_v1 the offset of 'ptr += reg' is masked. The program
is rejected if two paths need two masks, or if no mask is possible. To
find such paths, explore_alu_limits keeps all states with other numbers
apart. That cannot work in a pass that joins all states.
Allow the insn and put a barrier before it. A mask fits if it passes
the largest offset that was checked, and no larger one:
offset 2 or 3, joined to 2..3 mask passes 0..3
two masks, mixed signs, pointer to a packet barrier
A stack pointer with a variable offset is still rejected.
An error in a speculative state is no error. The insn gets a barrier
and the path ends, see nospec_ends_path(). What the state noted at the
insn is dropped. The barrier has to stop each edge that leads there:
fall-through, jump, call, return barrier before the insn
exit of a callback --> helper barrier at the exit
may_goto --> r0 = 0; exit barrier before r0 = 0
call --> recursion in frames barrier at the call
return --> dead code 'goto -1' replaces the code
That 'goto -1' takes no zext_dst from the insn that it replaces. A jump
that is dead itself is not hard-wired: it is removed later, and the
driver of an offloaded program refuses a new target.
A speculative state starts no async callback, and it asks for no slot
of a stack argument.
This also fixes two bugs of the old code:
- If many insns replace an insn, the barrier has to go before the first
one. In an inlined bpf_loop() it was before the last one, after the
callback.
- A speculative path ends after a store to the stack, because a barrier
follows the store. It also ended after other writes to the stack: an
atomic, a call, a fastcall spill that is removed. No barrier follows
them, and the code after them was not checked. Now the path continues.
explore_alu_limits and the REASON_* errors go away. Some unprivileged
programs load now.
Rename push_budget_exit() to push_unwind_exit(): the way to stop is
'r0 = 0; unwind_exit'. budget_exit_insn() becomes emit_unwind_exit(),
and budget_can_unwind() becomes state_can_unwind(). In the next patches
a budget is a limit on the work of the verifier.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 2 +-
kernel/bpf/const_fold.c | 2 +-
kernel/bpf/fixups.c | 74 ++++++------
kernel/bpf/liveness.c | 2 +-
kernel/bpf/verifier.c | 216 ++++++++++++++++++-----------------
5 files changed, 154 insertions(+), 142 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 8d20ed2a5fd..a7b6aaa3ce8 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1222,7 +1222,6 @@ struct bpf_verifier_env {
u32 insn_array_map_cnt; /* number of used maps of type BPF_MAP_TYPE_INSN_ARRAY */
u32 hidden_subprog_cnt; /* number of hidden subprogs */
int exception_callback_subprog;
- bool explore_alu_limits;
bool allow_ptr_leaks;
/* Allow access to uninitialized stack memory. Writes with fixed offset are
* always allowed, so this refers to reads (with fixed or variable offset),
@@ -1235,6 +1234,7 @@ struct bpf_verifier_env {
/* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */
bool no_rcu_read_lock;
bool signature;
+ u8 bug:1; /* ends the load, see bpf_note_bug() */
u32 insn_aux_data_len;
struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
struct bpf_patch_list patch_list;
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index 61e22636ff8..3310a55afa3 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -269,7 +269,7 @@ int bpf_compute_const_regs(struct bpf_verifier_env *env)
ci_in[start][r] = unknown;
/*
* The subprog gets there from where it is out of budget too, see
- * push_budget_exit(). That is not an edge of the graph.
+ * push_unwind_exit(). That is not an edge of the graph.
*/
for (r = 0; r < MAX_BPF_REG && env->subprog_info[i].bail_insn >= 0; r++)
ci_in[env->subprog_info[i].bail_insn][r] = unknown;
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index f8f439b3e41..2bbacd26180 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -303,10 +303,11 @@ static void patch_list_adj_aux(struct bpf_prog *prog, struct bpf_insn_patch *p,
memcmp(&patch[p->self], orig, sizeof(*orig)))
last->non_stack_access = true;
- /* indirect jumps to the replaced insn land on p->target */
- if (last->indirect_target && p->target != p->self) {
- slot[p->target].indirect_target = 1;
- last->indirect_target = 0;
+ /* jumps to the replaced insn land on p->target, so the barrier goes before it */
+ if (p->target != p->self) {
+ slot[p->target].indirect_target = last->indirect_target;
+ slot[p->target].nospec = last->nospec;
+ last->indirect_target = last->nospec = 0;
}
}
@@ -565,21 +566,31 @@ int bpf_patch_list_commit(struct bpf_verifier_env *env)
return err;
}
-/* How a subprog leaves when it is out of budget, see push_budget_exit() */
-static struct bpf_insn budget_exit_insn(struct bpf_verifier_env *env, struct bpf_subprog_info *sub)
+/* How subprog @s leaves when it has to stop, see push_unwind_exit(). Returns the length. */
+static int emit_unwind_exit(struct bpf_verifier_env *env, int s, struct bpf_insn *buf)
{
- if (!sub->bail_unwinds)
- return BPF_EXIT_INSN();
- sub->can_unwind = true;
- env->prog->jit_required = 1;
- return BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0);
+ struct bpf_subprog_info *sub = &env->subprog_info[s];
+ int n = 0;
+
+ if (!env->bypass_spec_v1)
+ buf[n++] = BPF_ST_NOSPEC();
+ buf[n++] = BPF_MOV64_IMM(BPF_REG_0, 0);
+ if (bpf_ret_reg_pair(env, s))
+ buf[n++] = BPF_MOV64_IMM(BPF_REG_2, 0);
+ buf[n] = BPF_EXIT_INSN();
+ if (sub->bail_unwinds) {
+ buf[n].src_reg = BPF_UNWIND_EXIT;
+ sub->can_unwind = true;
+ env->prog->jit_required = 1;
+ }
+ return n + 1;
}
/*
* Add may_goto in front of the insns that the main pass marked, see
* check_loop_guard(). It verified the edge from the insn to where may_goto goes.
*
- * may_goto to or may_goto leave see push_budget_exit()
+ * may_goto to or may_goto leave see push_unwind_exit()
* insn insn
* ...
* last insn of the subprog
@@ -697,8 +708,8 @@ int bpf_insert_recursion_checks(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *si = env->subprog_info;
struct bpf_insn buf[24], nop = BPF_JMP_A(0), *insn;
+ int s, n, jmp, bail, err;
bool found = false;
- int s, n, jmp, err;
for (s = 1; s < env->subprog_cnt; s++)
found |= si[s].recursive;
@@ -719,14 +730,12 @@ int bpf_insert_recursion_checks(struct bpf_verifier_env *env)
return PTR_ERR(insn);
}
jmp = n++;
- n += emit_stack_check(buf + n, bpf_ret_reg_pair(env, s) ? 4 : 3);
+ bail = emit_unwind_exit(env, s, buf + n);
+ n += emit_stack_check(buf + n, 1 + bail);
/* The arguments are not needed any more */
buf[n++] = BPF_EMIT_KERNEL_CALL(bpf_prog_report_recursion);
buf[jmp] = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, n - jmp - 1, 0);
- buf[n++] = BPF_MOV64_IMM(BPF_REG_0, 0);
- if (bpf_ret_reg_pair(env, s))
- buf[n++] = BPF_MOV64_IMM(BPF_REG_2, 0);
- buf[n++] = budget_exit_insn(env, &si[s]);
+ n += emit_unwind_exit(env, s, buf + n);
/* calls get to the check, jumps to the first insn do not */
err = bpf_patch_list_insert(env, si[s].start, buf, n);
if (err)
@@ -784,25 +793,24 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
struct bpf_insn *insn = env->prog->insnsi;
const int insn_cnt = env->prog->len;
+ bool unwinds, calls;
int i;
for (i = 0; i < insn_cnt; i++, insn++) {
- if (!bpf_insn_is_cond_jump(insn->code))
- continue;
-
- if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) {
- /*
- * The callee never exits. Its unwind_exit needs
- * unwind_goto all the same, and the cpu predicts that it
- * returns to what follows. Don't let the removal of dead
- * code put something else there.
- */
- if (!aux_data[i + 1].seen) {
- memcpy(insn + 1, &trap, sizeof(trap));
- aux_data[i + 1].seen = env->pass_cnt;
- }
- continue;
+ unwinds = bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen;
+ calls = (bpf_pseudo_call(insn) || bpf_is_callx(insn)) && aux_data[i].seen;
+ /*
+ * The callee never exits. Its unwind_exit needs unwind_goto all the same.
+ * The cpu predicts that it returns to what follows, or speculates to an exit.
+ * Don't let the removal of dead code put something else there.
+ */
+ if ((unwinds || (calls && !env->bypass_spec_v1)) && !aux_data[i + 1].seen) {
+ memcpy(insn + 1, &trap, sizeof(trap));
+ aux_data[i + 1].zext_dst = false;
+ aux_data[i + 1].seen = env->check_cnt;
}
+ if (unwinds || !aux_data[i].seen || !bpf_insn_is_cond_jump(insn->code))
+ continue;
if (!aux_data[i + 1].seen)
ja.off = insn->off;
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index e488ab97131..ebfe1603a65 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -2974,7 +2974,7 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
*/
insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32));
}
- /* the exit that stands for 'r0 = 0; exit' may be dead code, see push_budget_exit() */
+ /* the exit that stands for 'r0 = 0; exit' may be dead code, see push_unwind_exit() */
for (i = 0; i < env->subprog_cnt; i++)
if (env->subprog_info[i].bail_insn >= 0)
insn_aux[env->subprog_info[i].bail_insn].live_regs_before |=
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 8de65e5a3a6..de3e9675e50 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -4299,7 +4299,7 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s
return err;
/* Track the max outgoing stack arg slot count. */
- if (spi + 1 > subprog->max_out_stack_arg_cnt)
+ if (spi + 1 > subprog->max_out_stack_arg_cnt && !env->cur_state->speculative)
subprog->max_out_stack_arg_cnt = spi + 1;
arg = &state->stack_arg_regs[spi];
@@ -11111,6 +11111,9 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls
state->curframe + 2);
return -E2BIG;
}
+ /* in the frames of the callers it has no end */
+ if (state->speculative && env->subprog_info[subprog].recursive)
+ return -EACCES;
if (state->frame[state->curframe + 1]) {
verifier_bug(env, "Frame %d already allocated", state->curframe + 1);
@@ -11317,6 +11320,8 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
if (bpf_is_async_callback_calling_insn(insn)) {
struct bpf_verifier_state *async_cb;
+ if (state->speculative)
+ return 0;
/* there is no real recursion here. timer and workqueue callbacks are async */
env->subprog_info[subprog].is_async_cb = true;
async_cb = push_async_cb(env, env->subprog_info[subprog].start,
@@ -12278,6 +12283,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
state->unwound = true;
}
if (callee->in_callback_fn) {
+ /* the helper continues after the barrier of its call */
+ if (state->speculative)
+ return -EACCES;
if (r0->type != SCALAR_VALUE) {
verbose(env, "R0 not a scalar value\n");
return -EACCES;
@@ -16122,14 +16130,6 @@ static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
return true;
}
-enum {
- REASON_BOUNDS = -1,
- REASON_TYPE = -2,
- REASON_PATHS = -3,
- REASON_LIMIT = -4,
- REASON_STACK = -5,
-};
-
/*
* A far pointer is no error: a state with near offsets gets here, so this one does too.
* An access through it is unproven. But a constant offset has to fit in an int.
@@ -16151,6 +16151,8 @@ void bpf_blur_far_offset(struct bpf_verifier_env *env, struct bpf_reg_state *reg
reg_bounds_sync(reg);
}
+#define NO_MASK_FITS 1
+
static int retrieve_ptr_limit(const struct bpf_verifier_env *env,
const struct bpf_reg_state *ptr_reg,
u32 *alu_limit, bool mask_to_left)
@@ -16172,11 +16174,11 @@ static int retrieve_ptr_limit(const struct bpf_verifier_env *env,
ptr_limit = mask_to_left ? reg_smin(ptr_reg) : reg_umax(ptr_reg);
break;
default:
- return REASON_TYPE;
+ return NO_MASK_FITS;
}
if (ptr_limit >= max)
- return REASON_LIMIT;
+ return NO_MASK_FITS;
*alu_limit = ptr_limit;
return 0;
}
@@ -16184,7 +16186,7 @@ static int retrieve_ptr_limit(const struct bpf_verifier_env *env,
static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
const struct bpf_insn *insn)
{
- return env->bypass_spec_v1 ||
+ return env->bypass_spec_v1 || env->path_sensitive ||
BPF_SRC(insn->code) == BPF_K ||
cur_aux(env)->nospec;
}
@@ -16198,7 +16200,7 @@ static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
if (aux->alu_state &&
(aux->alu_state != alu_state ||
aux->alu_limit != alu_limit))
- return REASON_PATHS;
+ return NO_MASK_FITS;
/* Corresponding fixup done in do_misc_fixups(). */
aux->alu_state = alu_state;
@@ -16225,6 +16227,7 @@ static bool sanitize_needed(u8 opcode)
struct bpf_sanitize_info {
struct bpf_insn_aux_data aux;
bool mask_to_left;
+ u64 max_off;
};
static int sanitize_speculative_path(struct bpf_verifier_env *env,
@@ -16256,8 +16259,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
const bool commit_window)
{
struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
- struct bpf_verifier_state *vstate = env->cur_state;
- bool off_is_imm = tnum_is_const(off_reg->var_off);
+ bool off_is_imm = cnum64_is_const(off_reg->r64);
bool off_is_neg = reg_smin(off_reg) < 0;
bool ptr_is_dst_reg = ptr_reg == dst_reg;
u8 opcode = BPF_OP(insn->code);
@@ -16268,24 +16270,18 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
if (can_skip_alu_sanitation(env, insn))
return 0;
- /* We already marked aux for masking from non-speculative
- * paths, thus we got here in the first place. We only care
- * to explore bad access from here.
- */
- if (vstate->speculative)
- goto do_sim;
-
if (!commit_window) {
- if (!tnum_is_const(off_reg->var_off) &&
+ if (!off_is_imm &&
(reg_smin(off_reg) < 0) != (reg_smax(off_reg) < 0))
- return REASON_BOUNDS;
+ return NO_MASK_FITS;
+ info->max_off = off_is_neg ? -(u64)reg_smin(off_reg) : reg_umax(off_reg);
info->mask_to_left = (opcode == BPF_ADD && off_is_neg) ||
(opcode == BPF_SUB && !off_is_neg);
}
err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left);
- if (err < 0)
+ if (err)
return err;
if (commit_window) {
@@ -16294,24 +16290,24 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
*/
alu_state = info->aux.alu_state;
alu_limit = abs(info->aux.alu_limit - alu_limit);
+ /*
+ * The mask passes what was checked: it cuts no real offset and adds no other one.
+ * It passes all smaller offsets too. The slots of the stack are not alike.
+ * A stack pointer plus a range gets here if the step makes the sum a constant.
+ */
+ if (alu_limit != info->max_off ||
+ (ptr_reg->type == PTR_TO_STACK && !(alu_state & BPF_ALU_IMMEDIATE)))
+ return NO_MASK_FITS;
} else {
alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
alu_state |= ptr_is_dst_reg ?
BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
-
- /*
- * Keep states that differ in unknown scalars apart to find
- * potential masking differences from other program paths.
- */
- if (!off_is_imm)
- env->explore_alu_limits = true;
}
err = update_alu_sanitation_state(aux, alu_state, alu_limit);
- if (err < 0)
+ if (err)
return err;
-do_sim:
/* If we're in commit phase, we're done here given we already
* pushed the truncated dst_reg into the speculative verification
* stack.
@@ -16338,7 +16334,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
}
err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);
if (err < 0)
- return REASON_STACK;
+ return err;
if (!ptr_is_dst_reg)
*dst_reg = tmp;
return 0;
@@ -16357,40 +16353,38 @@ static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
env->insn_aux_data[env->insn_idx].seen = env->check_cnt;
}
-static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason)
+/*
+ * The insn at @idx gets a barrier, and no mask. A fall-through, a jump, a call or a return to
+ * @idx runs the barrier first. So it ends a speculative state that comes that way. The other
+ * edges have their own barrier, and a speculative state does not take them:
+ *
+ * exit of a callback --> call of the helper barrier at the exit
+ * may_goto --> r0 = 0; exit see emit_unwind_exit()
+ * return --> dead code see bpf_opt_hard_wire_dead_code_branches()
+ */
+static void need_nospec(struct bpf_verifier_env *env, u32 idx)
{
- static const char *err = "pointer arithmetic with it prohibited for !root";
- const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
- u32 dst = insn->dst_reg, src = insn->src_reg;
- struct bpf_reg_state *regs = cur_regs(env);
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
- switch (reason) {
- case REASON_BOUNDS:
- verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
- regs[src].type == SCALAR_VALUE ? src : dst, err);
- break;
- case REASON_TYPE:
- verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
- regs[src].type == SCALAR_VALUE ? dst : src, err);
- break;
- case REASON_PATHS:
- verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
- dst, op, err);
- break;
- case REASON_LIMIT:
- verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
- dst, op, err);
- break;
- case REASON_STACK:
- verbose(env, "R%d could not be pushed for speculative verification, %s\n",
- dst, err);
- return -ENOMEM;
- default:
- verifier_bug(env, "unknown reason (%d)", reason);
- break;
+ /* nothing fits between it and the call */
+ if (bpf_is_unwind_goto(insn)) {
+ need_nospec(env, idx + 1);
+ return need_nospec(env, idx + insn->off + 1);
}
+ note_pessimized_insn(env, idx);
+ aux->nospec = true;
+ aux->alu_state = 0;
+}
- return -EACCES;
+static int sanitize_err(struct bpf_verifier_env *env, int err)
+{
+ if (err != NO_MASK_FITS)
+ return err;
+ if (env->cur_state->speculative)
+ return -EACCES;
+ need_nospec(env, env->insn_idx);
+ return 0;
}
/* check that stack access falls within stack limits and that 'reg' doesn't
@@ -16611,10 +16605,10 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
}
if (sanitize_needed(opcode)) {
- ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
- &info, false);
- if (ret < 0)
- return sanitize_err(env, insn, ret);
+ ret = sanitize_err(env, sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
+ &info, false));
+ if (ret)
+ return ret;
}
/*
@@ -16755,14 +16749,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
&info, true);
if (verifier_bug_if(!can_skip_alu_sanitation(env, insn)
- && !env->cur_state->speculative
&& bounds_ret
&& !ret,
env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) {
return -EFAULT;
}
- if (ret < 0)
- return sanitize_err(env, insn, ret);
+ return sanitize_err(env, ret);
}
return 0;
@@ -17653,9 +17645,9 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_i
int ret;
if (sanitize_needed(opcode)) {
- ret = sanitize_val_alu(env, insn);
- if (ret < 0)
- return sanitize_err(env, insn, ret);
+ ret = sanitize_err(env, sanitize_val_alu(env, insn));
+ if (ret)
+ return ret;
}
if ((opcode == BPF_AND || opcode == BPF_OR) && tnum_is_const(src_val.var_off)) {
@@ -20630,8 +20622,8 @@ static bool zero_is_valid_retval(struct bpf_verifier_env *env)
* @or_exit: there is no such path. 'r0 = 0; exit' has to do where the subprog
* cannot unwind, and the caller goes on with 0 in place of the result.
*/
-static bool budget_can_unwind(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
- bool or_exit)
+static bool state_can_unwind(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ bool or_exit)
{
const struct bpf_func_state *frame = st->frame[st->curframe];
const struct bpf_subprog_info *sub = &env->subprog_info[frame->subprogno];
@@ -20671,12 +20663,14 @@ static bool guard_reads_unset_reg(struct bpf_verifier_env *env,
* can do that, with exit otherwise. The insns are added when the main pass is over. Until
* then an exit that the subprog has stands for them.
*/
-static int push_budget_exit(struct bpf_verifier_env *env)
+static int push_unwind_exit(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *sub = &env->subprog_info[cur_func(env)->subprogno];
struct bpf_verifier_state *branch;
struct bpf_reg_state *regs;
+ if (env->cur_state->speculative)
+ return 0;
branch = push_stack(env, sub->bail_insn, env->insn_idx, false);
if (IS_ERR(branch))
return PTR_ERR(branch);
@@ -20752,14 +20746,14 @@ static int check_loop_guard(struct bpf_verifier_env *env)
aux->guard_no_unwind = false;
}
if (!aux->guard_no_unwind &&
- !budget_can_unwind(env, env->cur_state, aux->guard_unset_reg)) {
+ !state_can_unwind(env, env->cur_state, aux->guard_unset_reg)) {
aux->guard_no_unwind = true;
/* What would leave the loop from the states that were here is gone. All again. */
if (aux->guard_edge_dropped && loop->guarded)
env->guards_late = true;
}
if (!aux->guard_no_unwind) {
- err = push_budget_exit(env);
+ err = push_unwind_exit(env);
if (err)
return err;
env->head->guard_edge = BPF_GUARD_LEAVES_SUBPROG;
@@ -20804,7 +20798,30 @@ static int check_recursion_guard(struct bpf_verifier_env *env)
if (!sub->recursive || env->insn_idx != sub->start)
return 0;
- return push_budget_exit(env);
+ return push_unwind_exit(env);
+}
+
+static bool is_error_of_state(struct bpf_verifier_env *env, int err)
+{
+ return err != -ENOMEM && err != -EINTR && !env->over_limit && !env->bug;
+}
+
+/* An error in a speculative state is no error */
+static bool nospec_ends_path(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st, int err)
+{
+ return st->speculative && err < 0 && is_error_of_state(env, err);
+}
+
+static void free_pushed_states(struct bpf_verifier_env *env)
+{
+ struct bpf_verifier_stack_elem *elem;
+
+ while ((elem = env->head)) {
+ env->head = elem->next;
+ bpf_free_verifier_state(env, &elem->st, false);
+ kfree(elem);
+ }
}
/* Process the current state until it gets to the start of a basic block. */
@@ -20818,6 +20835,7 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
for (;;) {
struct bpf_insn *insn;
struct bpf_insn_aux_data *insn_aux;
+ struct bpf_insn_fixups fixups;
int err;
err = bpf_check_limits(env);
@@ -20918,24 +20936,23 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
if (state->speculative && insn_aux->nospec)
goto process_bpf_exit;
- err = check_loop_guard(env);
if (err)
return err;
- err = check_recursion_guard(env);
+ err = check_loop_guard(env) ?: check_recursion_guard(env);
if (err)
return err;
err = do_check_insn(env, &do_print_state);
- if (error_recoverable_with_nospec(err) && state->speculative) {
+ fixups = insn_aux->fixups;
+ if (nospec_ends_path(env, state, err)) {
/* Prevent this speculative path from ever reaching the
* insn that would have been unsafe to execute.
+ * What it noted does not apply to the other paths.
*/
- insn_aux->nospec = true;
- /* If it was an ADD/SUB insn, potentially remove any
- * markings for alu sanitization.
- */
- insn_aux->alu_state = 0;
+ insn_aux->fixups = fixups;
+ need_nospec(env, prev_insn_idx);
+ free_pushed_states(env);
goto process_bpf_exit;
} else if (err < 0) {
return err;
@@ -20946,16 +20963,9 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
env->insn_idx++;
}
- if (state->speculative && insn_aux->nospec_result) {
- /*
- * All non-branch instructions have a single fall-through
- * edge. For these, nospec_result should already work.
- */
- if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP ||
- BPF_CLASS(insn->code) == BPF_JMP32) &&
- BPF_OP(insn->code) != BPF_CALL, env,
- "speculation barrier after jump instruction may not have the desired effect"))
- return -EFAULT;
+ /* only a store that stays gets it, see bpf_convert_ctx_accesses() */
+ if (state->speculative && insn_aux->nospec_result && !insn_aux->fastcall_pattern &&
+ bpf_is_mem_insn(insn) && BPF_MODE(insn->code) == BPF_MEM) {
process_bpf_exit:
account_current_path(env);
mark_verifier_state_scratched(env);
@@ -22058,15 +22068,9 @@ static void sanitize_dead_code(struct bpf_verifier_env *env)
static void free_states(struct bpf_verifier_env *env)
{
- struct bpf_verifier_stack_elem *elem;
-
bpf_free_verifier_state(env, env->cur_state, true);
env->cur_state = NULL;
- while ((elem = env->head)) {
- env->head = elem->next;
- bpf_free_verifier_state(&elem->st, false);
- kfree(elem);
- }
+ free_pushed_states(env);
bpf_flow_reset(env);
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 11/49] bpf: Put a barrier where no mask fits
2026-11-20 7:41 ` [PATCH bpf-next 11/49] bpf: Put a barrier where no mask fits Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 10750 bytes --]
> bpf: Put a barrier where no mask fits
TL;DR:
- Possible NULL env->head dereference in check_loop_guard() for speculative
states
- Build breaks: undefined budget_exit_insn() and note_pessimized_insn();
leave[] may overflow
- ~16 unprivileged selftests not updated
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
[ ... ]
> @@ -1235,6 +1234,7 @@ struct bpf_verifier_env {
> /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */
> bool no_rcu_read_lock;
> bool signature;
> + u8 bug:1; /* ends the load, see bpf_note_bug() */
> u32 insn_aux_data_len;
The comment points to bpf_note_bug(), but no such function exists in the
tree, and nothing sets env->bug. verifier_bug() and verifier_bug_if() only
do BPF_WARN_ONCE() and bpf_log().
The only reader is is_error_of_state() in verifier.c:
return err != -ENOMEM && err != -EINTR && !env->over_limit && !env->bug;
Since env->bug is always 0, can the -EFAULT returned by verifier_bug() in a
speculative state now be swallowed? The old error_recoverable_with_nospec()
only recovered -EPERM, -EACCES and -EINVAL, so with this change the insn
gets a barrier, the path ends, and the program loads instead of being
rejected. Should the setter be added (for example in verifier_bug()), or
the field and its use dropped?
> diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
> --- a/kernel/bpf/fixups.c
> +++ b/kernel/bpf/fixups.c
[ ... ]
> @@ -565,21 +566,31 @@ int bpf_patch_list_commit(struct bpf_verifier_env *env)
> return err;
> }
>
> -/* How a subprog leaves when it is out of budget, see push_budget_exit() */
> -static struct bpf_insn budget_exit_insn(struct bpf_verifier_env *env, struct bpf_subprog_info *sub)
> +/* How subprog @s leaves when it has to stop, see push_unwind_exit(). Returns the length. */
> +static int emit_unwind_exit(struct bpf_verifier_env *env, int s, struct bpf_insn *buf)
[ ... ]
This removes budget_exit_insn(), but bpf_insert_loop_guards() in this file
still calls it:
leave[1] = BPF_MOV64_IMM(BPF_REG_2, 0);
leave[n - 1] = budget_exit_insn(env, sub);
Is there a definition of budget_exit_insn() anywhere in the tree? It looks
like this is an undeclared function returning into a struct bpf_insn, so
fixups.o would not build with CONFIG_BPF_SYSCALL=y. The call is still
there at the end of the series.
Also, the commit message says:
may_goto --> r0 = 0; exit barrier before r0 = 0
but the leave[] sequence built in bpf_insert_loop_guards() is still
'r0 = 0; [r2 = 0;] exit' with no BPF_ST_NOSPEC(). Only
bpf_insert_recursion_checks() was converted to emit_unwind_exit(). Since
push_unwind_exit() now returns without pushing for a speculative state, the
may_goto -> leave edge is not checked speculatively, so this barrier
appears to be missing.
A plain rename would not be enough either. leave[] is sized 3 and n is
'bpf_ret_reg_pair() ? 3 : 2', which is also used as the jump-back distance
passed to bpf_patch_list_jump(). With the barrier and an R2 pair that is 4
insns, so leave[] would overflow and the may_goto would land on the wrong
insn. Should leave[] be sized for emit_unwind_exit() and n taken from its
return value?
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> +static void need_nospec(struct bpf_verifier_env *env, u32 idx)
> +{
> + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
> + struct bpf_insn *insn = &env->prog->insnsi[idx];
>
> + /* nothing fits between it and the call */
> + if (bpf_is_unwind_goto(insn)) {
> + need_nospec(env, idx + 1);
> + return need_nospec(env, idx + insn->off + 1);
> }
> + note_pessimized_insn(env, idx);
> + aux->nospec = true;
> + aux->alu_state = 0;
> +}
Where is note_pessimized_insn() defined? I could not find a function or
macro with that name at this commit, at the series base or at the series
end, and the kernel is built with -Werror=implicit-function-declaration, so
verifier.c would not compile. Is a definition missing from the series?
> +static int sanitize_err(struct bpf_verifier_env *env, int err)
> +{
> + if (err != NO_MASK_FITS)
> + return err;
> + if (env->cur_state->speculative)
> + return -EACCES;
> + need_nospec(env, env->insn_idx);
> + return 0;
> +}
The messages that the old sanitize_err() printed are gone:
R%d has unknown scalar with mixed signed bounds
R%d has pointer with unsupported alu operation
R%d tried to %s from different maps, paths or scalars
R%d tried to %s beyond pointer bounds
The commit says "Some unprivileged programs load now", but nothing in the
series touches tools/testing/selftests/bpf/. About 16 existing tests still
expect __failure_unpriv and/or these __msg_unpriv() strings:
progs/verifier_bounds_deduction.c (8)
progs/verifier_value_ptr_arith.c (5)
progs/verifier_map_ptr.c (2)
progs/verifier_bounds.c (1)
For example, "check deducing bounds from const, 2" ('r1 -= r0' on ctx)
expects "R1 has pointer with unsupported alu operation" unprivileged, but
retrieve_ptr_limit() now returns NO_MASK_FITS for PTR_TO_CTX, so the insn
just gets a barrier and loads. "map access: known scalar += value_ptr
unknown vs const" expects "tried to add from different maps, paths or
scalars", and "subtraction bounds (map value) variant 2" expects "R1 has
unknown scalar with mixed signed bounds".
Should these tests be updated in the same series, so that test_progs -t
verifier does not fail?
[ ... ]
> @@ -20669,12 +20661,14 @@ static bool guard_reads_unset_reg(struct bpf_verifier_env *env,
> static int push_unwind_exit(struct bpf_verifier_env *env)
> {
> struct bpf_subprog_info *sub = &env->subprog_info[cur_func(env)->subprogno];
> struct bpf_verifier_state *branch;
> struct bpf_reg_state *regs;
>
> + if (env->cur_state->speculative)
> + return 0;
> branch = push_stack(env, sub->bail_insn, env->insn_idx, false);
[ ... ]
> @@ -20750,14 +20744,14 @@ static int check_loop_guard(struct bpf_verifier_env *env)
> if (!aux->guard_no_unwind) {
> - err = push_budget_exit(env);
> + err = push_unwind_exit(env);
> if (err)
> return err;
> env->head->guard_edge = BPF_GUARD_LEAVES_SUBPROG;
Can this dereference a NULL env->head? push_unwind_exit() now returns 0
without pushing anything for a speculative state, but check_loop_guard()
still treats 0 as "a branch was pushed" and writes env->head->guard_edge.
check_loop_guard() runs from do_check_block() before do_check_insn(), and
env->head is emptied after every insn by flow_drain() (and by
free_pushed_states()), so nothing has been pushed for the current insn at
that point. If some other element were on the list, it would get the
BPF_GUARD_LEAVES_SUBPROG mark instead.
This looks reachable. A loader with CAP_BPF but not CAP_PERFMON
(bypass_spec_v1 false) can load a program with a loop that needs a may_goto
guard. A speculative state is pushed inside the loop body by
sanitize_speculative_path() (from sanitize_ptr_alu() or from a branch whose
outcome is known). copy_func_state() copies loop_stack, so on its next
trip that state reaches the guard_place insn. In the main prog with
zero_is_valid_retval(), state_can_unwind() keeps aux->guard_no_unwind false,
and the state reaches the write above. Before this commit
push_budget_exit() pushed a branch for speculative states too.
check_loop_guard() and push_unwind_exit() are unchanged at the end of the
series. Would skipping the block for speculative states, for example
'if (!aux->guard_no_unwind && !env->cur_state->speculative)', or having
push_unwind_exit() report whether it pushed, be the right fix?
[ ... ]
> @@ -20916,24 +20934,23 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
> if (state->speculative && insn_aux->nospec)
> goto process_bpf_exit;
>
> - err = check_loop_guard(env);
> if (err)
> return err;
>
> - err = check_recursion_guard(env);
> + err = check_loop_guard(env) ?: check_recursion_guard(env);
> if (err)
> return err;
Is the first 'if (err) return err;' now dead? At that point err only holds
a value that was already checked after bpf_check_limits() and
bpf_prog_offload_verify_insn(), so it can never fire.
> err = do_check_insn(env, &do_print_state);
> - if (error_recoverable_with_nospec(err) && state->speculative) {
> + fixups = insn_aux->fixups;
> + if (nospec_ends_path(env, state, err)) {
> /* Prevent this speculative path from ever reaching the
> * insn that would have been unsafe to execute.
> + * What it noted does not apply to the other paths.
> */
> + insn_aux->fixups = fixups;
The snapshot 'fixups = insn_aux->fixups;' is taken after do_check_insn()
has run, so 'insn_aux->fixups = fixups' writes back the same bytes. Does
this restore anything? The comment ("What it noted does not apply to the
other paths") and the commit message ("What the state noted at the insn is
dropped") describe a rollback that does not happen.
Whatever the failing speculative state wrote into struct bpf_insn_fixups
before its error stays for later states and passes: ptr_type through
save_aux_ptr_type(), non_arena_load in check_load_mem() (set before
save_aux_ptr_type() can fail), map_ptr_state, nospec_result and so on.
need_nospec() only clears alu_state. Since an insn has one fixup for all
states (fixups_differ()), a value that came only from a speculative path
can make a later non-speculative state fail. For example, non_arena_load
set on the speculative path, and then a real path that loads an __arena
member fails with "an insn loads an arena pointer member on one path and
something else on another". A stale ptr_type can likewise produce "same
insn cannot be used with different pointers".
Commit 0462c546cfe6 ("bpf: Make equations from ALU, spill, fill and call
insns") later in the series moves 'fixups = insn_aux->fixups;' in front of
do_check_insn(). Should that move be part of this patch, since at this
commit the restore is a no-op?
> need_nospec(env, prev_insn_idx);
> free_pushed_states(env);
> goto process_bpf_exit;
Related to the removal of the only caller of error_recoverable_with_nospec()
here, the function stays behind as an unused static function. -Wall
reports it as "defined but not used", which is a build error with
CONFIG_WERROR=y. It is still present at the end of the series.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (10 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 11/49] bpf: Put a barrier where no mask fits Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:18 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 13/49] bpf: Remember how many bytes of a packet are checked Alexei Starovoitov
` (36 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Six transfer functions lose what is known. Scalar evolution hid a part
of that: it had expressions for extensions and for partial spills. Fix
the functions.
1. An AND that clears no bit is a move.
r0 = 1..16; w0 &= 255 was 0..16, now 1..16
2. A sign extension does not change the low bits. The other bits are
copies of the sign. If the low 32 bits fit, they give the range.
w6 = -100..100; r7 = (s32)r6 was any s32, now -100..100
3. A fill can be larger than the spill. On little endian the low bytes
are from the spill.
*(u32 *)(r10 - 8) = 5; r1 = *(u64 *)(r10 - 8) w1 is 5
4. A write above the low bytes of a spill does not change them. Keep
them as a narrow spill. Zero bytes next to it stay zero.
struct { u32 i; u32 flag; } s; s.i = r6; s.flag = 1;
5. A range that crosses S64_MAX has no signed bounds. A later patch
makes such ranges. The known bits can have signed bounds, so a signed
shift takes the tighter ones.
r7 = 0..0xf0 << 56; r7 s>>= 56 -128..112, not -128..127
6. A number can have a step: 0, 12, 24, ... The known bits tell only a
power of two. A later patch counts a loop by the distance of the
counter to its bound:
for (i = 12 * a; i != 12 * b; i += 12)
distance = i - 12 * b ..., -24, -12, 0
distance != 0 ..., -24, -12 with the step
..., -2, -1 without it
Without the step, 'i += 12' goes past the bound. Keep it:
-3 * (3 * a + 1) step 9: a negative factor
3 * a + 6 * b step 3: the gcd of the two steps
step 3 and step 6 step 6: one is a multiple of the other
As before, a result that can wrap has no step.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 1 +
kernel/bpf/verifier.c | 126 +++++++++++++++++++++++------------
2 files changed, 85 insertions(+), 42 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index a7b6aaa3ce8..e1432f48fea 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -2063,6 +2063,7 @@ void bpf_blur_far_offset(struct bpf_verifier_env *env, struct bpf_reg_state *reg
void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax);
void scalar_set_unknown(struct scalar *s);
void scalar_set_const(struct scalar *s, u64 imm);
+void scalar_step_scale(struct scalar *dst, s64 amount);
void scalar_add(struct scalar *dst, const struct scalar *a, const struct scalar *b);
void scalar_alu(struct scalar *dst, const struct scalar *src, const struct bpf_insn *insn);
bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index de3e9675e50..3229372c5f4 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1901,44 +1901,39 @@ static void scalar_step_reset(struct scalar *s)
static void scalar_step_add(struct scalar *dst, const struct scalar *a, const struct scalar *b)
{
- const struct scalar *reg;
s64 amount, tmp;
+ u16 step;
- if (tnum_is_const(b->var_off)) {
- reg = a;
- amount = (s64)b->var_off.value;
- } else if (tnum_is_const(a->var_off)) {
- reg = b;
- amount = (s64)a->var_off.value;
- } else {
- scalar_step_reset(dst);
- return;
- }
+ if (tnum_is_const(a->var_off) && !tnum_is_const(b->var_off))
+ swap(a, b);
+ /* b is d + k * step. A constant has no step. */
+ amount = tnum_is_const(b->var_off) ? b->var_off.value : b->base;
+ step = tnum_is_const(b->var_off) ? a->step : gcd(a->step, b->step);
/*
* Let x be a possible signed value of the register,
- * d = amount, s = reg->step, and b = reg->base.
+ * d = amount, s = step, and b = a->base mod s.
* Then x mod s = b, with s > 0 and 0 <= b < s.
* If the addition causes signed underflow, the resulting value
* is x + d + 2**64.
*
* Taking the result modulo s and substituting x mod s = b gives:
* (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s (1)
- * Keeping reg->step = s and updating reg->base to (b + d) mod s
+ * Keeping the step s and updating the base to (b + d) mod s
* requires (1) to equal (b + d) mod s, which holds if and only
* if 2**64 mod s = 0.
* Signed overflow subtracts 2**64 and has the same requirement.
*
* Conservatively reset the stride if either wrap is possible.
*/
- if (check_add_overflow(cnum64_smin(reg->r64), amount, &tmp) ||
- check_add_overflow(cnum64_smax(reg->r64), amount, &tmp)) {
+ if (step == 1 || check_add_overflow(cnum64_smin(a->r64), cnum64_smin(b->r64), &tmp) ||
+ check_add_overflow(cnum64_smax(a->r64), cnum64_smax(b->r64), &tmp)) {
scalar_step_reset(dst);
return;
}
- dst->base = (reg->base + imod(amount, reg->step)) % reg->step;
- dst->step = reg->step;
+ dst->base = (a->base + imod(amount, step)) % step;
+ dst->step = step;
}
/*
@@ -1953,8 +1948,10 @@ void scalar_add(struct scalar *dst, const struct scalar *a, const struct scalar
dst->var_off = tnum_add(a->var_off, b->var_off);
}
-static void scalar_step_scale(struct scalar *dst, u64 amount)
+/* The step of @dst * @amount. The ranges are those before the product. */
+void scalar_step_scale(struct scalar *dst, s64 amount)
{
+ u64 factor = amount < 0 ? -(u64)amount : amount;
u16 step;
s64 tmp;
@@ -1973,18 +1970,17 @@ static void scalar_step_scale(struct scalar *dst, u64 amount)
*
* Conservatively reset the stride if a wrap is possible.
*/
- if (amount == 0 || check_mul_overflow(dst->step, amount, &step) ||
- check_mul_overflow(cnum64_smin(dst->r64), (s64)amount, &tmp) ||
- check_mul_overflow(cnum64_smax(dst->r64), (s64)amount, &tmp)) {
+ if (amount == 0 || check_mul_overflow(dst->step, factor, &step) ||
+ check_mul_overflow(cnum64_smin(dst->r64), amount, &tmp) ||
+ check_mul_overflow(cnum64_smax(dst->r64), amount, &tmp)) {
scalar_step_reset(dst);
return;
}
/*
- * The above established that `step = dst->step * amount`
- * does not overflow, therefore the base < step invariant holds
- * after assignments below.
+ * The above established that `step = dst->step * |amount|`
+ * does not overflow. For d < 0 the same holds modulo s * -d.
*/
- dst->base = dst->base * amount;
+ dst->base = imod(dst->base * amount, step);
dst->step = step;
}
@@ -3716,11 +3712,21 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
save_register_state(env, state, spi, reg, size);
} else {
u8 type = STACK_MISC;
-
- if (bpf_is_spilled_reg(ss))
- bpf_diag_record_scrub(env, &ss->spilled_ptr,
- BPF_DIAG_MOD_WRITE);
- scrub_special_slot(state, spi);
+ int keep = 0;
+
+ /* the bytes before the write still have the low bytes of a number */
+ if (bpf_is_spilled_scalar_reg(ss) && off % BPF_REG_SIZE)
+ keep = min_t(int, rounddown_pow_of_two(off & (BPF_REG_SIZE - 1)),
+ bpf_spill_size(ss));
+ if (keep && bpf_stack_narrow_access_ok(0, keep, bpf_spill_size(ss))) {
+ for (i = 0; i < BPF_REG_SIZE - keep; i++)
+ if (ss->slot_type[i] == STACK_SPILL)
+ ss->slot_type[i] = STACK_MISC;
+ } else {
+ if (bpf_is_spilled_reg(ss))
+ bpf_diag_record_scrub(env, &ss->spilled_ptr, BPF_DIAG_MOD_WRITE);
+ scrub_special_slot(state, spi);
+ }
/* when we zero initialize stack slots mark them as such */
if ((reg && bpf_register_is_null(reg)) ||
@@ -3731,7 +3737,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
for (i = 0; i < size; i++)
ss->slot_type[(slot - i) % BPF_REG_SIZE] = type;
}
-
return 0;
}
@@ -3994,6 +3999,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
/* coerce_scalar_to_size will adjust the boundaries. */
state->regs[dst_regno] = *reg;
} else {
+ /* the low bytes are from the spill */
+ bool fill_exceeds_spill =
+ size > spill_size && !(off % BPF_REG_SIZE) &&
+ !IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
int spill_cnt = 0, zero_cnt = 0;
for (i = 0; i < size; i++) {
@@ -4033,6 +4042,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
} else {
mark_reg_stack_read(env, reg_state, off, off + size,
dst_regno);
+ if (fill_exceeds_spill)
+ scalar_intersect(scalar_of(&state->regs[dst_regno]),
+ scalar_of(reg),
+ spill_size * BITS_PER_BYTE);
}
}
} else if (dst_regno >= 0) {
@@ -6203,7 +6216,12 @@ static void set_sext64_default_val(struct scalar *s, int size)
s->r64 = cnum64_from_srange(S32_MIN, S32_MAX);
s->r32 = cnum32_from_srange(S32_MIN, S32_MAX);
}
- s->var_off = tnum_unknown;
+}
+
+/* The low @num_bits bits do not change. The other bits are copies of the sign. */
+static struct tnum tnum_sext(struct tnum t, u32 num_bits)
+{
+ return tnum_arshift(tnum_lshift(t, 64 - num_bits), 64 - num_bits, 64);
}
static void coerce_scalar_to_size_sx(struct scalar *s, int size)
@@ -6212,6 +6230,7 @@ static void coerce_scalar_to_size_sx(struct scalar *s, int size)
u64 top_smax_value, top_smin_value;
u64 num_bits = size * 8;
s64 limit = 1LL << (num_bits - 1);
+ struct cnum32 r32;
/* If every value fits the signed N-bit range sign extension is a noop. */
if (cnum64_smin(s->r64) >= -limit && cnum64_smax(s->r64) < limit)
@@ -6236,6 +6255,7 @@ static void coerce_scalar_to_size_sx(struct scalar *s, int size)
return;
}
+ s->var_off = tnum_sext(s->var_off, num_bits);
top_smax_value = ((u64)cnum64_smax(s->r64) >> num_bits) << num_bits;
top_smin_value = ((u64)cnum64_smin(s->r64) >> num_bits) << num_bits;
@@ -6261,12 +6281,19 @@ static void coerce_scalar_to_size_sx(struct scalar *s, int size)
if ((s64_max >= 0) == (s64_min >= 0)) {
s->r64 = cnum64_from_srange(s64_min, s64_max);
s->r32 = cnum32_from_srange(s64_min, s64_max);
- s->var_off = tnum_range(s64_min, s64_max);
+ scalar_bounds_sync(s);
return;
}
out:
+ r32 = s->r32;
set_sext64_default_val(s, size);
+ /* the low 32 bits fit, they do not change */
+ if (cnum32_smin(r32) >= -limit && cnum32_smax(r32) < limit) {
+ s->r32 = r32;
+ s->r64 = cnum64_from_srange(cnum32_smin(r32), cnum32_smax(r32));
+ }
+ scalar_bounds_sync(s);
}
static void set_sext32_default_val(struct scalar *s, int size)
@@ -6276,7 +6303,6 @@ static void set_sext32_default_val(struct scalar *s, int size)
else
/* size == 2 */
s->r32 = cnum32_from_srange(S16_MIN, S16_MAX);
- s->var_off = tnum_subreg(tnum_unknown);
}
static void coerce_subreg_to_size_sx(struct scalar *s, int size)
@@ -6321,6 +6347,7 @@ static void coerce_subreg_to_size_sx(struct scalar *s, int size)
return;
}
+ s->var_off = tnum_sext(s->var_off, num_bits);
top_smax_value = ((u32)cnum32_smax(s->r32) >> num_bits) << num_bits;
top_smin_value = ((u32)cnum32_smin(s->r32) >> num_bits) << num_bits;
@@ -6341,7 +6368,6 @@ static void coerce_subreg_to_size_sx(struct scalar *s, int size)
if ((s32_min >= 0) == (s32_max >= 0)) {
s->r32 = cnum32_from_srange(s32_min, s32_max);
- s->var_off = tnum_subreg(tnum_range(s32_min, s32_max));
return;
}
@@ -17297,7 +17323,9 @@ static void scalar_min_max_rsh(struct scalar *dst, const struct scalar *src)
static void scalar32_min_max_arsh(struct scalar *dst, const struct scalar *src)
{
u32 umin_val = cnum32_umin(src->r32), umax_val = cnum32_umax(src->r32);
- s32 smin = cnum32_smin(dst->r32), smax = cnum32_smax(dst->r32);
+ struct cnum32 known = cnum32_from_tnum(dst->var_off);
+ s32 smin = max(cnum32_smin(dst->r32), cnum32_smin(known));
+ s32 smax = min(cnum32_smax(dst->r32), cnum32_smax(known));
/*
* A larger count brings the result closer to 0, or to -1.
@@ -17316,7 +17344,10 @@ static void scalar32_min_max_arsh(struct scalar *dst, const struct scalar *src)
static void scalar_min_max_arsh(struct scalar *dst, const struct scalar *src)
{
u64 umin_val = cnum64_umin(src->r64), umax_val = cnum64_umax(src->r64);
- s64 smin = cnum64_smin(dst->r64), smax = cnum64_smax(dst->r64);
+ struct cnum64 known = cnum64_from_tnum(dst->var_off);
+ /* a range that crosses S64_MAX has no signed bounds, the known bits have: 0..0xf0 << 56 */
+ s64 smin = max(cnum64_smin(dst->r64), cnum64_smin(known));
+ s64 smax = min(cnum64_smax(dst->r64), cnum64_smax(known));
/* A larger count brings the result closer to 0, or to -1 */
dst->r64 = cnum64_from_srange(smin >> (smin < 0 ? umin_val : umax_val),
@@ -17473,6 +17504,7 @@ void scalar_alu(struct scalar *dst, const struct scalar *src, const struct bpf_i
s16 off = insn->off;
bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
struct scalar tmp;
+ u64 cleared;
if (!is_safe_to_compute_dst_reg_range(insn, src)) {
scalar_set_unknown(dst);
@@ -17578,8 +17610,12 @@ void scalar_alu(struct scalar *dst, const struct scalar *src, const struct bpf_i
scalar_step_reset(dst);
break;
case BPF_AND:
+ cleared = (dst->var_off.value | dst->var_off.mask) & ~src->var_off.value;
+ if (!cleared)
+ break;
dst->var_off = tnum_and(dst->var_off, src->var_off);
- scalar32_min_max_and(dst, src);
+ if ((u32)cleared)
+ scalar32_min_max_and(dst, src);
scalar_min_max_and(dst, src);
scalar_step_reset(dst);
break;
@@ -18921,16 +18957,22 @@ bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits)
/* tnum_intersect() of two tnums that don't overlap is not empty */
if (!tnum_overlap(a, b))
return false;
+ s.var_off = tnum_intersect(a, b);
/*
* Equal in 64 bits is equal in the low 32 bits too. 'if w1 s> 200' on a zero extended int
* refines nothing but the 32-bit range.
*/
- s.r32 = cnum32_intersect(dst->r32, src->r32);
+ if (bits >= 32)
+ s.r32 = cnum32_intersect(dst->r32, src->r32);
if (bits == 64) {
s.r64 = cnum64_intersect(dst->r64, src->r64);
- s.var_off = tnum_intersect(a, b);
- } else {
- s.var_off = tnum_with_subreg(dst->var_off, tnum_intersect(a, b));
+ if (src->step % dst->step == 0) {
+ if (src->base % dst->step != dst->base)
+ return false;
+ s.step = src->step;
+ s.base = src->base;
+ }
+ s.r64 = cnum64_intersect_linear(s.r64, s.base, s.step);
}
scalar_bounds_sync(&s);
if (scalar_is_empty(&s) || const_tnum_range_mismatch(&s) ||
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number
2026-11-20 7:41 ` [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number Alexei Starovoitov
@ 2026-11-20 8:18 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:18 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [High] bpf: scalar_intersect() zeroes upper 32 bits of var_off when
bits is 32
--
commit cf38506673fee86eba3e381affed0c24f4518bbf
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Keep more of what is known about a number
This commit refines six BPF verifier scalar transfer functions to preserve
more range, known-bit, parcial spill, and linear step information.
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index de3e9675e509f..3229372c5f496 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -3994,6 +3999,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
> /* coerce_scalar_to_size will adjust the boundaries. */
> state->regs[dst_regno] = *reg;
> } else {
> + /* the low bytes are from the spill */
> + bool fill_exceeds_spill =
> + size > spill_size && !(off % BPF_REG_SIZE) &&
> + !IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
> int spill_cnt = 0, zero_cnt = 0;
>
> for (i = 0; i < size; i++) {
> @@ -4033,6 +4042,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
> } else {
> mark_reg_stack_read(env, reg_state, off, off + size,
> dst_regno);
> + if (fill_exceeds_spill)
> + scalar_intersect(scalar_of(&state->regs[dst_regno]),
> + scalar_of(reg),
> + spill_size * BITS_PER_BYTE);
> }
> }
> } else if (dst_regno >= 0) {
[ ... ]
> @@ -18921,16 +18957,22 @@ bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits)
> /* tnum_intersect() of two tnums that don't overlap is not empty */
> if (!tnum_overlap(a, b))
> return false;
> + s.var_off = tnum_intersect(a, b);
> /*
> * Equal in 64 bits is equal in the low 32 bits too. 'if w1 s> 200' on a zero extended int
> * refines nothing but the 32-bit range.
> */
> - s.r32 = cnum32_intersect(dst->r32, src->r32);
> + if (bits >= 32)
> + s.r32 = cnum32_intersect(dst->r32, src->r32);
> if (bits == 64) {
> s.r64 = cnum64_intersect(dst->r64, src->r64);
> - s.var_off = tnum_intersect(a, b);
> - } else {
> - s.var_off = tnum_with_subreg(dst->var_off, tnum_intersect(a, b));
> + if (src->step % dst->step == 0) {
> + if (src->base % dst->step != dst->base)
> + return false;
> + s.step = src->step;
> + s.base = src->base;
> + }
> + s.r64 = cnum64_intersect_linear(s.r64, s.base, s.step);
> }
[Severity: High]
Does dropping tnum_with_subreg() in scalar_intersect() falsely mark the
upper 32 bits of s.var_off as known zero when bits == 32?
When bits < 64, a and b are truncated with tnum_subreg(), which clears the
upper 32 bits of both value and mask to 0. Assigning s.var_off =
tnum_intersect(a, b) directly then causes scalar_bounds_sync(&s) to clamp
s.r64 to a zero-extended 32-bit range. For example, a 64-bit stack fill
overlapping a 32-bit scalar spill and 32 bits of STACK_MISC calls
scalar_intersect() with bits == 32 on an unknown 64-bit dst register,
leaving the verifier believing the upper 32 bits of dst are zero while at
runtime they hold arbitrary STACK_MISC data. (A later commit in this
series, commit "bpf: Mark a state that no run of the program has",
reworks the bits < 64 handling in scalar_intersect() to preserve the upper
bits, so moving that change here would avoid this regression in
intermediate commits.)
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=12
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number
2026-11-20 7:41 ` [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number Alexei Starovoitov
2026-11-20 8:18 ` sashiko-bot
@ 2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 5131 bytes --]
> bpf: Keep more of what is known about a number
TL;DR: scalar_intersect() may mark the upper 32 bits of a 64-bit fill over a
smaller spill as known zero, so the verifier could accept out-of-bounds map
accesses; fixed only later in the series, breaking bisection.
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -4033,6 +4042,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
> } else {
> mark_reg_stack_read(env, reg_state, off, off + size,
> dst_regno);
> + if (fill_exceeds_spill)
> + scalar_intersect(scalar_of(&state->regs[dst_regno]),
> + scalar_of(reg),
> + spill_size * BITS_PER_BYTE);
> }
Can this make the verifier believe the upper 32 bits of the loaded value are
known zero? The commit message says "w1 is 5", but the code seems to make
the whole 64-bit register known.
With a 4-byte spill, bits is 32 and scalar_intersect() now does:
if (bits < 64) {
a = tnum_subreg(a);
b = tnum_subreg(b);
}
...
s.var_off = tnum_intersect(a, b);
Both tnums are cut to their low 32 bits, so the result has the upper 32 bits
known 0. The previous code kept the destination's upper bits with
tnum_with_subreg(dst->var_off, tnum_intersect(a, b)), and that call is
removed here. scalar_bounds_sync() then narrows r64 to the low-32 range.
Take a CAP_BPF program:
r6 = 0
*(u32 *)(r10 - 8) = r6 ; 4-byte scalar spill
*(u32 *)(r10 - 4) = 1 ; upper half becomes STACK_MISC
r1 = *(u64 *)(r10 - 8) ; fill_exceeds_spill
mark_reg_stack_read() leaves r1 unknown because of the STACK_MISC byte, and
scalar_intersect() then turns r1 into the constant 0, while at run time r1 is
0x100000000. The same applies when an unknown 8-byte value is stored before
the u32 spill. If r1 is then added to a map value pointer, an access the
verifier believes is at offset 0 lands outside the object.
The same state may be reachable without CAP_BPF: a NULL-checked pointer spill
becomes SCALAR 0 in mark_ptr_or_null_reg(), and the new keep path in
check_stack_write_fixed_off() narrows it to a 4-byte spill after a 4-byte
write at r10 - 4.
This is fixed later in the series by 8b85922f37e3 ("bpf: Mark a state that no
run of the program has"), which replaces the tnum_subreg() of both operands
with:
b = tnum_or(tnum_cast(b, bits / BITS_PER_BYTE),
tnum_lshift(tnum_unknown, bits))
so this commit is unsound on its own, which breaks bisection.
[ ... ]
> @@ -18921,16 +18957,22 @@ bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits)
> /* tnum_intersect() of two tnums that don't overlap is not empty */
> if (!tnum_overlap(a, b))
> return false;
> + s.var_off = tnum_intersect(a, b);
> /*
> * Equal in 64 bits is equal in the low 32 bits too. 'if w1 s> 200' on a zero extended int
> * refines nothing but the 32-bit range.
> */
> - s.r32 = cnum32_intersect(dst->r32, src->r32);
> + if (bits >= 32)
> + s.r32 = cnum32_intersect(dst->r32, src->r32);
> if (bits == 64) {
> s.r64 = cnum64_intersect(dst->r64, src->r64);
> - s.var_off = tnum_intersect(a, b);
> - } else {
> - s.var_off = tnum_with_subreg(dst->var_off, tnum_intersect(a, b));
> + if (src->step % dst->step == 0) {
Does removing the else branch here make scalar_intersect() unsound for
bits == 32? s.var_off = tnum_intersect(a, b) is now used for every width,
and for bits < 64 both a and b have been cut with tnum_subreg(), so the
upper 32 bits of dst are reported as known zero even though they are
unknown.
scalar_bounds_sync() -> __update_scalar64_bounds() then clamps r64 to at most
U32_MAX, or to an exact constant when the spilled value was constant. The
only caller added by this patch is check_stack_read_fixed_off():
*(u64 *)(r10 - 8) = r6 // r6 known == 5, 8-byte scalar spill
*(u32 *)(r10 - 4) = 1 // bytes 7..4 kept as a 4-byte spill, 3..0 STACK_MISC
r1 = *(u64 *)(r10 - 8) // fill_exceeds_spill, r1 unknown after
// mark_reg_stack_read()
// scalar_intersect(r1, spilled r6, 32)
// -> var_off = const 5, r64 = [5, 5]
The verifier then treats r1 as the 64-bit constant 5, but at run time r1 is
0x100000005. Using r1 as an offset into a map value passes the bounds check
and accesses about 4 GiB past the object. The same happens with a BPF_ST
spill of size 4 and with uninitialized upper bytes.
Could the upper bits of dst->var_off be kept when bits < 64, for example by
restoring tnum_with_subreg(dst->var_off, ...), or by leaving a as the full
dst->var_off and widening b with unknown upper bits?
Commit 8b85922f37e3 later changes this in the way described above, without
mentioning the problem.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 13/49] bpf: Remember how many bytes of a packet are checked
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (11 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 12/49] bpf: Keep more of what is known about a number Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 14/49] bpf: Mark a state that no run of the program has Alexei Starovoitov
` (35 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The range of a packet pointer is lost when the number that is added can
be negative:
if (data + 54 > data_end) return;
p = data + 20; range 54
p += -4..4; range 0
*p fails, but the offset is 16..24
The check of 54 bytes holds for the packet, wherever the pointer goes.
Keep that number in pkt_min_len. If the largest offset of the pointer is
below pkt_min_len, the range is at least pkt_min_len.
A join takes the smaller number. A state with the larger number does
not subsume a state with the smaller number. The join rule does not
look at the number: each trip of a scan loop checks one byte more, so
all exits of the loop would stay apart.
Equations will let a jump refine a pointer that has any offset, and
they bring a far pointer back:
r9 = data + r2 r2 is 0..255, 8 bytes are checked
r8 = r9 + 0x7fffffffffffffc0 the largest offset wraps
if r2 > 100 goto out refines r2, r9 and r8
r8 -= 0x7fffffffffffffc0 r8 is r9 again
An earlier patch has the rule for that: an id and a range hold only
within BPF_MAX_VAR_OFF of the start. So r8 has no range, and pointers
with one id are all near, or they are copies of one far pointer.
smax can go from S64_MAX to -3, and the difference does not fit in s64.
bpf_trim_pkt_range() computes it in u64.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 11 ++++++++++-
kernel/bpf/log.c | 2 ++
kernel/bpf/verifier.c | 31 ++++++++++++++++++++-----------
3 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index e1432f48fea..ed15e63456f 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -117,7 +117,10 @@ struct bpf_reg_state {
u32 id;
union {
/* valid when type == PTR_TO_PACKET */
- int range;
+ struct {
+ int range;
+ int pkt_min_len; /* see pkt_min_len() */
+ };
/*
* Valid when type == PTR_TO_STACK. Inside the callee two registers
* can be both PTR_TO_STACK like R1=fp-8 and R2=fp-8, but one of them
@@ -2212,6 +2215,12 @@ int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
+/* How many bytes are checked from the start of the packet, wherever the pointer is */
+static inline int pkt_min_len(const struct bpf_reg_state *reg)
+{
+ return reg->r64.size ? reg->pkt_min_len : max(reg->range, reg->pkt_min_len);
+}
+
static inline u32 bpf_xreg_cnt(const struct bpf_verifier_state *st)
{
u32 fr, nr = 0;
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index e58202955f1..0d381e31421 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -666,6 +666,8 @@ static void print_reg_state(struct bpf_verifier_env *env,
if (type_is_pkt_pointer(t)) {
verbose_a("r=");
verbose_unum(env, reg->range);
+ if (pkt_min_len(reg) && pkt_min_len(reg) != reg->range)
+ verbose_a("min_len=%d", pkt_min_len(reg));
}
if (base_type(t) == PTR_TO_MEM) {
verbose_a("sz=");
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3229372c5f4..e8377d15b02 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -16513,7 +16513,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
u8 opcode = BPF_OP(insn->code);
u32 dst = insn->dst_reg;
const char *reason;
- int ret, bounds_ret;
+ int ret, bounds_ret, min_len;
bool to_scalar;
s64 smax;
@@ -16644,6 +16644,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
* constraints violations.
*/
__mark_scalar32_unbounded(scalar_of(dst_reg));
+ min_len = pkt_min_len(ptr_reg);
switch (opcode) {
case BPF_ADD:
@@ -16673,7 +16674,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
*/
if (dst_reg->range < 0 ||
(!known && (smin_val < 0 || reg_umax(dst_reg) >= dst_reg->range)))
- memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
+ dst_reg->range = 0;
}
break;
case BPF_SUB:
@@ -16719,7 +16720,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
* prior comparison as arithmetic invalidates them.
*/
if ((!known && smin_val < 0) || dst_reg->range < 0)
- memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
+ dst_reg->range = 0;
}
scalar_step_reset(scalar_of(dst_reg));
break;
@@ -16759,14 +16760,15 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
* step tell about it (alignment, umin). No other pointer has its id, so
* it is enough that its range loses what its umax loses.
*/
+ smax = reg_smax(dst_reg);
if (reg_is_pkt_pointer(dst_reg) &&
- (known || (opcode == BPF_SUB && dst_reg->range > 0))) {
+ (known || (opcode == BPF_SUB && dst_reg->range > 0)))
__update_scalar32_bounds(scalar_of(dst_reg));
- } else {
- smax = reg_smax(dst_reg);
+ else
reg_bounds_sync(dst_reg);
- bpf_trim_pkt_range(dst_reg, smax);
- }
+ if (reg_is_pkt_pointer(dst_reg))
+ dst_reg->pkt_min_len = min_len;
+ bpf_trim_pkt_range(dst_reg, smax);
bpf_blur_far_offset(env, dst_reg);
bounds_ret = sanitize_check_bounds(env, insn, dst_reg);
if (bounds_ret == -EACCES)
@@ -17866,12 +17868,19 @@ bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a,
* 'range - smax' bytes from a packet pointer are in the packet. reg_bounds_sync() can make smax
* smaller than @smax: with a step of 8, or with 3 zero bits in var_off, 58 becomes 56. Make the
* range smaller by the same number, so that the number of bytes does not grow. umax is not
- * sufficient: it is U64_MAX when the offset can be negative.
+ * sufficient: it is U64_MAX when the offset can be negative. A jump can take smax from S64_MAX
+ * to -3: the difference does not fit in s64. If smax is larger than @smax the range is lost.
+ * If smax is below pkt_min_len, the range is at least pkt_min_len.
*/
void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax)
{
- if (type_is_pkt_pointer(reg->type) && reg->range > 0)
- reg->range = max_t(s64, reg->range - (smax - reg_smax(reg)), 0);
+ u64 lost = (u64)smax - reg_smax(reg);
+
+ if (!type_is_pkt_pointer(reg->type) || reg->range < 0)
+ return;
+ reg->range = lost < reg->range ? reg->range - lost : 0;
+ if (reg->pkt_min_len > reg_smax(reg))
+ reg->range = max(reg->range, reg->pkt_min_len);
}
/*
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 13/49] bpf: Remember how many bytes of a packet are checked
2026-11-20 7:41 ` [PATCH bpf-next 13/49] bpf: Remember how many bytes of a packet are checked Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2026 bytes --]
> bpf: Remember how many bytes of a packet are checked
TL;DR: Possible verifier state-pruning hole: if regsafe() doesn't compare the
new pkt_min_len, an old state may subsume a current one and accept an
out-of-bounds packet access; the described rules weren't found.
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -117,7 +117,10 @@ struct bpf_reg_state {
> u32 id;
> union {
> /* valid when type == PTR_TO_PACKET */
> - int range;
> + struct {
> + int range;
> + int pkt_min_len; /* see pkt_min_len() */
> + };
A subsystem pattern flags this as potentially concerning: a new field that
affects safety for PTR_TO_PACKET and PTR_TO_PACKET_META needs an explicit
comparison in regsafe(), since the packet case compares fields one by one.
The commit message says "A join takes the smaller number. A state with the
larger number does not subsume a state with the smaller number." I could not
find either rule in this commit or later in the series. regsafe() is called
from stacksafe() and func_states_equal() in kernel/bpf/states.c, but no
regsafe() is defined in the tree after 32e8f52de3c8 ("bpf: Delete the old main
pass of the verifier").
If the regsafe() that is brought back lets an old state with pkt_min_len = 54
subsume a current state with pkt_min_len = 0, with the other fields matching,
then a later:
p += -4..4;
*p
would be accepted on the current path with range 54, although only the
smaller length was checked on that path. That would be an out-of-bounds
packet access.
Could the changelog point to the patch that adds these rules? And should the
PTR_TO_PACKET case of the new regsafe() require
old->pkt_min_len <= cur->pkt_min_len?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 14/49] bpf: Mark a state that no run of the program has
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (12 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 13/49] bpf: Remember how many bytes of a packet are checked Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:24 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 15/49] bpf: Peel the first trips of a loop and join the other trips Alexei Starovoitov
` (34 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A jump can leave an xreg without a value: the ranges and the known bits
have no number in common. No run of the program has such a state.
jump -> range is empty -> infeasible -> a pass: continues
a run without joins: drops it
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 2 ++
kernel/bpf/verifier.c | 19 ++++++++++---------
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index ed15e63456f..cc90e237d53 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -522,6 +522,8 @@ struct bpf_verifier_state {
struct bpf_reference_state *refs;
u32 insn_idx;
u32 curframe;
+ /* an xreg has no value: unreachable. Only a path-sensitive run drops such a state. */
+ bool infeasible;
struct_group(held,
void *active_lock_ptr;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e8377d15b02..1523ae1be51 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1808,6 +1808,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_st
dst_state->unwound = src->unwound;
dst_state->changed_pkt_data = src->changed_pkt_data;
dst_state->flow_entry = src->flow_entry;
+ dst_state->infeasible = src->infeasible;
dst_state->insn_idx = src->insn_idx;
dst_state->curframe = src->curframe;
dst_state->last_insn_idx = src->last_insn_idx;
@@ -2437,6 +2438,7 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
reg->base, reg->step);
if (env->test_reg_invariants)
return -EFAULT;
+ env->cur_state->infeasible = true;
__mark_scalar_unbounded(s);
scalar_step_reset(s);
return 0;
@@ -18947,8 +18949,8 @@ static bool scalars_fit_in_bits(const struct scalar *a, const struct scalar *b,
/*
* The values of @dst and @src are equal in their low @bits bits: @dst becomes the intersection.
- * False: @dst is left as it is. Fewer than 32 bits say nothing about a range. Or the intersection
- * is empty: no execution gets here with these values, the path is infeasible.
+ * Fewer than 32 bits say nothing about a range. Returns false if the intersection is empty.
+ * Then no execution gets here with these values, and @dst does not change.
*/
bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits)
{
@@ -18957,12 +18959,8 @@ bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits)
if (scalars_fit_in_bits(dst, src, bits))
bits = 64;
- if (bits < 32)
- return false;
- if (bits < 64) {
- a = tnum_subreg(a);
- b = tnum_subreg(b);
- }
+ if (bits < 64)
+ b = tnum_or(tnum_cast(b, bits / BITS_PER_BYTE), tnum_lshift(tnum_unknown, bits));
/* tnum_intersect() of two tnums that don't overlap is not empty */
if (!tnum_overlap(a, b))
return false;
@@ -19000,6 +18998,7 @@ static int refine_live_branch(struct bpf_verifier_env *env, struct bpf_insn *ins
struct bpf_reg_state *reg1, struct bpf_reg_state *reg2)
{
struct bpf_reg_state *regs = cur_regs(env);
+ bool empty;
int err;
/* is_branch_taken() knew the branch before it refined anything */
@@ -19010,7 +19009,9 @@ static int refine_live_branch(struct bpf_verifier_env *env, struct bpf_insn *ins
* The fall-through branch of JSET forgets the ranges. Or this branch is dead too: the path
* is infeasible. The registers are left as they are.
*/
- if (BPF_OP(insn->code) == BPF_JSET || reg_is_empty(reg1) || reg_is_empty(reg2))
+ empty = reg_is_empty(reg1) || reg_is_empty(reg2);
+ env->cur_state->infeasible |= empty;
+ if (BPF_OP(insn->code) == BPF_JSET || empty)
return 0;
err = reg_bounds_sanity_check(env, reg1, "live_reg1");
err = err ?: reg_bounds_sanity_check(env, reg2, "live_reg2");
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 14/49] bpf: Mark a state that no run of the program has
2026-11-20 7:41 ` [PATCH bpf-next 14/49] bpf: Mark a state that no run of the program has Alexei Starovoitov
@ 2026-11-20 8:24 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:24 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 3982 bytes --]
> bpf: Mark a state that no run of the program has
TL;DR:
- scalar_intersect() change may leave 32-bit spill fills unsoundly bounded, so
the verifier could accept unsafe programs; undocumented in the commit
message
- the new infeasible flag is never read, and may be set on the wrong state
> bpf: Mark a state that no run of the program has
>
> A jump can leave an xreg without a value: the ranges and the known bits
> have no number in common. No run of the program has such a state.
>
> jump -> range is empty -> infeasible -> a pass: continues
> a run without joins: drops it
>
> Signed-off-by: Alexei Starovoitov <ast@kernel.org>
The commit message only describes marking infeasible states, but the patch
also changes scalar_intersect().
First, it removes the forced zeroing of the upper 32 bits of dst->var_off.
In the parent, with bits == 32, the code did:
a = tnum_subreg(a);
b = tnum_subreg(b);
...
s.var_off = tnum_intersect(a, b);
check_stack_read_fixed_off() passes bits = 32 for a 4-byte spill read by an
8-byte fill (fill_exceeds_spill), where the upper 4 bytes come from
STACK_MISC. For example:
*(u32 *)(r10 - 4) = 0xffffffff;
*(u32 *)(r10 - 8) = w1;
r2 = *(u64 *)(r10 - 8)
scalar_bounds_sync() then bounds r2 to [0, U32_MAX], while at run time r2
is 0xffffffff_xxxxxxxx.
Second, the removal of "if (bits < 32) return false;" turns on refinement
for 1 and 2 byte spills.
Could the soundness fix be folded into 20d7b7b266b5 ("bpf: Keep more of
what is known about a number") so each prefix of the series stays sound for
bisection, or at least be described in this changelog?
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
[ ... ]
> @@ -522,6 +522,8 @@ struct bpf_verifier_state {
> struct bpf_reference_state *refs;
> u32 insn_idx;
> u32 curframe;
> + /* an xreg has no value: unreachable. Only a path-sensitive run drops such a state. */
> + bool infeasible;
Is anything reading this field? It is written in reg_bounds_sanity_check()
and refine_live_branch() and copied in bpf_copy_verifier_state(), but
states_equal() does not compare it and log.c does not print it. The only
later reader in the series, infeasible_state_ends(), tests
env->cur_state->equations.infeasible, a different flag.
The comment says "Only a path-sensitive run drops such a state", and the
commit message says "a run without joins: drops it". No code does this,
and env->path_sensitive is never set.
Where the flag is set, it does not always match "an xreg has no value":
In check_cond_jmp_op() with both branches live, push_stack() copies the
current state into other_branch before regs_bounds_sanity_check_branches()
runs. If true_reg1 or true_reg2 fails the check, reg_bounds_sanity_check()
marks env->cur_state:
env->cur_state->infeasible = true;
That is this_branch, which takes false_reg1/false_reg2. The jump state,
which takes the bad registers, is left unmarked.
reg_bounds_sanity_check() also reaches the out: label through
verifier_bug_if(reg->step == 0 || reg->base >= reg->step, ...), which its
own comment calls a verifier bug rather than an empty value set, yet it
still marks the state infeasible.
If a later reader drops states using this flag, both cases would skip a
path that runs. The later equations-based reader also checks
env->bypass_spec_v1 first, since a speculative run can still reach such a
path, and this field's comment gives no such condition.
Could the field be added in the patch that first reads it, with the
producers changed to mark the state that actually holds the empty register?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 15/49] bpf: Peel the first trips of a loop and join the other trips
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (13 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 14/49] bpf: Mark a state that no run of the program has Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 16/49] bpf: Add equations and their arithmetic Alexei Starovoitov
` (33 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A loop can be verified trip by trip, or with one state for all trips.
The first way is precise and too long for a long loop. The second way
is short and loses what differs between trips:
char buf[64]; off = 255; ok = 0; trip 0 is different
for (...) { if (ok) buf[off] = 0; off = x & 63; ok = 1; }
Use both. A loop has peeled trips and a leftover:
entry -> [0] -> [1] -> ... -> [d-1] -> +----------+ <--+
peeled trips, kept apart | LEFTOVER | |
| widened |----+
+----------+
Pass 0 peels nothing. A later pass peels if an error asks for it, see
try_peel_trip(). bpf_flow_loop_enter() keeps the loop stack of a frame.
The calls of a callback and recursive calls go around like the body of
a loop. One struct bpf_loop describes the three:
kind header
BPF_LOOP_JUMPS the target of the back edge
BPF_LOOP_CALLBACK the call of the helper
BPF_LOOP_RECURSION the start of the first function of the cycle
What is held has no join. A loop that acquires in each trip makes new
states without end. check_held_in_leftover() stops it: a reference that
is in no xreg is a leak, and the peeled trips limit the locks.
The log has the innermost loop. Level 2 has all loops for each block.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 79 ++++++++
kernel/bpf/fixups.c | 15 +-
kernel/bpf/flow.c | 17 ++
kernel/bpf/log.c | 18 +-
kernel/bpf/loops.c | 1 +
kernel/bpf/verifier.c | 346 ++++++++++++++++++++++++++++++++---
6 files changed, 444 insertions(+), 32 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index cc90e237d53..77f1fe969a2 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -93,6 +93,8 @@ struct scalar {
*/
u16 base;
u16 step;
+ /* Not a part of the value: it grew once at a loop header, see join_range() */
+ bool grew_once;
};
/* The struct scalar that is an anonymous member of *@p */
@@ -420,6 +422,24 @@ static inline u32 bpf_xreg_to_spi(u32 xregno)
return xregno - MAX_BPF_EXT_REG;
}
+/*
+ * A state in a loop, see bpf_flow_loop_enter()
+ *
+ * entry -> [0] -> [1] -> ... -> [d-1] -> +----------+ <--+
+ * peeled trips, kept apart | LEFTOVER | |
+ * | widened |----+
+ * +----------+
+ */
+struct loop_stack_entry {
+ struct bpf_loop *loop;
+ /* How many times the state came back to the header. A join has the larger number. */
+ u32 trip;
+ /* The state that entered the loop. What comes back to the header is for that state. */
+ u32 entry_id;
+ u32 held_cnt; /* when the state entered the loop */
+ bool peeled; /* in a peeled trip, not in the leftover */
+};
+
#define LOOP_STACK_SIZE 16
/* state of the program:
@@ -524,6 +544,11 @@ struct bpf_verifier_state {
u32 curframe;
/* an xreg has no value: unreachable. Only a path-sensitive run drops such a state. */
bool infeasible;
+ /* an error only in a leftover, see check_held_in_leftover() */
+ bool holds_too_much;
+ u32 max_held_cnt; /* when it first held too much */
+ /* the loops that made an xreg of data_deps imprecise, see bpf_loop.suspect_bit */
+ u64 suspect_loops;
struct_group(held,
void *active_lock_ptr;
@@ -754,7 +779,28 @@ struct bpf_backedge {
int latch;
};
+/* Calls of a callback and recursive calls go around like the body of a loop does */
+enum bpf_loop_kind {
+ BPF_LOOP_JUMPS,
+ BPF_LOOP_CALLBACK, /* the header is the call of the helper */
+ /* the header is the first insn of the first function of the cycle */
+ BPF_LOOP_RECURSION,
+};
+
+/* What a pass finds about a loop */
+struct bpf_loop_per_pass {
+ bool has_leftover; /* the peeled trips were not enough */
+ /* a check failed in a state that the loop kept apart, with no other suspect */
+ bool has_unproven;
+ u32 peeled_cnt; /* trips that the pass peeled */
+ u64 cost; /* for the states that the loop keeps apart */
+};
+
struct bpf_loop {
+ enum bpf_loop_kind kind;
+ u32 header;
+ /* in bpf_verifier_state.suspect_loops. Loops with the same bit are suspects together. */
+ u64 suspect_bit;
struct bpf_backedge backedges[MAX_BACKEDGES];
int exit_to; /* where the first edge that leaves the loop goes, or -1 */
int backedges_cnt;
@@ -766,8 +812,29 @@ struct bpf_loop {
u32 nr_insns;
/* not rt_bounded: there are places for may_goto, see guard_to in bpf_insn_aux_data */
bool has_guards;
+
+ /* What one pass gives to the next pass */
+ struct_group_tagged(bpf_loop_precision, precision,
+ u32 peel_cnt; /* trips before the leftover */
+ /* while bpf_loop_per_pass.cost is not above @budget: */
+ u64 budget;
+ u32 max_peel_cnt; /* peel up to this trip */
+ );
+ bool partitioned; /* the last partition was for this loop */
+ struct bpf_loop_per_pass this_pass;
};
+static inline bool bpf_loop_has_budget(const struct bpf_loop *loop)
+{
+ return loop->this_pass.cost <= loop->budget;
+}
+
+/* A partition for a loop ends when a state that it kept apart is unproven */
+static inline bool bpf_loop_partition_is_active(const struct bpf_loop *loop)
+{
+ return bpf_loop_has_budget(loop) && !(loop->partitioned && loop->this_pass.has_unproven);
+}
+
struct bpf_insn_aux_data {
/* What a pass writes for the rewrite of the insn. Each pass starts from the snapshot. */
struct_group_tagged(bpf_insn_fixups, fixups,
@@ -899,6 +966,8 @@ struct bpf_insn_aux_data {
s32 guard_to;
/* additional information about the loop if this instruction is a loop header */
struct bpf_loop *loop;
+ /* the call of a callback, or the first insn of a recursive function */
+ struct bpf_loop *callback_loop, *recursion_loop;
};
/*
@@ -995,6 +1064,7 @@ struct bpf_subprog_info {
u16 stack_depth; /* max. stack depth used by this function */
u16 stack_extra;
u16 arena_stack_depth; /* size of the frame in the stack in arena */
+ u32 cycle; /* the first subprog of its cycle of the call graph, 0 if it is on none */
u32 insns_total;
u32 insns_self;
/* offsets in range [stack_depth .. fastcall_stack_off)
@@ -1539,6 +1609,9 @@ int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, i
void bpf_flow_next_is_no_bound(struct bpf_verifier_env *env, int insn_idx, int h);
bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env);
int bpf_flow_place_guards(struct bpf_verifier_env *env);
+/* verifier.c, for flow.c */
+int bpf_flow_loop_enter(struct bpf_verifier_env *env, struct bpf_loop **backedge_loop,
+ bool *starts_leftover);
#define BPF_EXIT_NEEDS_CALLER 1 /* verify the call in the frames of the caller */
int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
const struct bpf_verifier_state *exit, int call_idx);
@@ -2223,6 +2296,12 @@ static inline int pkt_min_len(const struct bpf_reg_state *reg)
return reg->r64.size ? reg->pkt_min_len : max(reg->range, reg->pkt_min_len);
}
+/* The loop that the frame is in, or NULL */
+static inline struct loop_stack_entry *bpf_top_loop(const struct bpf_func_state *frame)
+{
+ return frame->loop_stack_cnt ? &frame->loop_stack[frame->loop_stack_cnt - 1] : NULL;
+}
+
static inline u32 bpf_xreg_cnt(const struct bpf_verifier_state *st)
{
u32 fr, nr = 0;
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 2bbacd26180..751b25c92c9 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -754,6 +754,13 @@ int bpf_insert_recursion_checks(struct bpf_verifier_env *env)
return 0;
}
+static void free_loop(struct bpf_loop *loop)
+{
+ if (!loop)
+ return;
+ kvfree(loop);
+}
+
/* Clean up dynamically allocated fields of aux data for instructions [start, ...] */
void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
{
@@ -762,10 +769,10 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
int i;
for (i = start; i < end; i++) {
- if (aux_data[i].loop) {
- kvfree(aux_data[i].loop);
- aux_data[i].loop = NULL;
- }
+ free_loop(aux_data[i].loop);
+ free_loop(aux_data[i].callback_loop);
+ free_loop(aux_data[i].recursion_loop);
+ aux_data[i].loop = aux_data[i].callback_loop = aux_data[i].recursion_loop = NULL;
}
}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index cbbc663acac..2c1835d1b3e 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -311,3 +311,20 @@ int bpf_check_limits(struct bpf_verifier_env *env)
env->over_limit = true;
return -E2BIG;
}
+
+static struct bpf_loop *flow_new_call_loop(struct bpf_verifier_env *env, enum bpf_loop_kind kind,
+ int header, int func)
+{
+ struct bpf_loop *loop = kvzalloc_obj(*loop, BPF_VERIFIER_GFP);
+
+ env->mem += sizeof(*loop);
+ if (!loop)
+ return NULL;
+ loop->kind = kind;
+ loop->header = header;
+ /* the helper stops, the function checks the stack, see check_recursion_guard() */
+ loop->rt_bounded = true;
+ loop->nr_insns = func < 0 ? 1 :
+ env->subprog_info[func + 1].start - env->subprog_info[func].start;
+ return loop;
+}
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 0d381e31421..7b1390d9054 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -702,7 +702,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
{
const struct bpf_func_state *state = vstate->frame[frameno];
const struct bpf_reg_state *reg;
- int i;
+ int i, first;
if (state->frameno)
verbose(env, " frame%d:", state->frameno);
@@ -789,15 +789,19 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
verbose(env, " cb");
if (state->in_async_callback_fn)
verbose(env, " async_cb");
- /* the iterations of the loops in which they are kept apart */
- for (i = 0; i < state->loop_stack_cnt; i++)
- if (state->loop_stack[i].apart)
- verbose(env, " iter(%d)=%u", state->loop_stack[i].loop_id,
+ /* The innermost loop. Level 2 has all loops, once for each block. */
+ first = print_all && (env->log.level & BPF_LOG_LEVEL2) ? 0 :
+ max(state->loop_stack_cnt, 1U) - 1;
+ /* the peeled trips */
+ for (i = first; i < state->loop_stack_cnt; i++)
+ if (state->loop_stack[i].peeled)
+ verbose(env, " iter(%d)=%u", state->loop_stack[i].loop->header,
state->loop_stack[i].trip);
if (state->loop_stack_cnt) {
verbose(env, " loop_stack=");
- for (i = 0; i < state->loop_stack_cnt; i++)
- verbose(env, "%s%d", i ? "," : "", state->loop_stack[i].loop_id);
+ for (i = first; i < state->loop_stack_cnt; i++)
+ verbose(env, "%s%d", i > first ? "," : "",
+ state->loop_stack[i].loop->header);
}
verbose(env, "\n");
if (!print_all)
diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
index e1f39629f60..f8b2f8befa6 100644
--- a/kernel/bpf/loops.c
+++ b/kernel/bpf/loops.c
@@ -251,6 +251,7 @@ static int mark_as_header(struct bpf_verifier_env *env, int h)
if (!aux[h].loop)
return -ENOMEM;
env->mem += sizeof(struct bpf_loop);
+ aux[h].loop->header = h;
}
return 0;
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1523ae1be51..e4c9ced59b2 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1731,10 +1731,17 @@ static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *st
return NULL;
}
+static void leave_loops(struct bpf_func_state *state, u32 cnt)
+{
+ while (state->loop_stack_cnt > cnt)
+ bpf_flow_put_header_state(&state->loop_stack[--state->loop_stack_cnt]);
+}
+
static void free_func_state(struct bpf_func_state *state)
{
if (!state)
return;
+ leave_loops(state, 0);
kfree(state->loop_stack);
kfree(state->stack_arg_regs);
kfree(state->stack);
@@ -1777,13 +1784,16 @@ void bpf_free_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
static int copy_func_state(struct bpf_func_state *dst,
const struct bpf_func_state *src)
{
+ leave_loops(dst, 0);
memcpy(dst, src, offsetof(struct bpf_func_state, stack));
/* Instruction accounting is path-local, not part of verifier state. */
dst->insns_subtotal = 0;
+ dst->loop_stack_cnt = 0;
dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt,
sizeof(*src->loop_stack), BPF_VERIFIER_GFP);
if (!dst->loop_stack)
return -ENOMEM;
+ dst->loop_stack_cnt = src->loop_stack_cnt;
return copy_stack_state(dst, src);
}
@@ -1809,6 +1819,8 @@ int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_st
dst_state->changed_pkt_data = src->changed_pkt_data;
dst_state->flow_entry = src->flow_entry;
dst_state->infeasible = src->infeasible;
+ dst_state->max_held_cnt = src->max_held_cnt;
+ dst_state->suspect_loops = src->suspect_loops;
dst_state->insn_idx = src->insn_idx;
dst_state->curframe = src->curframe;
dst_state->last_insn_idx = src->last_insn_idx;
@@ -2574,6 +2586,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
* Initialize it similar to do_check_common().
*/
elem->st.in_sleepable = is_sleepable;
+ elem->st.suspect_loops = env->cur_state->suspect_loops;
frame = kzalloc_obj(*frame, BPF_VERIFIER_GFP);
if (!frame)
return ERR_PTR(-ENOMEM);
@@ -6019,6 +6032,8 @@ static int mark_recursive_subprogs(struct bpf_verifier_env *env)
break;
}
si[s].recursive = true;
+ j = cg.cycle[s];
+ si[s].cycle = cg.order[j] = min_t(u32, cg.order[j], s);
err = 0;
}
out:
@@ -8783,15 +8798,14 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_
}
/*
- * bpf_iter_*_next() that every trip of a loop goes through stops the loop if the iterator is
- * older than the loop: every trip takes an element, and there are only so many. It doesn't stop
- * - the loop that makes the iterator, which starts from the beginning in every trip then:
+ * Every trip of a loop goes through bpf_iter_*_next(). It stops the loop if the iterator is older
+ * than the loop: every trip takes an element, and the number of elements is finite.
+ * It does not stop
+ * - the loop that makes the iterator. The iterator starts again in every trip:
* while (v->x) { bpf_for(i, 0, 10) ...; }
- * Of the loops that the program is in, the ones that it entered before it made the iterator
- * are among the first iter.loops.
- * - any loop if nothing was left already: the program was told so and did not leave.
- * A loop with a known number of trips doesn't count on the iterator. Nor does a walk on patience:
- * there is none in a loop that is taken for bounded, see flow_try_walk().
+ * The loops that the program entered before it made the iterator are among the first
+ * iter.loops loops.
+ * - any loop, if no element was left already. The program knew that and did not leave.
*/
static void iter_stops_loops(struct bpf_verifier_env *env, int insn_idx,
const struct bpf_reg_state *iter)
@@ -8804,12 +8818,12 @@ static void iter_stops_loops(struct bpf_verifier_env *env, int insn_idx,
int i;
/* the loops of this subprog: flow_is_bound() doesn't look into the functions it calls */
- for (i = frame->loop_stack_cnt - 1; i >= 0; i--) {
+ for (i = frame->loop_stack_cnt - 1; i >= 0 && !st->speculative; i--) {
entry = &frame->loop_stack[i];
stops = iter->iter.state != BPF_ITER_STATE_DRAINED && below + i >= iter->iter.loops;
- if (entry->terminates || stops)
+ if (entry->loop->kind != BPF_LOOP_JUMPS || stops)
continue;
- bpf_flow_next_is_no_bound(env, insn_idx, entry->loop_id);
+ bpf_flow_next_is_no_bound(env, insn_idx, entry->loop->header);
return;
}
}
@@ -11311,6 +11325,16 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog)
return err;
}
+/* The helper returns */
+static void leave_callback(struct bpf_func_state *caller)
+{
+ struct loop_stack_entry *entry = bpf_top_loop(caller);
+
+ caller->callback_depth = 0;
+ if (entry && entry->loop->kind == BPF_LOOP_CALLBACK)
+ leave_loops(caller, caller->loop_stack_cnt - 1);
+}
+
static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int insn_idx, int subprog,
set_callee_state_fn set_callee_state_cb)
@@ -11398,7 +11422,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
return err;
callback_state->frame[callback_state->curframe - 1]->callback_depth++;
- caller->callback_depth = 0;
+ leave_callback(caller);
return 0;
}
@@ -11484,6 +11508,10 @@ static bool func_is_on_its_own(struct bpf_verifier_env *env, struct bpf_verifier
return true;
}
+static int enter_recursion(struct bpf_verifier_env *env, struct bpf_func_state *callee);
+static int check_held_in_leftover(struct bpf_verifier_env *env,
+ const struct loop_stack_entry *entry);
+
static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int insn_idx)
{
u16 live = env->insn_aux_data[env->subprog_info[subprog].start].live_regs_before;
@@ -11497,6 +11525,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
return -ENOMEM;
entry.frame[0] = callee;
entry.in_sleepable = state->in_sleepable;
+ entry.suspect_loops = state->suspect_loops;
entry.callers_unknown = state->callers_unknown;
init_func_state(env, callee, BPF_MAIN_FUNC, 0, subprog);
/* the frame that says so is not there when the function is verified */
@@ -11504,7 +11533,8 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
for (i = BPF_REG_1; i <= BPF_REG_5; i++)
if (live & BIT(i))
callee->regs[i] = caller->regs[i];
- err = grow_stack_arg_slots(env, callee, bpf_in_stack_arg_cnt(&env->subprog_info[subprog]));
+ i = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);
+ err = grow_stack_arg_slots(env, callee, i) ?: enter_recursion(env, callee);
if (err) {
bpf_free_verifier_state(env, &entry, false);
return err;
@@ -11517,6 +11547,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
bpf_free_verifier_state(env, &entry, false);
return err;
}
+ entry.max_held_cnt = state->max_held_cnt;
callee->entry_refs = entry.acquired_refs;
callee->entry_rcu_locks = entry.active_rcu_locks;
callee->entry_preempt_locks = entry.active_preempt_locks;
@@ -11549,6 +11580,9 @@ static int exit_func_on_its_own(struct bpf_verifier_env *env)
/* Invalidate callee-local dynptrs and their slices before the frame goes away. */
err = destroy_dynptrs_in_stack_slots(env, callee, 0,
callee->allocated_stack / BPF_REG_SIZE - 1);
+ /* what comes out of a recursion goes around too */
+ if (!err && env->subprog_info[callee->subprogno].recursive)
+ err = check_held_in_leftover(env, callee->loop_stack);
if (err)
return err;
@@ -11596,6 +11630,8 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
return -EINVAL;
}
caller->unwound = exit->unwound;
+ caller->max_held_cnt = max(caller->max_held_cnt, exit->max_held_cnt);
+ caller->suspect_loops |= exit->suspect_loops;
/*
* bpf_check_cfg() takes the subprog that loads the address of a function for the
@@ -12959,7 +12995,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
err = push_callback_call(env, insn, insn_idx, meta.subprogno,
set_loop_callback_state);
} else {
- cur_func(env)->callback_depth = 0;
+ leave_callback(cur_func(env));
if (env->log.level & BPF_LOG_LEVEL2)
verbose(env, "frame%d bpf_loop iteration limit reached\n",
env->cur_state->curframe);
@@ -20607,6 +20643,105 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
return -EFAULT;
}
+static u32 held_cnt(const struct bpf_verifier_state *st)
+{
+ return st->acquired_refs + st->active_rcu_locks + st->active_preempt_locks;
+}
+
+/* The current state enters @loop in @frame. It is in trip 0. */
+static struct loop_stack_entry *push_loop(struct bpf_verifier_env *env,
+ struct bpf_func_state *frame, struct bpf_loop *loop)
+{
+ u32 cnt = frame->loop_stack_cnt;
+ struct loop_stack_entry *stack;
+
+ stack = krealloc(frame->loop_stack, kmalloc_size_roundup((cnt + 1) * sizeof(*stack)),
+ BPF_VERIFIER_GFP);
+ if (!stack)
+ return NULL;
+ frame->loop_stack = stack;
+ stack[cnt] = (struct loop_stack_entry){
+ .loop = loop,
+ .held_cnt = held_cnt(env->cur_state),
+ .entry_id = ++env->loop_entry_cnt,
+ .peeled = true,
+ };
+ frame->loop_stack_cnt++;
+ return &frame->loop_stack[cnt];
+}
+
+static bool try_peel_trip(const struct loop_stack_entry *entry)
+{
+ struct bpf_loop *loop = entry->loop;
+
+ if (entry->trip >= loop->peel_cnt &&
+ (entry->trip >= loop->max_peel_cnt || !bpf_loop_partition_is_active(loop)))
+ return false;
+ loop->this_pass.peeled_cnt = max(loop->this_pass.peeled_cnt, entry->trip + 1);
+ return true;
+}
+
+/* The loop of @entry starts its leftover. Nothing grew in it so far. */
+static void leave_peeled_trips(struct bpf_verifier_env *env, struct loop_stack_entry *entry)
+{
+ struct bpf_func_state *frame;
+ struct bpf_reg_state *reg;
+
+ entry->peeled = false;
+ entry->loop->this_pass.has_leftover = true;
+ bpf_for_each_reg_in_vstate(env->cur_state, frame, reg, reg->grew_once = false);
+}
+
+/*
+ * Push the loop entered at insn_idx onto the stack. Usually this adds a single
+ * entry on top of its enclosing loop. However, an edge may enter an inner loop
+ * directly, bypassing the header(s) of its enclosing loop(s), e.g.:
+ *
+ * 1: for (...): // enclosing loop, header at 1
+ * 2: for (...): // inner loop, header at 2
+ * ...
+ * 3: if ...:
+ * goto 2b; // enters loop 2 without going through header 1
+ *
+ * Such a bypass makes the enclosing loop irreducible, so its header is missing
+ * from the stack and both headers (1) and (2) need to be pushed onto stack.
+ *
+ * Assumes loop_stack_pop() has already truncated the stack to the common ancestor
+ * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack.
+ */
+static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct bpf_func_state *frame = cur_func(env);
+ struct loop_stack_entry *entry = bpf_top_loop(frame);
+ u32 missing_headers[LOOP_STACK_SIZE];
+ u32 num_missing = 0;
+ int h;
+
+ /* bpf_compute_loops() checked how deep loops nest */
+ for (h = bpf_loop_at_index(env, env->insn_idx);
+ h >= 0 && !(entry && entry->loop == aux[h].loop); h = aux[h].loop_header)
+ missing_headers[num_missing++] = h;
+
+ *pushed = num_missing;
+ while (num_missing--) {
+ h = missing_headers[num_missing];
+ /* wherever it is entered: a loop can have more than one entry */
+ if (aux[h].loop->exit_to < 0) {
+ verbose(env, "loop at insn %d never exits\n", h);
+ return -EINVAL;
+ }
+ entry = push_loop(env, frame, aux[h].loop);
+ if (!entry)
+ return -ENOMEM;
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "entering loop %d\n", h);
+ if (h != env->insn_idx && !env->path_sensitive && !try_peel_trip(entry))
+ leave_peeled_trips(env, entry);
+ }
+ return 0;
+}
+
/*
* An exit from a loop can cross several nested loops, e.g.:
*
@@ -20616,13 +20751,15 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
* if ...: // before goto the loop stack is [1, 2, 3 <top>]
* goto 1b; // after goto it should become [1]
*/
-static int loop_stack_pop(struct bpf_verifier_env *env)
+static void loop_stack_pop(struct bpf_verifier_env *env)
{
struct bpf_insn_aux_data *aux = env->insn_aux_data;
struct bpf_func_state *frame = cur_func(env);
- int new_cnt = 0;
+ int cnt = frame->loop_stack_cnt, new_cnt;
int h, i;
+ /* the function stays in its recursion */
+ new_cnt = cnt && frame->loop_stack[0].loop->kind == BPF_LOOP_RECURSION;
/*
* Walk the loop nest of insn_idx outwards (innermost first) and stop at
* the first loop that is present on the stack - that loop is the common
@@ -20630,8 +20767,8 @@ static int loop_stack_pop(struct bpf_verifier_env *env)
* stack, drop all loops.
*/
for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) {
- for (i = frame->loop_stack_cnt - 1; i >= 0; i--) {
- if (frame->loop_stack[i].loop_id == h) {
+ for (i = cnt - 1; i >= 0; i--) {
+ if (frame->loop_stack[i].loop == aux[h].loop) {
new_cnt = i + 1;
goto pop;
}
@@ -20640,12 +20777,179 @@ static int loop_stack_pop(struct bpf_verifier_env *env)
pop:
if (env->log.level & BPF_LOG_LEVEL2)
- for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--)
- verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id);
- frame->loop_stack_cnt = new_cnt;
+ for (i = cnt - 1; i >= new_cnt; i--)
+ if (frame->loop_stack[i].loop->kind == BPF_LOOP_JUMPS)
+ verbose(env, "exiting loop %d\n",
+ frame->loop_stack[i].loop->header);
+ leave_loops(frame, new_cnt);
+}
+
+static bool xreg_has_id(struct bpf_verifier_state *st, u32 id)
+{
+ u32 mask = BIT(STACK_SPILL) | BIT(STACK_DYNPTR) | BIT(STACK_ITER) | BIT(STACK_IRQ_FLAG);
+ struct bpf_stack_state *slot;
+ struct bpf_func_state *frame;
+ struct bpf_reg_state *reg;
+
+ bpf_for_each_reg_in_vstate_mask(st, frame, reg, slot, mask, ({
+ if (reg->id == id || reg->parent_id == id)
+ return true;
+ }));
+ return false;
+}
+
+/*
+ * The current state went around a loop. If it holds more than the states at the header, none of
+ * them subsumes it, and it has no join. It goes around again:
+ * bpf_for(i, 0, n) { if (prev) release(prev); prev = cur; cur = acquire(); }
+ * The number of xregs limits the references: a reference that is in no xreg is a leak. The
+ * number of peeled trips limits the rest. The state continues until it holds more, so it is
+ * unproven where a check fails on a path. Returns 1 if it ends here.
+ */
+static int check_held_in_leftover(struct bpf_verifier_env *env,
+ const struct loop_stack_entry *entry)
+{
+ struct bpf_verifier_state *st = env->cur_state;
+ u32 held = held_cnt(st), i;
+ bool leak = false;
+
+ if (entry->peeled || held <= entry->held_cnt)
+ return 0;
+ /* the callers have xregs for the first ones */
+ for (i = st->frame[0]->entry_refs; i < st->acquired_refs && !leak; i++) {
+ if (st->refs[i].type != REF_TYPE_PTR || st->refs[i].ctx_arg)
+ continue;
+ bpf_add_cost(env, bpf_xreg_cnt(st), BPF_COST_SCAN);
+ leak = !xreg_has_id(st, st->refs[i].id);
+ held--;
+ }
+ if (!leak && held <= entry->held_cnt + entry->loop->this_pass.peeled_cnt)
+ return 0;
+ if (st->speculative)
+ return -EINVAL;
+ verbose(env, "loop at insn %d acquires more than it releases\n", entry->loop->header);
+ env->imprecise |= !env->err;
+ st->suspect_loops |= entry->loop->suspect_bit;
+ st->holds_too_much = true;
+ if (bpf_same_insns_unproven(env))
+ return -EINVAL;
+ note_error(env, -EINVAL);
+ st->holds_too_much = false;
+ st->max_held_cnt = st->max_held_cnt ?: held_cnt(st);
+ return held_cnt(st) > st->max_held_cnt;
+}
+
+/* The current state came back to the header of the loop of @entry. Returns 1 if it ends here. */
+static int next_trip(struct bpf_verifier_env *env, struct loop_stack_entry *entry,
+ bool *starts_leftover)
+{
+ entry->trip++;
+ if (env->path_sensitive)
+ return failing_path_trip(env, entry);
+ if (!entry->peeled)
+ return check_held_in_leftover(env, entry);
+ *starts_leftover = !bpf_flow_trip_is_new(env, entry) || !try_peel_trip(entry);
+ if (*starts_leftover)
+ entry->entry_id = ++env->loop_entry_cnt;
return 0;
}
+/* @callee is the frame of a function that is verified on its own */
+static int enter_recursion(struct bpf_verifier_env *env, struct bpf_func_state *callee)
+{
+ u32 cycle = env->subprog_info[callee->subprogno].cycle;
+ struct bpf_loop *loop = env->insn_aux_data[env->subprog_info[cycle].start].recursion_loop;
+ struct loop_stack_entry *entry, *from = env->cur_state->frame[0]->loop_stack;
+ bool starts_leftover;
+ int err = 0;
+
+ if (!loop)
+ return 0;
+ entry = push_loop(env, callee, loop);
+ if (!entry)
+ return -ENOMEM;
+ starts_leftover = !try_peel_trip(entry);
+ if (env->cur_state->frame[0]->loop_stack_cnt && from->loop == loop) {
+ *entry = *from;
+ starts_leftover = false;
+ err = next_trip(env, entry, &starts_leftover);
+ }
+ if (starts_leftover)
+ leave_peeled_trips(env, entry);
+ return err;
+}
+
+/* The helper at env->insn_idx calls the callback again and again. @back: the callback returned. */
+static int enter_callback(struct bpf_verifier_env *env, bool back, struct bpf_loop **backedge_loop)
+{
+ struct bpf_loop *loop = cur_aux(env)->callback_loop;
+ struct bpf_func_state *frame = cur_func(env);
+ struct loop_stack_entry *entry;
+ bool starts_leftover = false;
+ int err = 0;
+
+ if (!loop)
+ return 0;
+ if (back) {
+ entry = bpf_top_loop(frame);
+ err = next_trip(env, entry, &starts_leftover);
+ *backedge_loop = starts_leftover ? NULL : loop;
+ } else {
+ entry = push_loop(env, frame, loop);
+ if (!entry)
+ return -ENOMEM;
+ if (env->path_sensitive)
+ return failing_path_trip(env, entry);
+ starts_leftover = !try_peel_trip(entry);
+ }
+ if (starts_leftover)
+ leave_peeled_trips(env, entry);
+ return err;
+}
+
+/*
+ * The current state gets to env->insn_idx. Leave the loops that the insn is not in. Enter
+ * the loops that it is in. Returns 1 if the state ends here.
+ * @backedge_loop: the state came back to the header of that loop.
+ * @starts_leftover: see bpf_flow_start_leftover()
+ */
+int bpf_flow_loop_enter(struct bpf_verifier_env *env, struct bpf_loop **backedge_loop,
+ bool *starts_leftover)
+{
+ struct bpf_func_state *frame = cur_func(env);
+ struct loop_stack_entry *entry;
+ bool pushed;
+ int err;
+
+ /* The callback came back to the helper that calls it, in the same trip. */
+ if (frame->callback_depth)
+ return enter_callback(env, true, backedge_loop);
+
+ loop_stack_pop(env);
+
+ err = loop_stack_push(env, &pushed);
+ if (err)
+ return err;
+
+ entry = bpf_top_loop(frame);
+ if (!entry || entry->loop != cur_aux(env)->loop)
+ return enter_callback(env, false, NULL);
+ if (env->path_sensitive)
+ return (pushed ? failing_path_trip(env, entry) :
+ next_trip(env, entry, starts_leftover)) ?:
+ enter_callback(env, false, NULL);
+ if (pushed) {
+ *starts_leftover = !try_peel_trip(entry);
+ return *starts_leftover ? 0 : enter_callback(env, false, NULL);
+ }
+
+ *backedge_loop = entry->loop;
+ err = next_trip(env, entry, starts_leftover);
+ if (err || *starts_leftover)
+ return err;
+ return enter_callback(env, false, NULL);
+}
+
/* Can the main program return 0? */
static bool zero_is_valid_retval(struct bpf_verifier_env *env)
{
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 15/49] bpf: Peel the first trips of a loop and join the other trips
2026-11-20 7:41 ` [PATCH bpf-next 15/49] bpf: Peel the first trips of a loop and join the other trips Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 10674 bytes --]
> bpf: Peel the first trips of a loop and join the other trips
TL;DR:
- check_loop_guard() still uses removed loop_stack_entry members, so it likely
fails to compile
- Peeling and loop-stack code is never reached: no writers or callers, unused
flow_new_call_loop()
- enter_recursion() may update the caller's state instead of the callee's
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> index cc90e237d535..77f1fe969a2a 100644
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
[ ... ]
> @@ -420,6 +422,24 @@ static inline u32 bpf_xreg_to_spi(u32 xregno)
> return xregno - MAX_BPF_EXT_REG;
> }
>
> +/*
> + * A state in a loop, see bpf_flow_loop_enter()
> + *
> + * entry -> [0] -> [1] -> ... -> [d-1] -> +----------+ <--+
> + * peeled trips, kept apart | LEFTOVER | |
> + * | widened |----+
> + * +----------+
> + */
> +struct loop_stack_entry {
> + struct bpf_loop *loop;
> + /* How many times the state came back to the header. A join has the larger number. */
> + u32 trip;
> + /* The state that entered the loop. What comes back to the header is for that state. */
> + u32 entry_id;
> + u32 held_cnt; /* when the state entered the loop */
> + bool peeled; /* in a peeled trip, not in the leftover */
> +};
> +
The members loop_id, terminates and guarded are gone from
struct loop_stack_entry. iter_stops_loops(), loop_stack_pop() and
print_verifier_state() are converted, but check_loop_guard() in
kernel/bpf/verifier.c, which do_check_block() calls for every insn, is
not:
if (loop->rt_bounded || !env->insn_aux_data[env->insn_idx].guard_place ||
entry->loop_id != h)
return 0;
if (verifier_bug_if(!entry->terminates && !entry->guarded, env,
"nothing stops the loop at %d", h))
return -EFAULT;
if (!entry->guarded && !bpf_loop_walk_may_give_up(entry))
return 0;
Doesn't this fail to compile, since entry->loop_id, entry->terminates and
entry->guarded no longer exist?
Even once it compiles, check_loop_guard() decides whether a loop needs a
may_goto exit from terminates and guarded, and the verifier_bug_if() assumes
one of them is set. The new peeled/leftover entries carry neither. Should
the guard logic be restated in terms of entry->loop and entry->peeled?
iter_stops_loops() also no longer skips a loop with a known trip count
(the old "if (entry->terminates || stops) continue;"), so it now calls
bpf_flow_next_is_no_bound() for such loops.
None of the later patches up to a50a112fb866 ("bpf: Delete the equations
that say nothing") touch check_loop_guard().
[ ... ]
> @@ -766,8 +812,29 @@ struct bpf_loop {
> u32 nr_insns;
> /* not rt_bounded: there are places for may_goto, see guard_to in bpf_insn_aux_data */
> bool has_guards;
> +
> + /* What one pass gives to the next pass */
> + struct_group_tagged(bpf_loop_precision, precision,
> + u32 peel_cnt; /* trips before the leftover */
> + /* while bpf_loop_per_pass.cost is not above @budget: */
> + u64 budget;
> + u32 max_peel_cnt; /* peel up to this trip */
> + );
> + bool partitioned; /* the last partition was for this loop */
> + struct bpf_loop_per_pass this_pass;
> };
Is anything writing peel_cnt, max_peel_cnt, budget, partitioned or
suspect_bit, or this_pass.cost and this_pass.has_unproven? I could not
find a writer in this patch or later in the series.
The precision group is never read or copied between passes either. Since
the struct is zeroed at allocation, try_peel_trip() sees:
if (entry->trip >= loop->peel_cnt &&
(entry->trip >= loop->max_peel_cnt || !bpf_loop_partition_is_active(loop)))
return false;
which is always true for peel_cnt == 0 and max_peel_cnt == 0, so no trip is
ever peeled. this_pass.peeled_cnt then stays 0, and
check_held_in_leftover() uses it as the allowance for locks. Likewise
"st->suspect_loops |= entry->loop->suspect_bit" always ORs in 0.
The commit message says "A later pass peels if an error asks for it, see
try_peel_trip()". Can any code path peel a trip with this patch applied?
Similarly, scalar.grew_once is only ever cleared, and the join_range()
function that its comment refers to is not in the tree.
> diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
> index cbbc663acac8..2c1835d1b3ef 100644
> --- a/kernel/bpf/flow.c
> +++ b/kernel/bpf/flow.c
> @@ -311,3 +311,20 @@ int bpf_check_limits(struct bpf_verifier_env *env)
> env->over_limit = true;
> return -E2BIG;
> }
> +
> +static struct bpf_loop *flow_new_call_loop(struct bpf_verifier_env *env, enum bpf_loop_kind kind,
> + int header, int func)
> +{
> + struct bpf_loop *loop = kvzalloc_obj(*loop, BPF_VERIFIER_GFP);
> +
> + env->mem += sizeof(*loop);
> + if (!loop)
> + return NULL;
> + loop->kind = kind;
> + loop->header = header;
> + /* the helper stops, the function checks the stack, see check_recursion_guard() */
> + loop->rt_bounded = true;
> + loop->nr_insns = func < 0 ? 1 :
> + env->subprog_info[func + 1].start - env->subprog_info[func].start;
> + return loop;
> +}
Is flow_new_call_loop() ever called? It is static and nothing in the tree
calls it, so gcc and clang will warn about an unused function, which breaks
the build with CONFIG_WERROR=y.
It is also the only allocator of BPF_LOOP_CALLBACK and BPF_LOOP_RECURSION
loops, so insn_aux_data[].callback_loop and .recursion_loop are never set
(only reset to NULL in bpf_clear_insn_aux_data()). enter_callback() and
enter_recursion() then always take their "if (!loop) return 0;" path.
In the same way, bpf_flow_loop_enter() is declared in bpf_verifier.h as
"/* verifier.c, for flow.c */", but nothing calls it, so loop_stack_push()
and loop_stack_pop() never run. The commit message says "The calls of a
callback and recursive calls go around like the body of a loop" and
"bpf_flow_loop_enter() keeps the loop stack of a frame". Should the
callers be added here, or the unused allocator and declaration be left out
until a patch uses them?
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index c0a83c10cf27..808adc4b7f82 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -11497,6 +11525,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
> return -ENOMEM;
> entry.frame[0] = callee;
> entry.in_sleepable = state->in_sleepable;
> + entry.suspect_loops = state->suspect_loops;
> entry.callers_unknown = state->callers_unknown;
> init_func_state(env, callee, BPF_MAIN_FUNC, 0, subprog);
> /* the frame that says so is not there when the function is verified */
> @@ -11504,7 +11533,8 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
> for (i = BPF_REG_1; i <= BPF_REG_5; i++)
> if (live & BIT(i))
> callee->regs[i] = caller->regs[i];
> - err = grow_stack_arg_slots(env, callee, bpf_in_stack_arg_cnt(&env->subprog_info[subprog]));
> + i = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);
> + err = grow_stack_arg_slots(env, callee, i) ?: enter_recursion(env, callee);
> if (err) {
> bpf_free_verifier_state(env, &entry, false);
> return err;
> @@ -11517,6 +11547,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
> bpf_free_verifier_state(env, &entry, false);
> return err;
> }
> + entry.max_held_cnt = state->max_held_cnt;
> callee->entry_refs = entry.acquired_refs;
> callee->entry_rcu_locks = entry.active_rcu_locks;
> callee->entry_preempt_locks = entry.active_preempt_locks;
[ ... ]
> +/* @callee is the frame of a function that is verified on its own */
> +static int enter_recursion(struct bpf_verifier_env *env, struct bpf_func_state *callee)
> +{
> + u32 cycle = env->subprog_info[callee->subprogno].cycle;
> + struct bpf_loop *loop = env->insn_aux_data[env->subprog_info[cycle].start].recursion_loop;
> + struct loop_stack_entry *entry, *from = env->cur_state->frame[0]->loop_stack;
> + bool starts_leftover;
> + int err = 0;
> +
> + if (!loop)
> + return 0;
> + entry = push_loop(env, callee, loop);
> + if (!entry)
> + return -ENOMEM;
> + starts_leftover = !try_peel_trip(entry);
> + if (env->cur_state->frame[0]->loop_stack_cnt && from->loop == loop) {
> + *entry = *from;
> + starts_leftover = false;
> + err = next_trip(env, entry, &starts_leftover);
> + }
> + if (starts_leftover)
> + leave_peeled_trips(env, entry);
> + return err;
> +}
enter_recursion() works on the callee frame of the local "entry" state that
call_func_on_its_own() is building, but leave_peeled_trips() and
check_held_in_leftover() act on env->cur_state, which is the caller's
state. Can this write to the wrong state?
leave_peeled_trips() does:
bpf_for_each_reg_in_vstate(env->cur_state, frame, reg, reg->grew_once = false);
so the grew_once flags of the waiting caller are cleared, while the
callee's R1-R5 (copied from the caller just before enter_recursion())
keep the caller's grew_once values. The stack args are copied after the
call and do get the cleared values. With try_peel_trip() always false in
pass 0, this happens on every first call into a recursive function from
outside its cycle.
Also, next_trip() -> check_held_in_leftover() sets
"st->suspect_loops |= entry->loop->suspect_bit" and st->max_held_cnt on the
caller's state. call_func_on_its_own() copies max_held_cnt after
enter_recursion() but suspect_loops before it:
entry.suspect_loops = state->suspect_loops;
...
entry.max_held_cnt = state->max_held_cnt;
so the callee path does not carry the suspect bit of the recursion loop.
Should leave_peeled_trips() run on the callee entry state, and
entry.suspect_loops be copied after enter_recursion()?
Separately, the other loop entry points treat env->path_sensitive
specially, for example in enter_callback():
if (env->path_sensitive)
return failing_path_trip(env, entry);
enter_recursion() has no such branch. func_is_on_its_own() does not look
at path_sensitive, so in a path-sensitive run a first recursive entry that
cannot peel sets loop->this_pass.has_leftover, clears entry->peeled, and
never calls failing_path_trip() for trip 0. The cleared peeled flag is
copied into re-entries by "*entry = *from" and reaches
exit_func_on_its_own(), which calls check_held_in_leftover() without
checking path_sensitive.
Is a "if (env->path_sensitive)" branch missing in enter_recursion() for the
first entry?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 16/49] bpf: Add equations and their arithmetic
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (14 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 15/49] bpf: Peel the first trips of a loop and join the other trips Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:24 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state Alexei Starovoitov
` (32 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A state has a range for each xreg. It does not have the relation of two
xregs:
r2 = r1
r2 <<= 2
if r1 > 7 goto out r2 is 0..28, but nothing says so
An equation relates two or three numbers. A sym is a number that the
program had or could have had. There are three kinds:
xreg = coef * sym + off r2 = 4 * sym0 + 8
sym = coef * sym + coef * sym + off sum sym2 = sym0 + sym1
sym = what an insn makes of two exprs op sym3 = sym2 & 0xf8
All equations hold mod 2^64, as the registers do. Add the types and
the arithmetic on ranges that the next patches use:
scalar_scale() sym0 = 0..7 -> 4 * sym0 = 0..28
refine_from_product() 4 * sym0 = 8..20 -> sym0 = 2..5
bpf_expr_div_exact() (4 * sym0 + 8) / 4 -> sym0 + 2
bpf_find_next_diff() sym2 - sym0 -> sym1
If the product can wrap, the range tells nothing. A power of two still
tells the low bits. If a range becomes empty, the equations cannot
hold: the state is infeasible.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 14 +++
kernel/bpf/Makefile | 1 +
kernel/bpf/equation.c | 232 +++++++++++++++++++++++++++++++++++
kernel/bpf/equation.h | 112 +++++++++++++++++
kernel/bpf/states.c | 8 +-
kernel/bpf/verifier.c | 2 +-
6 files changed, 366 insertions(+), 3 deletions(-)
create mode 100644 kernel/bpf/equation.c
create mode 100644 kernel/bpf/equation.h
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 77f1fe969a2..4d9cddb3588 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -535,6 +535,18 @@ struct bpf_func_state {
#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \
MAX_CALL_FRAMES)
+/* See equation.c */
+struct bpf_equations {
+ struct bpf_xreg_equation *xreg_equations; /* the three tables are one allocation */
+ struct bpf_sym_equation *sym_equations;
+ struct scalar *syms;
+ u32 xreg_equation_cnt, sym_equation_cnt, sym_cnt;
+ u32 xreg_equation_cap, sym_equation_cap, sym_cap;
+ bool normalized; /* see normalize_equations() */
+ bool infeasible; /* the equations cannot hold: the program does not get here */
+ bool needs_refine; /* widened: the equations are not applied, see bpf_flow_next() */
+};
+
struct bpf_verifier_state {
/* call stack tracking */
struct bpf_func_state *frame[MAX_CALL_FRAMES];
@@ -2289,6 +2301,8 @@ int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
+void scalar_bounds_sync(struct scalar *s);
+bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
/* How many bytes are checked from the start of the packet, wherever the pointer is */
static inline int pkt_min_len(const struct bpf_reg_state *reg)
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 4e5bb513a8a..d1f9ce7cc0c 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o check_btf.o flow.o join.o
+obj-$(CONFIG_BPF_SYSCALL) += equation.o equation_join.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
obj-$(CONFIG_BPF_JIT) += trampoline.o
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
new file mode 100644
index 00000000000..d49c01fa057
--- /dev/null
+++ b/kernel/bpf/equation.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Each state has a table of equations. An equation relates two or three numbers.
+ *
+ * xreg = coef * sym + off r2 = 4 * sym0 + 8
+ * sym = coef * sym + coef * sym + off (sum) sym2 = sym0 + sym1
+ * sym = what an insn makes of two exprs (op) sym3 = (sym2 + 7) & 0xf8
+ *
+ * An xreg is a register, a stack slot or the trip index of a loop. The number in a pointer is
+ * its offset. A sym is a number that the program had or could have had. The state says: there
+ * are numbers for the syms, in their ranges, for which all equations hold (mod 2^64).
+ *
+ * insn deletes the equation of the xreg that it writes, the result gets a new equation
+ * jump refines an xreg -> its sym -> the other syms -> their xregs
+ * join guesses the equations, see equation_join.c
+ * prune checks them, see check_equations()
+ */
+#include <linux/bpf.h>
+#include <linux/cnum.h>
+#include <linux/filter.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/tnum.h>
+#include "equation.h"
+
+static u64 low_bits(u32 bits)
+{
+ return bits ? GENMASK_ULL(bits - 1, 0) : 0;
+}
+
+/* Arithmetic */
+
+static bool scalar_is_unknown(const struct scalar *s)
+{
+ return s->r64.size == U64_MAX && s->r32.size == U32_MAX && tnum_is_unknown(s->var_off) &&
+ s->step == 1;
+}
+
+static void scalar_scale(struct scalar *dst, const struct scalar *src, s64 coef)
+{
+ struct cnum64 r64 = src->r64;
+ struct cnum32 r32 = src->r32;
+ s32 coef32 = coef;
+ u64 size;
+
+ if (coef == 1) {
+ *dst = *src;
+ return;
+ }
+ if ((coef & 1) && scalar_is_unknown(src))
+ return scalar_set_unknown(dst);
+ *dst = *src;
+ scalar_step_scale(dst, coef);
+ dst->r64 = CNUM64_UNBOUNDED;
+ dst->r32 = CNUM32_UNBOUNDED;
+ if (!check_mul_overflow(abs64(coef), r64.size, &size) && size != U64_MAX)
+ dst->r64 = (struct cnum64){ coef * (r64.base + (coef < 0 ? r64.size : 0)), size };
+ size = abs64(coef32) * r32.size;
+ if (size < U32_MAX)
+ dst->r32 = (struct cnum32){ coef32 * (r32.base + (coef32 < 0 ? r32.size : 0)),
+ size };
+ dst->var_off = tnum_mul(tnum_const(abs64(coef)), dst->var_off);
+ if (coef < 0)
+ dst->var_off = tnum_neg(dst->var_off);
+}
+
+/* @dst = @a + @coef * @b. It is not reduced, see scalar_add(). */
+static void scalar_add_scaled(struct scalar *dst, const struct scalar *a,
+ const struct scalar *b, s64 coef)
+{
+ struct scalar tmp;
+
+ if (scalar_is_unknown(a) || ((coef & 1) && scalar_is_unknown(b)))
+ return scalar_set_unknown(dst);
+ scalar_scale(&tmp, b, coef);
+ scalar_add(dst, a, &tmp);
+}
+
+static void scalar_add_const(struct scalar *dst, u64 off)
+{
+ struct scalar tmp;
+
+ if (!off || scalar_is_unknown(dst))
+ return;
+ scalar_set_const(&tmp, off);
+ scalar_add(dst, dst, &tmp);
+}
+
+static void scalar_set_below(struct scalar *dst, u64 end)
+{
+ scalar_set_unknown(dst);
+ dst->r64 = cnum64_from_urange(0, end - 1);
+ scalar_bounds_sync(dst);
+}
+
+/* The ends of the ranges have the low bits that are known. False: there is no such number. */
+static bool scalar_align_bounds(struct scalar *s)
+{
+ u64 low = (s->var_off.mask & -s->var_off.mask) - 1;
+ u64 up = (s->var_off.value - s->r64.base) & low;
+ u64 down = (s->r64.base + s->r64.size - s->var_off.value) & low;
+ u32 up32 = (s->var_off.value - s->r32.base) & low;
+ u32 down32 = (s->r32.base + s->r32.size - s->var_off.value) & low;
+
+ if (up > s->r64.size || down > s->r64.size - up ||
+ up32 > s->r32.size || down32 > s->r32.size - up32)
+ return false;
+ if (!up && !down && !up32 && !down32)
+ return true;
+ s->r64 = (struct cnum64){ s->r64.base + up, s->r64.size - up - down };
+ s->r32 = (struct cnum32){ s->r32.base + up32, s->r32.size - up32 - down32 };
+ return scalar_intersect(s, s, 64);
+}
+
+/* The number is in @s and in @bounds. Returns true if @s got smaller. */
+static bool refine_bounds(struct bpf_equations *eqs, struct scalar *s, const struct scalar *bounds)
+{
+ struct scalar new;
+
+ if (scalar_subsumes(bounds, s))
+ return false;
+ new = *s;
+ if (!scalar_intersect(&new, bounds, 64) || !scalar_align_bounds(&new)) {
+ eqs->infeasible = true;
+ return false;
+ }
+ if (scalar_equal(s, &new))
+ return false;
+ *s = new;
+ eqs->normalized = false;
+ return true;
+}
+
+/* @coef * (the number in @s) is in @bounds */
+static bool refine_from_product(struct bpf_equations *eqs, struct scalar *s, s64 coef,
+ const struct scalar *bounds)
+{
+ u64 span, lo, hi, step = abs64(coef);
+ struct scalar new;
+ struct tnum bits;
+
+ if (is_plus_minus_one(coef)) {
+ scalar_scale(&new, bounds, coef);
+ return refine_bounds(eqs, s, &new);
+ }
+ /* the products are on an arc of @span + 1 numbers. [lo, hi] is @bounds from its start */
+ if (check_mul_overflow(step, s->r64.size, &span) || bounds->r64.size == U64_MAX) {
+ if (hweight64(step) != 1)
+ return false;
+ /* a shift to the left: the bits that stay are known */
+ bits = tnum_rshift(coef < 0 ? tnum_neg(bounds->var_off) : bounds->var_off,
+ ilog2(step));
+ scalar_set_unknown(&new);
+ new.var_off = (struct tnum){ bits.value, bits.mask | ~(U64_MAX >> ilog2(step)) };
+ return refine_bounds(eqs, s, &new);
+ }
+ lo = bounds->r64.base - coef * (s->r64.base + (coef < 0 ? s->r64.size : 0));
+ hi = lo + bounds->r64.size;
+ if (hi < lo && lo <= span)
+ return false;
+ lo = hi < lo || !lo ? 0 : div64_u64(lo - 1, step) + 1;
+ hi = div64_u64(min(hi, span), step);
+ if (lo > hi) {
+ eqs->infeasible = true;
+ return false;
+ }
+ scalar_set_unknown(&new);
+ new.r64.base = s->r64.base + (coef > 0 ? lo : s->r64.size - hi);
+ new.r64.size = hi - lo;
+ return new.r64.size != U64_MAX && refine_bounds(eqs, s, &new);
+}
+
+/* @expr = (@expr - @off) / @coef. False if the result is not an expr. */
+bool bpf_expr_div_exact(struct expr *expr, u64 off, s64 coef)
+{
+ struct expr quot;
+
+ if (!coef)
+ return false;
+ if (is_plus_minus_one(coef))
+ quot = make_expr(expr->coef * coef, expr->var, (expr->off - off) * coef);
+ else
+ quot = make_expr(div64_s64(expr->coef, coef), expr->var,
+ div64_s64(expr->off - off, coef));
+ if (quot.coef * coef != expr->coef || quot.off * coef != expr->off - off)
+ return false;
+ *expr = quot;
+ return true;
+}
+
+/*
+ * @a - @b as an expr. There can be more than one. False: no more.
+ * The search goes around the table once, from the equation that @pos starts with.
+ */
+bool bpf_find_next_diff(struct bpf_verifier_env *env, const struct bpf_equations *eqs,
+ const struct expr *a, const struct expr *b, struct equation_pos *pos,
+ struct expr *diff)
+{
+ const struct bpf_sym_equation *sym_eq;
+ u32 seen_cnt = pos->seen_cnt;
+ struct expr factor;
+ int i, j;
+
+ if (!a->coef || !b->coef || a->var == b->var) {
+ *diff = make_expr(a->coef - b->coef, a->coef ? a->var : b->var, a->off - b->off);
+ return !pos->seen_cnt++;
+ }
+ if (!var_is_sym(a->var) || !var_is_sym(b->var) || over_limit(env))
+ return false;
+ while (pos->seen_cnt < eqs->sym_equation_cnt) {
+ pos->next = pos->next < eqs->sym_equation_cnt ? pos->next : 0;
+ sym_eq = eqs->sym_equations + pos->next++;
+ pos->seen_cnt++;
+ i = find_sym_pos(sym_eq, var_symno(a->var));
+ j = i < 3 ? find_sym_pos(sym_eq, var_symno(b->var)) : 3;
+ if (j == 3 || is_op(sym_eq))
+ continue;
+ factor = make_expr(0, 0, is_plus_minus_one(sym_eq->coef[j]) ? -b->coef : a->coef);
+ if (!bpf_expr_div_exact(&factor, 0,
+ sym_eq->coef[is_plus_minus_one(sym_eq->coef[j]) ? j :
+ i]) ||
+ factor.off * sym_eq->coef[i] != a->coef ||
+ factor.off * sym_eq->coef[j] != -b->coef)
+ continue;
+ *diff = sym_expr(-factor.off * sym_eq->coef[3 - i - j], sym_eq->sym[3 - i - j],
+ a->off - b->off - factor.off * sym_eq->off);
+ bpf_add_cost(env, pos->seen_cnt - seen_cnt, BPF_COST_SCAN);
+ return true;
+ }
+ bpf_add_cost(env, pos->seen_cnt - seen_cnt, BPF_COST_SCAN);
+ return false;
+}
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
new file mode 100644
index 00000000000..cb7132739a3
--- /dev/null
+++ b/kernel/bpf/equation.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __BPF_EQUATION_H
+#define __BPF_EQUATION_H
+
+#include <linux/bpf_verifier.h>
+
+/* A var is an xreg, see bpf_xreg(), or a sym */
+#define VAR_SYM BIT_ULL(63)
+#define VAR_NONE U64_MAX
+#define NO_SYM U32_MAX
+
+/* coef * var + off. A constant has no var and coef == 0. */
+struct expr {
+ s64 coef;
+ u64 off;
+ u64 var;
+};
+
+/* xreg = coef * sym + off. The table is sorted by xreg. */
+struct bpf_xreg_equation {
+ u64 xreg;
+ s64 coef;
+ u64 off;
+ u32 sym;
+ bool is_ptr; /* the number is the offset of a pointer */
+};
+
+/*
+ * A sum: 0 = coef[0] * sym[0] + coef[1] * sym[1] + coef[2] * sym[2] + off, coef[0] is -1.
+ * An op: sym[0] is what @insn_idx makes of the operands, see rhs_expr(). coef[0] is 1.
+ */
+struct bpf_sym_equation {
+ s64 coef[3];
+ u32 sym[3];
+ s32 insn_idx; /* that made the equation, -1 for a sum of a join */
+ u64 off, off_src;
+};
+
+/* See bpf_find_next_diff() */
+struct equation_pos {
+ u32 next, seen_cnt;
+};
+
+/* The pass is over its budget. It finds no more equations, a scan in a loop does not start. */
+static inline bool over_limit(struct bpf_verifier_env *env)
+{
+ return env->over_limit || bpf_check_limits(env);
+}
+
+static inline bool var_is_sym(u64 var)
+{
+ return var != VAR_NONE && (var & VAR_SYM);
+}
+
+static inline u32 var_symno(u64 var)
+{
+ return var;
+}
+
+static inline u64 abs64(s64 x)
+{
+ return x < 0 ? -(u64)x : x;
+}
+
+static inline bool is_plus_minus_one(s64 coef)
+{
+ return coef == 1 || coef == -1;
+}
+
+static inline struct expr make_expr(s64 coef, u64 var, u64 off)
+{
+ return (struct expr){ coef, off, coef ? var : VAR_NONE };
+}
+
+static inline struct expr sym_expr(s64 coef, u32 sym, u64 off)
+{
+ return make_expr(coef, VAR_SYM | sym, off);
+}
+
+static inline struct expr expr_scale(struct expr expr, s64 coef, u64 off)
+{
+ return make_expr(expr.coef * coef, expr.var, expr.off * coef + off);
+}
+
+static inline bool expr_equal(const struct expr *a, const struct expr *b)
+{
+ return a->coef == b->coef && a->off == b->off && a->var == b->var;
+}
+
+/* Operand @i on the right side */
+static inline struct expr rhs_expr(const struct bpf_sym_equation *sym_eq, int i)
+{
+ return sym_expr(sym_eq->coef[i + 1], sym_eq->sym[i + 1], i ? sym_eq->off_src : sym_eq->off);
+}
+
+static inline bool is_op(const struct bpf_sym_equation *sym_eq)
+{
+ return sym_eq->coef[0] == 1;
+}
+
+/* The position of @sym in @sym_eq. 3 if it is not there. */
+static inline int find_sym_pos(const struct bpf_sym_equation *sym_eq, u32 sym)
+{
+ return sym_eq->sym[0] == sym ? 0 : sym_eq->sym[1] == sym ? 1 :
+ sym_eq->sym[2] == sym ? 2 : 3;
+}
+
+bool bpf_expr_div_exact(struct expr *expr, u64 off, s64 coef);
+bool bpf_find_next_diff(struct bpf_verifier_env *env, const struct bpf_equations *eqs,
+ const struct expr *a, const struct expr *b, struct equation_pos *pos,
+ struct expr *diff);
+#endif
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 1e264757be4..df1866c2b55 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -6,8 +6,7 @@
#include <linux/filter.h>
/* check %cur's range satisfies %old's */
-static bool range_within(const struct bpf_reg_state *old,
- const struct bpf_reg_state *cur)
+static bool range_within(const struct scalar *old, const struct scalar *cur)
{
if (!cnum64_is_subset(old->r64, cur->r64) ||
!cnum32_is_subset(old->r32, cur->r32))
@@ -35,6 +34,11 @@ static bool range_within(const struct bpf_reg_state *old,
(cur->base - old->base) % old->step == 0;
}
+bool scalar_subsumes(const struct scalar *old, const struct scalar *cur)
+{
+ return range_within(old, cur) && tnum_in(old->var_off, cur->var_off);
+}
+
static void renumber_id(struct bpf_verifier_env *env, u32 *id, u32 new_id)
{
struct bpf_idmap *map = &env->idmap_scratch;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e4c9ced59b2..49e98667b60 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2318,7 +2318,7 @@ static void __update_bounds_from_step(struct scalar *s)
static bool range_bounds_violation(const struct scalar *s);
-static void scalar_bounds_sync(struct scalar *s)
+void scalar_bounds_sync(struct scalar *s)
{
/* If the input reg_state is invalid, we can exit early */
if (range_bounds_violation(s))
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 16/49] bpf: Add equations and their arithmetic
2026-11-20 7:41 ` [PATCH bpf-next 16/49] bpf: Add equations and their arithmetic Alexei Starovoitov
@ 2026-11-20 8:24 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:24 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 4167 bytes --]
> bpf: Add equations and their arithmetic
TL;DR:
- Makefile lists equation_join.o but no such file exists in the series, so
builds with BPF_SYSCALL may fail
- comments and needs_refine refer to missing code
- unused static helpers may warn under -Werror
> diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
> --- a/kernel/bpf/Makefile
> +++ b/kernel/bpf/Makefile
> @@ -12,6 +12,7 @@ obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list
> obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
> obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
> obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o check_btf.o flow.o join.o
> +obj-$(CONFIG_BPF_SYSCALL) += equation.o equation_join.o
Is there a kernel/bpf/equation_join.c to go with this? The commit adds only
equation.c and equation.h, so every build with CONFIG_BPF_SYSCALL=y should
stop with "No rule to make target 'kernel/bpf/equation_join.o'".
The file does not appear anywhere in the series. A listing of kernel/bpf at
the last commit of the range (a50a112fb866) shows only equation.c and
equation.h, and the Makefile line is still there. The header comment of
equation.c also refers to it:
> * join guesses the equations, see equation_join.c
Was equation_join.c left out of the series, or should the object not be
listed until the patch that adds it?
[ ... ]
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -535,6 +535,18 @@ struct bpf_func_state {
> +/* See equation.c */
> +struct bpf_equations {
[ ... ]
> + bool normalized; /* see normalize_equations() */
> + bool infeasible; /* the equations cannot hold: the program does not get here */
> + bool needs_refine; /* widened: the equations are not applied, see bpf_flow_next() */
> +};
The comments here point at functions that do not exist. normalize_equations()
is added by a later commit, but bpf_flow_next() appears only in this comment
(flow.c has only bpf_flow_next_is_no_bound()), and check_equations(), named in
the equation.c header ("prune checks them, see check_equations()"), appears
only in comments.
Nothing in the series sets needs_refine to true either; the only store is
"st->equations.needs_refine = false;". At the end of the range states.c
refers to equations only through bpf_equations_clean(), and states_equal()
calls nothing that compares them.
Should the comments and the needs_refine field match the code that exists, or
are the missing pieces meant to be part of this series?
[ ... ]
> diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
[ ... ]
> +static u64 low_bits(u32 bits)
[ ... ]
> +static void scalar_add_scaled(struct scalar *dst, const struct scalar *a,
> + const struct scalar *b, s64 coef)
[ ... ]
> +static void scalar_add_const(struct scalar *dst, u64 off)
[ ... ]
> +static void scalar_set_below(struct scalar *dst, u64 end)
[ ... ]
> +static bool refine_from_product(struct bpf_equations *eqs, struct scalar *s, s64 coef,
> + const struct scalar *bounds)
This isn't a bug in the logic, but low_bits(), scalar_add_scaled(),
scalar_add_const(), scalar_set_below() and refine_from_product() have no
caller in this commit. Will compiling equation.c give five "defined but not
used [-Wunused-function]" warnings, which become errors with CONFIG_WERROR=y?
The callers arrive later: scalar_add_scaled() and scalar_add_const() in
fe0262cce204 ("bpf: Keep a table of equations in each state"), low_bits()
and refine_from_product() in 969a062e4b35 ("bpf: Refine all related numbers
after a jump"), and scalar_set_below() in 0462c546cfe6 ("bpf: Make equations
from ALU, spill, fill and call insns"). Until then each intermediate commit
warns, which affects bisection under -Werror. Could each helper be added in
the patch that first calls it?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (15 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 16/49] bpf: Add equations and their arithmetic Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:17 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 18/49] bpf: Refine all related numbers after a jump Alexei Starovoitov
` (31 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
What holds on one path has to be in the state of that path. Give each
state a table of equations. The state says: there are numbers for the
syms, in their ranges, for which all equations hold.
xreg equations sym equations syms
r1 = sym0 sym2 = sym0 + sym1 sym0 0..7
r2 = 4 * sym0 sym1 0..255
fp-8 = sym2 sym2 0..262
The three arrays are one allocation. It is copied, freed and charged
with the state. An insn deletes the sym equations that it made before,
so a loop does not make the table grow.
Each scan of a table adds to the cost. At the limit a scan does not
start, see over_limit(). The work on equations stops half way then, and
the pass proves nothing. The last state can get there after the last
test of the limits. So do_check() tests them again when it ends.
An equation does not outlive its xreg. bpf_find_var_bounds() has no
bounds for a register that is not set, or for a slot without a spill.
A pointer that became a scalar is another number.
The trip index of a loop is an xreg too. A later patch counts it.
The number in a pointer is its offset. The cpu can speculate past the
jump that refines it, so pointers need bypass_spec_v1. There are no
equations without CAP_BPF.
The log has the table: 0:r2=4*sym0+0 sym0=[0,+7]
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 22 ++
kernel/bpf/equation.c | 439 +++++++++++++++++++++++++++++++++++
kernel/bpf/equation.h | 58 +++++
kernel/bpf/flow.c | 2 +
kernel/bpf/log.c | 3 +
kernel/bpf/verifier.c | 10 +-
6 files changed, 533 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 4d9cddb3588..8d1a1ff87b7 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -115,6 +115,7 @@ struct bpf_reg_state {
* the dynptr to determine proper reference freeing.
* Similarly to dynptrs, we use ID to track "belonging" of a reference
* to a specific instance of bpf_iter.
+ * A scalar has no id, see struct bpf_equations.
*/
u32 id;
union {
@@ -422,6 +423,13 @@ static inline u32 bpf_xreg_to_spi(u32 xregno)
return xregno - MAX_BPF_EXT_REG;
}
+/* The trip index of a loop is an xreg too. The loops of a frame come after its slots. */
+#define BPF_XREG_TRIP MAX_BPF_XREG
+static inline u64 bpf_xreg(u32 frameno, u32 xregno)
+{
+ return (u64)frameno << 32 | xregno;
+}
+
/*
* A state in a loop, see bpf_flow_loop_enter()
*
@@ -438,6 +446,8 @@ struct loop_stack_entry {
u32 entry_id;
u32 held_cnt; /* when the state entered the loop */
bool peeled; /* in a peeled trip, not in the leftover */
+ /* an xreg: how many times the program came back to the header */
+ struct scalar trip_index;
};
#define LOOP_STACK_SIZE 16
@@ -548,6 +558,7 @@ struct bpf_equations {
};
struct bpf_verifier_state {
+ struct bpf_equations equations;
/* call stack tracking */
struct bpf_func_state *frame[MAX_CALL_FRAMES];
/* Acquired reference states */
@@ -1279,6 +1290,7 @@ struct bpf_verifier_env {
u64 max_cost, max_mem;
u64 pass_max_cost; /* see bpf_undo_precision() */
bool aborted; /* a thaw does not take it back */
+ bool no_mem; /* a table cannot grow, see bpf_check_limits() */
/* What a pass writes. Each pass starts from the snapshot. */
struct_group_tagged(bpf_env_per_pass, per_pass,
int err; /* the first error */
@@ -1422,6 +1434,7 @@ struct bpf_verifier_env {
struct bpf_iarray *succ;
struct bpf_iarray *gotox_tmp_buf;
int *idoms;
+ struct bpf_equation_env *eq_env;
struct bpf_flow *flow;
};
@@ -2120,6 +2133,7 @@ static inline bool bpf_is_kfunc_sleepable(struct bpf_call_arg_meta *meta)
return meta->kfunc_flags & KF_SLEEPABLE;
}
bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta);
+bool bpf_grow_array(struct bpf_verifier_env *env, void *array, u32 *cap, u32 cnt, size_t size);
struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem);
int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off);
bool bpf_insn_is_cond_jump(u8 code);
@@ -2301,6 +2315,14 @@ int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
+/* equation.c */
+int bpf_equation_env_init(struct bpf_verifier_env *env);
+void bpf_equation_env_free(struct bpf_verifier_env *env);
+int bpf_equations_copy(struct bpf_verifier_env *env, struct bpf_equations *dst,
+ const struct bpf_equations *src);
+void bpf_equations_free(struct bpf_equations *eqs);
+void bpf_equations_print(struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
+u32 bpf_equations_bytes(const struct bpf_equations *eqs);
void scalar_bounds_sync(struct scalar *s);
bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index d49c01fa057..a85fef30e8e 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -230,3 +230,442 @@ bool bpf_find_next_diff(struct bpf_verifier_env *env, const struct bpf_equations
bpf_add_cost(env, pos->seen_cnt - seen_cnt, BPF_COST_SCAN);
return false;
}
+
+/* The table */
+
+/* Make room for @cnt elements. @array is the address of the pointer. New elements are not set. */
+bool bpf_grow_array(struct bpf_verifier_env *env, void *array, u32 *cap, u32 cnt, size_t size)
+{
+ void *p;
+
+ if (cnt <= *cap)
+ return true;
+ p = kvrealloc(*(void **)array, array_size(2 * cnt, size), BPF_VERIFIER_GFP);
+ if (!p) {
+ env->no_mem = true;
+ return false;
+ }
+ *(void **)array = p;
+ env->mem += (2 * cnt - *cap) * size;
+ *cap = 2 * cnt;
+ return true;
+}
+
+u32 bpf_equations_bytes(const struct bpf_equations *eqs)
+{
+ return eqs->xreg_equation_cap * sizeof(*eqs->xreg_equations) +
+ eqs->sym_equation_cap * sizeof(*eqs->sym_equations) +
+ eqs->sym_cap * sizeof(*eqs->syms);
+}
+
+/* @dst becomes a copy of @src. @grow: with room for more. */
+static bool realloc_equations(struct bpf_verifier_env *env, struct bpf_equations *dst,
+ const struct bpf_equations *src, bool grow)
+{
+ struct bpf_equations new = *src;
+ u32 bytes, spare;
+
+ new.xreg_equation_cap = src->xreg_equation_cnt +
+ (grow ? src->xreg_equation_cnt / 2 + 4 : 0);
+ new.sym_equation_cap = src->sym_equation_cnt + (grow ? src->sym_equation_cnt / 2 + 4 : 0);
+ new.sym_cap = src->sym_cnt + (grow ? src->sym_cnt / 2 + 4 : 0);
+ bytes = kmalloc_size_roundup(bpf_equations_bytes(&new));
+ spare = (bytes - bpf_equations_bytes(&new)) /
+ (sizeof(*new.xreg_equations) + sizeof(*new.sym_equations) + sizeof(*new.syms));
+ new.xreg_equation_cap += spare;
+ new.sym_equation_cap += spare;
+ new.sym_cap += spare;
+ bpf_add_cost(env, BPF_COST_INSN + bytes / 64, BPF_COST_SCAN);
+ new.xreg_equations = kvmalloc(bytes, BPF_VERIFIER_GFP);
+ if (!new.xreg_equations) {
+ env->no_mem = true;
+ return false;
+ }
+ new.sym_equations = (void *)(new.xreg_equations + new.xreg_equation_cap);
+ new.syms = (void *)(new.sym_equations + new.sym_equation_cap);
+ memcpy(new.xreg_equations, src->xreg_equations,
+ src->xreg_equation_cnt * sizeof(*src->xreg_equations));
+ memcpy(new.sym_equations, src->sym_equations,
+ src->sym_equation_cnt * sizeof(*src->sym_equations));
+ memcpy(new.syms, src->syms, src->sym_cnt * sizeof(*src->syms));
+ kvfree(dst->xreg_equations);
+ *dst = new;
+ return true;
+}
+
+int bpf_equations_copy(struct bpf_verifier_env *env, struct bpf_equations *dst,
+ const struct bpf_equations *src)
+{
+ if (src->sym_cnt)
+ return realloc_equations(env, dst, src, false) ? 0 : -ENOMEM;
+ bpf_equations_free(dst);
+ dst->infeasible = src->infeasible;
+ return 0;
+}
+
+u64 bpf_new_sym(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ const struct scalar *bounds)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct scalar copy = *bounds;
+
+ if (!bpf_grow_array(env, &eq_env->scratch, &eq_env->scratch_cap,
+ eqs->sym_cnt + 1, sizeof(*eq_env->scratch)) ||
+ (eqs->sym_cnt == eqs->sym_cap &&
+ !realloc_equations(env, eqs, eqs, true)))
+ return VAR_NONE;
+ eq_env->scratch[eqs->sym_cnt].refined_sweep = 0;
+ copy.grew_once = false;
+ eqs->syms[eqs->sym_cnt] = copy;
+ eqs->normalized = false;
+ return VAR_SYM | eqs->sym_cnt++;
+}
+
+/* Put a sym with coef 1 or -1 on the left side. False if there is none, or it says nothing. */
+static bool isolate_sym(struct bpf_sym_equation *sym_eq)
+{
+ int i, j, lhs = -1;
+ s64 sign;
+
+ for (i = 0; i < 3; i++) {
+ for (j = i + 1; j < 3; j++) {
+ if (sym_eq->sym[j] != sym_eq->sym[i])
+ continue;
+ sym_eq->coef[i] += sym_eq->coef[j];
+ sym_eq->coef[j] = 0;
+ }
+ if (lhs < 0 && is_plus_minus_one(sym_eq->coef[i]))
+ lhs = i;
+ }
+ if (lhs < 0)
+ return false;
+ for (i = lhs; i > 0; i--) {
+ swap(sym_eq->coef[i], sym_eq->coef[i - 1]);
+ swap(sym_eq->sym[i], sym_eq->sym[i - 1]);
+ }
+ if (!sym_eq->coef[1]) {
+ swap(sym_eq->coef[1], sym_eq->coef[2]);
+ swap(sym_eq->sym[1], sym_eq->sym[2]);
+ }
+ sign = -sym_eq->coef[0];
+ for (i = 0; i < 3; i++) {
+ sym_eq->coef[i] *= sign;
+ sym_eq->sym[i] = sym_eq->coef[i] ? sym_eq->sym[i] : NO_SYM;
+ }
+ sym_eq->off *= sign;
+ return sym_eq->coef[1];
+}
+
+/* Delete the equations that are marked, and the equations of @insn_idx */
+static void forget_sym_equations(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ s32 insn_idx)
+{
+ struct bpf_sym_equation *sym_eq, *next = eqs->sym_equations;
+
+ for_each_sym_equation(env, eqs, sym_eq) {
+ if (sym_eq->sym[0] == NO_SYM || sym_eq->insn_idx == insn_idx)
+ continue;
+ if (next != sym_eq)
+ *next = *sym_eq;
+ next++;
+ }
+ eqs->sym_equation_cnt = next - eqs->sym_equations;
+}
+
+/* An insn deletes the equations that it made before. So there are at most as many as insns. */
+static bool append_sym_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ struct bpf_sym_equation sym_eq)
+{
+ if (env->no_mem)
+ return false;
+ if (sym_eq.insn_idx >= 0 && !env->eq_env->equation_added)
+ forget_sym_equations(env, eqs, sym_eq.insn_idx);
+ env->eq_env->equation_added = true;
+ if (eqs->sym_equation_cnt == eqs->sym_equation_cap &&
+ !realloc_equations(env, eqs, eqs, true))
+ return false;
+ eqs->sym_equations[eqs->sym_equation_cnt++] = sym_eq;
+ eqs->normalized = false;
+ return true;
+}
+
+bool bpf_add_sym_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ struct bpf_sym_equation sym_eq)
+{
+ return isolate_sym(&sym_eq) && append_sym_equation(env, eqs, sym_eq);
+}
+
+/* @res is what @insn makes of @op */
+bool bpf_add_op(struct bpf_verifier_env *env, struct bpf_equations *eqs, s32 insn,
+ const struct expr *op, const struct expr *res)
+{
+ return res->coef == 1 && !res->off && var_is_sym(res->var) &&
+ res->var != op[0].var && res->var != op[1].var &&
+ append_sym_equation(env, eqs, (struct bpf_sym_equation){
+ { 1, op[0].coef, op[1].coef },
+ { res->var, op[0].var, op[1].var },
+ insn, op[0].off, op[1].off });
+}
+
+/* An insn like @insn made @res of @op, or it makes a constant */
+bool bpf_find_op(struct bpf_verifier_env *env, const struct bpf_equations *eqs, s32 insn,
+ const struct expr *op, struct expr *res)
+{
+ const struct bpf_insn *a = &env->prog->insnsi[insn], *b;
+ u8 same = BPF_OP(a->code) == BPF_END ? 0xff : ~BPF_X;
+ const struct bpf_sym_equation *sym_eq;
+ struct scalar bounds, src;
+ struct expr expr[2];
+
+ if (!op[0].coef && !op[1].coef) {
+ scalar_set_const(&bounds, op[0].off);
+ scalar_set_const(&src, op[1].off);
+ scalar_alu(&bounds, &src, a);
+ *res = make_expr(0, 0, bounds.var_off.value);
+ return tnum_is_const(bounds.var_off);
+ }
+ if (over_limit(env))
+ return false;
+ for_each_sym_equation(env, eqs, sym_eq) {
+ if (!is_op(sym_eq) || sym_eq->sym[0] == NO_SYM)
+ continue;
+ b = &env->prog->insnsi[sym_eq->insn_idx];
+ expr[0] = rhs_expr(sym_eq, 0);
+ expr[1] = rhs_expr(sym_eq, 1);
+ if (((a->code ^ b->code) & same) || a->off != b->off ||
+ !expr_equal(&expr[0], &op[0]) || !expr_equal(&expr[1], &op[1]))
+ continue;
+ *res = sym_expr(1, sym_eq->sym[0], 0);
+ return true;
+ }
+ return false;
+}
+
+/* The equation of @xreg, or of the first xreg after it */
+static struct bpf_xreg_equation *xreg_equation_pos(const struct bpf_equations *eqs, u64 xreg)
+{
+ u32 lo = 0, hi = eqs->xreg_equation_cnt, mid;
+
+ while (lo < hi) {
+ mid = (lo + hi) / 2;
+ if (eqs->xreg_equations[mid].xreg < xreg)
+ lo = mid + 1;
+ else
+ hi = mid;
+ }
+ return eqs->xreg_equations + lo;
+}
+
+struct bpf_xreg_equation *bpf_find_xreg_equation(const struct bpf_equations *eqs, u64 xreg)
+{
+ struct bpf_xreg_equation *xreg_eq = xreg_equation_pos(eqs, xreg);
+
+ return xreg_eq < eqs->xreg_equations + eqs->xreg_equation_cnt &&
+ xreg_eq->xreg == xreg ? xreg_eq : NULL;
+}
+
+/* @xreg holds @expr. It has no equation if @expr has no sym. */
+void bpf_set_xreg_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs, u64 xreg,
+ const struct expr *expr, bool is_ptr)
+{
+ struct bpf_xreg_equation *xreg_eq, *end;
+ bool found, insert = expr && var_is_sym(expr->var);
+
+ if (insert && eqs->xreg_equation_cnt == eqs->xreg_equation_cap)
+ insert = realloc_equations(env, eqs, eqs, true);
+ xreg_eq = xreg_equation_pos(eqs, xreg);
+ end = eqs->xreg_equations + eqs->xreg_equation_cnt;
+ found = xreg_eq < end && xreg_eq->xreg == xreg;
+ if (found != insert) {
+ bpf_add_cost(env, end - xreg_eq, BPF_COST_SCAN);
+ memmove(xreg_eq + insert, xreg_eq + found,
+ (end - xreg_eq - found) * sizeof(*xreg_eq));
+ eqs->xreg_equation_cnt += insert - found;
+ }
+ if (insert)
+ *xreg_eq = (struct bpf_xreg_equation){ xreg, expr->coef, expr->off,
+ var_symno(expr->var), is_ptr };
+ eqs->normalized &= !found && !insert;
+}
+
+void bpf_equations_free(struct bpf_equations *eqs)
+{
+ kvfree(eqs->xreg_equations);
+ memset(eqs, 0, sizeof(*eqs));
+}
+
+/* The cpu can speculate past the jump that refines the offset */
+static bool ptr_off_is_tracked(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
+{
+ return env->bypass_spec_v1 && bpf_reg_has_range(reg) && reg->type != PTR_TO_INSN &&
+ !(reg->type & PTR_UNTRUSTED);
+}
+
+/* The bounds of @var. NULL if there are none. */
+struct scalar *bpf_find_var_bounds(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st, u64 var, bool *is_ptr)
+{
+ u32 fr = var >> 32, idx = var;
+ struct bpf_reg_state *reg;
+ struct bpf_func_state *f;
+
+ *is_ptr = false;
+ if (var_is_sym(var))
+ return &st->equations.syms[idx];
+ f = fr <= st->curframe ? st->frame[fr] : NULL;
+ if (!f)
+ return NULL;
+ if (idx >= BPF_XREG_TRIP)
+ return idx - BPF_XREG_TRIP < f->loop_stack_cnt ?
+ &f->loop_stack[idx - BPF_XREG_TRIP].trip_index : NULL;
+ reg = idx < MAX_BPF_REG ? &f->regs[idx] : !bpf_xreg_is_slot(idx) ? NULL :
+ bpf_get_spilled_reg(bpf_xreg_to_spi(idx), f, BIT(STACK_SPILL));
+ if (!reg || reg->type == NOT_INIT)
+ return NULL;
+ *is_ptr = reg->type != SCALAR_VALUE;
+ return !*is_ptr || ptr_off_is_tracked(env, reg) ? scalar_of(reg) : NULL;
+}
+
+/* A hidden register has no value, see MAX_BPF_XREG */
+u64 bpf_reg_to_xreg(const struct bpf_verifier_state *st, u32 regno)
+{
+ return regno <= BPF_REG_FP ? bpf_xreg(st->curframe, regno) : VAR_NONE;
+}
+
+/* A pointer that became a scalar is another number */
+static struct scalar *equation_xreg_bounds(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st,
+ const struct bpf_xreg_equation *xreg_eq)
+{
+ bool is_ptr;
+ struct scalar *s = bpf_find_var_bounds(env, st, xreg_eq->xreg, &is_ptr);
+
+ return is_ptr == xreg_eq->is_ptr ? s : NULL;
+}
+
+/* The value of @xreg: a constant, the right side of its equation, or the xreg itself */
+bool bpf_xreg_expr(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 xreg,
+ struct expr *expr, bool *is_ptr)
+{
+ const struct scalar *s = bpf_find_var_bounds(env, st, xreg, is_ptr);
+ const struct bpf_xreg_equation *xreg_eq;
+
+ bpf_add_cost(env, 1, BPF_COST_XREG);
+ if (!s)
+ return false;
+ if (tnum_is_const(s->var_off)) {
+ *expr = make_expr(0, 0, s->var_off.value);
+ return true;
+ }
+ xreg_eq = bpf_find_xreg_equation(&st->equations, xreg);
+ *expr = xreg_eq && xreg_eq->is_ptr == *is_ptr ?
+ sym_expr(xreg_eq->coef, xreg_eq->sym, xreg_eq->off) : make_expr(1, xreg, 0);
+ return true;
+}
+
+/* It is not reduced */
+static void eval_expr_nosync(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ const struct expr *expr, struct scalar *bounds)
+{
+ bool is_ptr;
+
+ bpf_add_cost(env, 1, BPF_COST_XREG);
+ if (!expr->coef)
+ return scalar_set_const(bounds, expr->off);
+ scalar_scale(bounds, var_is_sym(expr->var) ? &st->equations.syms[var_symno(expr->var)] :
+ bpf_find_var_bounds(env, st, expr->var, &is_ptr), expr->coef);
+ scalar_add_const(bounds, expr->off);
+}
+
+void bpf_eval_expr(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ const struct expr *expr, struct scalar *bounds)
+{
+ eval_expr_nosync(env, st, expr, bounds);
+ if (!expr->coef || (expr->coef == 1 && !expr->off) || scalar_is_unknown(bounds))
+ return;
+ bpf_add_cost(env, 4, BPF_COST_XREG);
+ scalar_bounds_sync(bounds);
+}
+
+void bpf_eval_diff(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ const struct expr *a, const struct expr *b, struct scalar *bounds)
+{
+ struct scalar b_bounds;
+
+ eval_expr_nosync(env, st, a, bounds);
+ eval_expr_nosync(env, st, b, &b_bounds);
+ scalar_add_scaled(bounds, bounds, &b_bounds, -1);
+ if (!scalar_is_unknown(bounds))
+ scalar_bounds_sync(bounds);
+}
+
+static void print_operand(struct bpf_verifier_log *log, const char *prefix, struct expr expr)
+{
+ if (expr.coef)
+ bpf_log(log, "%s%lld*sym%u%+lld", prefix, expr.coef, var_symno(expr.var), expr.off);
+ else
+ bpf_log(log, "%s%lld", prefix, expr.off);
+}
+
+static void print_sym_equation(struct bpf_verifier_log *log, const struct bpf_sym_equation *sym_eq)
+{
+ bpf_log(log, " sym%u=", sym_eq->sym[0]);
+ if (is_op(sym_eq)) {
+ bpf_log(log, "insn%d", sym_eq->insn_idx);
+ print_operand(log, "(", rhs_expr(sym_eq, 0));
+ print_operand(log, ",", rhs_expr(sym_eq, 1));
+ bpf_log(log, ")");
+ return;
+ }
+ bpf_log(log, "%lld*sym%u", sym_eq->coef[1], sym_eq->sym[1]);
+ if (sym_eq->coef[2])
+ bpf_log(log, "%+lld*sym%u", sym_eq->coef[2], sym_eq->sym[2]);
+ bpf_log(log, "%+lld", sym_eq->off);
+}
+
+void bpf_equations_print(struct bpf_verifier_env *env, const struct bpf_verifier_state *st)
+{
+ const struct bpf_equations *eqs = &st->equations;
+ struct bpf_verifier_log *log = &env->log;
+ const struct bpf_xreg_equation *xreg_eq;
+ const struct bpf_sym_equation *sym_eq;
+ u32 i, xregno;
+
+ for (xreg_eq = eqs->xreg_equations;
+ xreg_eq < eqs->xreg_equations + eqs->xreg_equation_cnt;
+ xreg_eq++) {
+ xregno = xreg_eq->xreg;
+ bpf_log(log,
+ xregno < MAX_BPF_REG ? " %llu:r%d" : xregno < BPF_XREG_TRIP ? " %llu:fp%d" :
+ " %llu:trip%d", xreg_eq->xreg >> 32, xregno < MAX_BPF_REG ? xregno :
+ xregno < BPF_XREG_TRIP ? (MAX_BPF_EXT_REG - xregno - 1) * BPF_REG_SIZE :
+ xregno - BPF_XREG_TRIP);
+ bpf_log(log, "=%lld*sym%u%+lld", xreg_eq->coef, xreg_eq->sym, xreg_eq->off);
+ }
+ for (sym_eq = eqs->sym_equations;
+ sym_eq < eqs->sym_equations + eqs->sym_equation_cnt; sym_eq++)
+ print_sym_equation(log, sym_eq);
+ for (i = 0; i < eqs->sym_cnt; i++)
+ bpf_log(log, " sym%u=[%lld,+%llu]", i, eqs->syms[i].r64.base,
+ eqs->syms[i].r64.size);
+}
+
+int bpf_equation_env_init(struct bpf_verifier_env *env)
+{
+ if (!env->bpf_capable)
+ return 0;
+ env->eq_env = kvzalloc_obj(*env->eq_env, BPF_VERIFIER_GFP);
+ return env->eq_env ? 0 : -ENOMEM;
+}
+
+void bpf_equation_env_free(struct bpf_verifier_env *env)
+{
+ if (!env->eq_env)
+ return;
+ kvfree(env->eq_env->scratch);
+ kvfree(env->eq_env->use_links);
+ kvfree(env->eq_env->phis);
+ kvfree(env->eq_env->phi_index);
+ bpf_equations_free(&env->eq_env->join_state.equations);
+ kvfree(env->eq_env);
+}
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index cb7132739a3..26c100e715d 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -41,6 +41,42 @@ struct equation_pos {
u32 next, seen_cnt;
};
+/* Scratch data of a function about a sym */
+struct sym_scratch {
+ u32 use_cnt; /* equations that use it */
+ /* propagate_bounds(): the sweep that has to look at its equations */
+ u32 refined_sweep;
+ u32 new_symno;
+ bool determined, by_op;
+ const struct bpf_sym_equation *op; /* that makes it */
+ struct expr cur_expr; /* check_equations(): its value in the other state */
+};
+
+struct bpf_equation_env {
+ struct sym_scratch *scratch;
+ u32 *use_links;
+ u32 scratch_cap, use_link_cap, refined_cnt;
+ bool has_expr[2], is_ptr[2], equation_added, divided;
+ /* equation_join.c */
+ struct bpf_verifier_state join_state; /* it has equations and no frames */
+ struct join_phi *phis;
+ u32 *phi_index;
+};
+
+#define for_each_sym_equation(env, eqs, pos) \
+ for (bpf_add_cost(env, (eqs)->sym_equation_cnt, BPF_COST_COPY), \
+ pos = (eqs)->sym_equations; \
+ pos < (eqs)->sym_equations + (eqs)->sym_equation_cnt; pos++)
+
+#define for_each_sum(env, eqs, pos) \
+ for_each_sym_equation(env, eqs, pos) \
+ if (!is_op(pos))
+
+#define for_each_xreg_equation(env, eqs, pos) \
+ for (bpf_add_cost(env, (eqs)->xreg_equation_cnt, BPF_COST_COPY), \
+ pos = (eqs)->xreg_equations; \
+ pos < (eqs)->xreg_equations + (eqs)->xreg_equation_cnt; pos++)
+
/* The pass is over its budget. It finds no more equations, a scan in a loop does not start. */
static inline bool over_limit(struct bpf_verifier_env *env)
{
@@ -105,8 +141,30 @@ static inline int find_sym_pos(const struct bpf_sym_equation *sym_eq, u32 sym)
sym_eq->sym[2] == sym ? 2 : 3;
}
+/* equation.c */
+u64 bpf_new_sym(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ const struct scalar *bounds);
+bool bpf_add_sym_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ struct bpf_sym_equation sym_eq);
+bool bpf_add_op(struct bpf_verifier_env *env, struct bpf_equations *eqs, s32 insn,
+ const struct expr *op, const struct expr *res);
+bool bpf_find_op(struct bpf_verifier_env *env, const struct bpf_equations *eqs, s32 insn,
+ const struct expr *op, struct expr *res);
+struct bpf_xreg_equation *bpf_find_xreg_equation(const struct bpf_equations *eqs, u64 xreg);
+void bpf_set_xreg_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs, u64 xreg,
+ const struct expr *expr, bool is_ptr);
+u64 bpf_reg_to_xreg(const struct bpf_verifier_state *st, u32 regno);
+struct scalar *bpf_find_var_bounds(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st, u64 var, bool *is_ptr);
+bool bpf_xreg_expr(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 xreg,
+ struct expr *expr, bool *is_ptr);
+void bpf_eval_expr(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ const struct expr *expr, struct scalar *bounds);
bool bpf_expr_div_exact(struct expr *expr, u64 off, s64 coef);
bool bpf_find_next_diff(struct bpf_verifier_env *env, const struct bpf_equations *eqs,
const struct expr *a, const struct expr *b, struct equation_pos *pos,
struct expr *diff);
+void bpf_eval_diff(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ const struct expr *a, const struct expr *b, struct scalar *bounds);
+
#endif
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 2c1835d1b3e..0cb56390ffd 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -303,6 +303,8 @@ int bpf_check_limits(struct bpf_verifier_env *env)
env->aborted |= bpf_must_abort();
if (env->aborted)
return -EINTR;
+ if (env->no_mem)
+ return -ENOMEM;
if (env->cost <= min(env->max_cost, env->pass_max_cost) && env->mem <= env->max_mem &&
!env->over_limit)
return 0;
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 7b1390d9054..8a905f4df9b 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -803,6 +803,9 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
verbose(env, "%s%d", i > first ? "," : "",
state->loop_stack[i].loop->header);
}
+ /* one table for all frames */
+ if (print_all && frameno == vstate->curframe)
+ bpf_equations_print(env, vstate);
verbose(env, "\n");
if (!print_all)
mark_verifier_state_clean(env);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 49e98667b60..199108c2226 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1755,6 +1755,7 @@ void bpf_charge_state(struct bpf_verifier_env *env, struct bpf_verifier_state *s
u32 bytes = 2 * sizeof(*st) + ksize(st->refs), fr;
struct bpf_func_state *f;
+ bytes += bpf_equations_bytes(&st->equations);
for (fr = 0; fr <= st->curframe && (f = st->frame[fr]); fr++)
bytes += kmalloc_size_roundup(sizeof(*f)) + ksize(f->stack) +
ksize(f->stack_arg_regs) + ksize(f->loop_stack);
@@ -1774,6 +1775,7 @@ void bpf_free_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
state->frame[i] = NULL;
}
kfree(state->refs);
+ bpf_equations_free(&state->equations);
if (free_self)
kfree(state);
}
@@ -1808,7 +1810,8 @@ int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_st
free_func_state(dst_state->frame[i]);
dst_state->frame[i] = NULL;
}
- err = copy_reference_state(dst_state, src);
+ err = copy_reference_state(dst_state, src) ?:
+ bpf_equations_copy(env, &dst_state->equations, &src->equations);
if (err)
return err;
bpf_add_cost(env, bpf_xreg_cnt(src) + src->acquired_refs, BPF_COST_COPY);
@@ -24853,6 +24856,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (env->log.level & BPF_LOG_LEVEL2)
log_program(env);
+ ret = bpf_equation_env_init(env);
+ if (ret < 0)
+ goto skip_full_check;
+
ret = bpf_flow_prepare(env);
if (ret < 0)
goto skip_full_check;
@@ -24994,6 +25001,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
kvfree(env->used_index.slots);
kvfree(env->kfunc_index.slots);
kvfree(env->insn_array_maps);
+ bpf_equation_env_free(env);
bpf_flow_free(env);
bpf_stack_liveness_free(env);
kvfree(env->cfg.postorder_nums);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state
2026-11-20 7:41 ` [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state Alexei Starovoitov
@ 2026-11-20 8:17 ` sashiko-bot
2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:17 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found no new issues
in this patch, but noticed 1 related pre-existing issue it does not
introduce:
- [Critical] bpf: out-of-bounds heap read in print_verifier_state() [1]
--
commit 50d5a04552747afb8f6e6e42345fff3aae2918b7
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Keep a table of equations in each state
This commit adds a per-state equation table tracking xreg equations, symbol
equations, and symbol ranges across verifier state copies, logging, and
memory accounting.
> diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
> index 7b1390d905498..8a905f4df9b18 100644
> --- a/kernel/bpf/log.c
> +++ b/kernel/bpf/log.c
> @@ -803,6 +803,9 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
> verbose(env, "%s%d", i > first ? "," : "",
> state->loop_stack[i].loop->header);
> }
> + /* one table for all frames */
> + if (print_all && frameno == vstate->curframe)
> + bpf_equations_print(env, vstate);
> verbose(env, "\n");
> if (!print_all)
> mark_verifier_state_clean(env);
[Severity: Critical]
This problem wasn't introduced by this patch, but can print_verifier_state()
read past the end of state->stack [1]?
When a BPF program reserves a dynptr at the current maximum allocated stack
depth and then overwrites only its second slot,
check_stack_write_fixed_off() calls destroy_if_dynptr_stack_slot() and
scrub_special_slot() on the written slot, leaving the first slot (at the
last allocated index of state->stack) with slot_type STACK_DYNPTR. If a
verifier log dump then runs, the STACK_DYNPTR case in print_verifier_state()
advances i past bpf_stack_nr_slots(state) before indexing state->stack:
kernel/bpf/log.c:print_verifier_state() {
...
for (i = 0; i < bpf_stack_nr_slots(state); i++) {
...
case STACK_DYNPTR:
/* skip to main dynptr slot */
i += BPF_DYNPTR_NR_SLOTS - 1;
reg = &bpf_stack_slot(state, i)->spilled_ptr;
...
}
This triggers an out-of-bounds heap read that can crash the kernel or leak
adjacent heap memory into the verifier log.
[1] https://sashiko.dev/#/bug/linux-75762bbc-6239-4798-be8a-b4058144a0ca
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=17
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state
2026-11-20 7:41 ` [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state Alexei Starovoitov
2026-11-20 8:17 ` sashiko-bot
@ 2026-11-20 8:25 ` bot+bpf-ci
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2955 bytes --]
> bpf: Keep a table of equations in each state
TL;DR:
- Unused static helper equation_xreg_bounds() may fail the build with
CONFIG_WERROR=y, breaking bisection
- Comment on equation count per insn looks inaccurate (nit)
> diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
> --- a/kernel/bpf/equation.c
> +++ b/kernel/bpf/equation.c
[ ... ]
> +/* An insn deletes the equations that it made before. So there are at most as many as insns. */
> +static bool append_sym_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs,
> + struct bpf_sym_equation sym_eq)
> +{
> + if (env->no_mem)
> + return false;
> + if (sym_eq.insn_idx >= 0 && !env->eq_env->equation_added)
> + forget_sym_equations(env, eqs, sym_eq.insn_idx);
> + env->eq_env->equation_added = true;
This isn't a bug, but is the comment accurate about the bound?
forget_sym_equations() runs only on the first append of an insn visit, because
of the equation_added flag, so every later append in the same visit is kept.
Equations with insn_idx < 0 are never deleted this way.
With the later patch "bpf: Make equations from ALU, spill, fill and call
insns", one visit of w1 += w2 can append up to three equations:
bpf_equations_after_insn() -> alu_scalar() -> alu_linear() -> alu_add()
-> bpf_add_sym_equation() /* sum sym2 = sym0 + sym1 */
alu_add() -> expr_low_bits() -> add_divmod_equation()
-> bpf_add_sym_equation() /* rem = sym2 - 2^32 * quot */
alu_scalar() -> bpf_add_op() /* op rem = insn(sym0, sym1) */
The next visit of that insn deletes all three and adds new ones, so the bound
is a few per insn rather than one per insn. Later patches also set
insn_idx = -1 on some sums, and those are limited by SUMS_PER_XREG instead.
Could the comment say "a few per insn", or explain that the equations of one
visit replace those of the previous visit?
[ ... ]
> +/* A pointer that became a scalar is another number */
> +static struct scalar *equation_xreg_bounds(struct bpf_verifier_env *env,
> + const struct bpf_verifier_state *st,
> + const struct bpf_xreg_equation *xreg_eq)
> +{
Is equation_xreg_bounds() used anywhere in this patch? It is static and the
only occurrence of the name in equation.c is this definition, so gcc should
report:
kernel/bpf/equation.c: warning: 'equation_xreg_bounds' defined but not used
[-Wunused-function]
With CONFIG_WERROR=y this would fail the build at this commit and break
bisection. The first callers are refine_sym_from_xreg(), refine_xreg() and
refine_from_reg() in the next patch, 969a062e4b35 ("bpf: Refine all related
numbers after a jump").
Could the helper be added in that patch together with its first user?
[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 18/49] bpf: Refine all related numbers after a jump
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (16 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 17/49] bpf: Keep a table of equations in each state Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns Alexei Starovoitov
` (30 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A jump refines its operands. Take that through the equations:
xreg -> its sym -> the other syms -> their xregs
r1 = sym0, r2 = 4 * sym0
if r1 > 7 goto out r1 0..7 -> sym0 0..7 -> r2 0..28
propagate_bounds() keeps the refined syms in a queue, and links the
equations that use a sym. A sym propagates once, so the work is linear
in the size of the table. A sum refines all of its syms. An op refines
only its result: run_op() runs scalar_alu() on the new operands.
A jump that compares two numbers compares their difference with 0:
end = p + sym0
if (p != end) sym0 != 0
A 32-bit test for equal does that for the numbers before a truncation
too, see skip_truncation(): w1 += 1; if w1 != w2
Code with no 32-bit insns can test the carry of a sum, see
refine_carry():
r5 = r2; r5 <<= 32; r5 >>= 32
if r5 != r2 goto carry if the jump is taken, r2 is above U32_MAX
Packet pointers that share an id move together, see refine_ptr().
The two branches can get other bounds for one pointer. The test for the
end of the packet takes the operands of one branch for both. So
bpf_equations_jump() is last in check_cond_jmp_op().
If the equations cannot hold, the state ends after the next conditional
jump. The removal of dead code deletes an edge only from such a jump: a
call or a goto has to keep its target. The cpu can speculate past the
jump, so that needs bypass_spec_v1. See infeasible_state_ends().
Two pointers to one object have the order of their offsets:
for (p = buf; p < end; p++)
*p = 0;
The jump refines both offsets, and is_branch_taken() decides it. This
holds if no address wraps, so both offsets have to be near the object.
It is off without bypass_spec_v1.
This replaces refine_linked_regs().
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 12 +
kernel/bpf/equation.c | 440 +++++++++++++++++++++++++++++++++++
kernel/bpf/equation.h | 6 +
kernel/bpf/verifier.c | 49 +++-
4 files changed, 503 insertions(+), 4 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 8d1a1ff87b7..297f19271ff 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -2321,10 +2321,22 @@ void bpf_equation_env_free(struct bpf_verifier_env *env);
int bpf_equations_copy(struct bpf_verifier_env *env, struct bpf_equations *dst,
const struct bpf_equations *src);
void bpf_equations_free(struct bpf_equations *eqs);
+void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
+ bool taken);
void bpf_equations_print(struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
u32 bpf_equations_bytes(const struct bpf_equations *eqs);
void scalar_bounds_sync(struct scalar *s);
bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
+bool scalars_fit_in_bits(const struct scalar *a, const struct scalar *b, u32 bits);
+void scalars_refine_cond_op(struct scalar *a, struct scalar *b, u8 opcode, bool is_jmp32);
+
+static inline u8 bpf_signed_opcode(u8 op)
+{
+ if (op == BPF_JLT || op == BPF_JLE)
+ return op + BPF_JSLT - BPF_JLT;
+ return op == BPF_JGT || op == BPF_JGE ? op + BPF_JSGT - BPF_JGT : op;
+}
/* How many bytes are checked from the start of the packet, wherever the pointer is */
static inline int pkt_min_len(const struct bpf_reg_state *reg)
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index a85fef30e8e..16b50253a6e 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -599,6 +599,446 @@ void bpf_eval_diff(struct bpf_verifier_env *env, const struct bpf_verifier_state
scalar_bounds_sync(bounds);
}
+/* Jumps refine */
+
+/*
+ * Packet pointers that share an id move together, see find_good_pkt_pointers().
+ * Do not reduce them one by one: the known bits cut one range and leave the other.
+ * p = 20..40, bit 4 is set q = p - 16 = 4..24, not 4..12
+ */
+static bool refine_ptr(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ struct scalar *s, const struct scalar *bounds)
+{
+ struct bpf_reg_state *ptr = container_of(&s->var_off, struct bpf_reg_state, var_off), *reg;
+ struct scalar new = *s, shift;
+ struct cnum64 old = s->r64;
+ struct bpf_func_state *f;
+ struct tnum known;
+ s64 smax;
+
+ /* a constant offset has to fit in an int */
+ if (!refine_bounds(&st->equations, &new, bounds) ||
+ cnum64_smin(new.r64) <= -BPF_MAX_VAR_OFF || cnum64_smax(new.r64) >= BPF_MAX_VAR_OFF)
+ return false;
+ if (!type_is_pkt_pointer(ptr->type)) {
+ *s = new;
+ return true;
+ }
+ bpf_add_cost(env, bpf_xreg_cnt(st), BPF_COST_SCAN);
+ bpf_for_each_reg_in_vstate(st, f, reg, ({
+ if (reg->type != ptr->type || reg->id != ptr->id)
+ continue;
+ smax = reg_smax(reg);
+ scalar_set_const(&shift, reg->r64.base - old.base);
+ known = reg->var_off;
+ scalar_add(scalar_of(reg), &shift, &new);
+ known = tnum_intersect(known, reg->var_off);
+ if (!tnum_subreg_is_const(known))
+ reg->var_off = known;
+ bpf_trim_pkt_range(reg, smax);
+ }));
+ return true;
+}
+
+static void mark_sym_refined(struct bpf_verifier_env *env, u32 sym, u32 sweep)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+
+ if (eq_env->scratch[sym].refined_sweep)
+ return;
+ eq_env->scratch[sym].refined_sweep = sweep;
+ if (sweep > 1)
+ eq_env->scratch[eq_env->refined_cnt++].queued_sym = sym;
+}
+
+/* @expr is in @bounds. If its sym got smaller, @sweep has to look at the equations of the sym. */
+static void refine_sym(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ struct expr expr, struct scalar bounds, u32 sweep)
+{
+ scalar_add_const(&bounds, -expr.off);
+ if (refine_from_product(eqs, &eqs->syms[var_symno(expr.var)], expr.coef, &bounds))
+ mark_sym_refined(env, var_symno(expr.var), sweep);
+}
+
+/* eval_expr_nosync(), and what the jump tells about a multiple of its operand */
+static void expr_bounds_after_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct expr *expr, struct scalar *bounds)
+{
+ const struct expr *jump_expr = &env->eq_env->jump_expr;
+ struct scalar tmp;
+ s64 coef;
+
+ eval_expr_nosync(env, st, expr, bounds);
+ if (!jump_expr->coef || !expr->coef || expr->var != jump_expr->var)
+ return;
+ coef = div64_s64(expr->coef, jump_expr->coef);
+ if (coef * jump_expr->coef != expr->coef)
+ return;
+ scalar_scale(&tmp, &env->eq_env->jump_bounds, coef);
+ scalar_add_const(&tmp, expr->off - coef * jump_expr->off);
+ refine_bounds(&st->equations, bounds, &tmp);
+}
+
+static bool sym_is_refined(struct bpf_verifier_env *env, u32 sym, u32 sweep)
+{
+ return sym != NO_SYM && (!sweep || env->eq_env->scratch[sym].refined_sweep == sweep);
+}
+
+/* What @op makes of its operands. False: they have no number. */
+static bool run_op(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct bpf_sym_equation *op, struct scalar *res)
+{
+ struct expr expr[2] = { rhs_expr(op, 0), rhs_expr(op, 1) };
+ struct scalar src;
+
+ bpf_add_cost(env, 1, BPF_COST_INSN);
+ expr_bounds_after_jump(env, st, &expr[0], res);
+ expr_bounds_after_jump(env, st, &expr[1], &src);
+ if (!scalar_intersect(res, res, 64) || !scalar_intersect(&src, &src, 64))
+ return false;
+ scalar_alu(res, &src, &env->prog->insnsi[op->insn_idx]);
+ return true;
+}
+
+static void refine_syms_by_equation(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ struct bpf_sym_equation *sym_eq, u32 sweep)
+{
+ struct expr expr[2] = { rhs_expr(sym_eq, 0), rhs_expr(sym_eq, 1) };
+ struct bpf_equations *eqs = &st->equations;
+ struct scalar operand[2], bounds, *lhs = &eqs->syms[sym_eq->sym[0]];
+ bool refined[3];
+ int i;
+
+ for (i = 0; i < 3; i++)
+ refined[i] = sym_is_refined(env, sym_eq->sym[i], sweep);
+ if ((!refined[0] || is_op(sym_eq)) && !refined[1] && !refined[2])
+ return;
+ if (is_op(sym_eq)) {
+ if (run_op(env, st, sym_eq, &bounds) && refine_bounds(eqs, lhs, &bounds))
+ mark_sym_refined(env, sym_eq->sym[0], sweep + 1);
+ return;
+ }
+ bpf_add_cost(env, 1, BPF_COST_INSN);
+ expr_bounds_after_jump(env, st, &expr[0], &operand[0]);
+ expr_bounds_after_jump(env, st, &expr[1], &operand[1]);
+ for (i = 1; i >= 0; i--) {
+ if (!expr[i].coef || !(refined[0] || refined[2 - i]))
+ continue;
+ scalar_add_scaled(&bounds, lhs, &operand[!i], -1);
+ if (!refine_bounds(eqs, &operand[i], &bounds))
+ continue;
+ refined[i + 1] = true;
+ refine_sym(env, eqs, expr[i], operand[i], sweep + 1);
+ }
+ scalar_add_scaled(&bounds, &operand[0], &operand[1], 1);
+ if (tnum_is_const(operand[1].var_off)) {
+ sym_eq->off += operand[1].var_off.value;
+ sym_eq->coef[2] = 0;
+ sym_eq->sym[2] = NO_SYM;
+ }
+ if ((refined[1] || refined[2]) && refine_bounds(eqs, lhs, &bounds))
+ mark_sym_refined(env, sym_eq->sym[0], sweep + 1);
+}
+
+static void refine_sym_from_xreg(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct bpf_xreg_equation *xreg_eq)
+{
+ struct expr expr = sym_expr(xreg_eq->coef, xreg_eq->sym, xreg_eq->off);
+ const struct scalar *s = equation_xreg_bounds(env, st, xreg_eq);
+ struct scalar bounds;
+
+ if (!s)
+ return;
+ bpf_add_cost(env, 2, BPF_COST_XREG);
+ eval_expr_nosync(env, st, &expr, &bounds);
+ if (refine_bounds(&st->equations, &bounds, s))
+ refine_sym(env, &st->equations, expr, bounds, 1);
+}
+
+/* The number in the xreg of @xreg_eq is in @bounds */
+static void refine_xreg(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct bpf_xreg_equation *xreg_eq, const struct scalar *bounds)
+{
+ struct scalar *s = equation_xreg_bounds(env, st, xreg_eq);
+ u32 xregno = xreg_eq->xreg;
+
+ bpf_add_cost(env, 4, BPF_COST_XREG);
+ if (!s || over_limit(env) ||
+ !(xreg_eq->is_ptr ? refine_ptr(env, st, s, bounds) :
+ refine_bounds(&st->equations, s, bounds)) ||
+ xreg_eq->xreg >> 32 != st->curframe)
+ return;
+ if (xregno < BPF_REG_FP)
+ mark_reg_scratched(env, xregno);
+ else if (xregno < BPF_XREG_TRIP)
+ mark_stack_slot_scratched(env, bpf_xreg_to_spi(xregno));
+}
+
+/* Link the equations that use a sym. A link is 3 * equation + position of the sym + 1. */
+static bool link_sym_equations(struct bpf_verifier_env *env, const struct bpf_equations *eqs)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ u32 i = 3 * eqs->sym_equation_cnt, sym;
+
+ if (!bpf_grow_array(env, &eq_env->use_links, &eq_env->use_link_cap, i,
+ sizeof(*eq_env->use_links)))
+ return false;
+ bpf_add_cost(env, eqs->sym_equation_cnt, BPF_COST_COPY);
+ for (sym = 0; sym < eqs->sym_cnt; sym++)
+ eq_env->scratch[sym].first_use = 0;
+ while (i--) {
+ sym = eqs->sym_equations[i / 3].sym[i % 3];
+ if (sym == NO_SYM)
+ continue;
+ eq_env->use_links[i] = eq_env->scratch[sym].first_use;
+ eq_env->scratch[sym].first_use = i + 1;
+ }
+ return true;
+}
+
+/* Each sweep goes one equation further. A sym propagates once. @all: to all xregs. */
+static void propagate_bounds(struct bpf_verifier_env *env, struct bpf_verifier_state *st, bool all)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct sym_scratch *scratch = eq_env->scratch, *refined;
+ struct bpf_equations *eqs = &st->equations;
+ struct bpf_xreg_equation *xreg_eq;
+ struct bpf_sym_equation *sym_eq;
+ struct scalar bounds;
+ struct expr expr;
+ u32 i, link;
+
+ bpf_add_cost(env, eqs->sym_cnt, BPF_COST_SCAN);
+ for (i = eq_env->refined_cnt = 0; i < eqs->sym_cnt; i++)
+ if (scratch[i].refined_sweep)
+ scratch[eq_env->refined_cnt++].queued_sym = i;
+ if (!eq_env->refined_cnt && !all)
+ return;
+ for (i = 0;
+ i < eq_env->refined_cnt && (i || link_sym_equations(env, eqs)) &&
+ !over_limit(env); i++) {
+ refined = &scratch[scratch[i].queued_sym];
+ for (link = refined->first_use; link--; link = eq_env->use_links[link]) {
+ sym_eq = &eqs->sym_equations[link / 3];
+ /* once in a sweep */
+ if ((link % 3 < 1 ||
+ !sym_is_refined(env, sym_eq->sym[0], refined->refined_sweep)) &&
+ (link % 3 < 2 ||
+ !sym_is_refined(env, sym_eq->sym[1], refined->refined_sweep)))
+ refine_syms_by_equation(env, st, sym_eq, refined->refined_sweep);
+ }
+ }
+ for_each_xreg_equation(env, eqs, xreg_eq) {
+ if (!all && !scratch[xreg_eq->sym].refined_sweep)
+ continue;
+ expr = sym_expr(xreg_eq->coef, xreg_eq->sym, xreg_eq->off);
+ expr_bounds_after_jump(env, st, &expr, &bounds);
+ refine_xreg(env, st, xreg_eq, &bounds);
+ }
+ for (i = 0; i < eqs->sym_cnt; i++)
+ scratch[i].refined_sweep = 0;
+}
+
+static void refine_from_reg(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 regno)
+{
+ const struct bpf_xreg_equation *xreg_eq =
+ bpf_find_xreg_equation(&st->equations, bpf_reg_to_xreg(st, regno));
+ struct bpf_reg_state *reg = &st->frame[st->curframe]->regs[regno];
+ struct scalar bounds = *scalar_of(reg);
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct expr expr;
+ u64 span;
+
+ if (reg->type == SCALAR_VALUE && scalar_align_bounds(&bounds))
+ *scalar_of(reg) = bounds;
+ else if (reg->type != SCALAR_VALUE && ptr_off_is_tracked(env, reg) &&
+ !type_is_pkt_pointer(reg->type) && scalar_align_bounds(&bounds))
+ refine_ptr(env, st, scalar_of(reg), &bounds);
+ if (!xreg_eq)
+ return;
+ refine_sym_from_xreg(env, st, xreg_eq);
+ if (is_plus_minus_one(xreg_eq->coef) || xreg_eq->is_ptr ||
+ !equation_xreg_bounds(env, st, xreg_eq))
+ return;
+ /* the sym cannot show it if the product wraps */
+ expr = sym_expr(xreg_eq->coef, xreg_eq->sym, xreg_eq->off);
+ bpf_eval_expr(env, st, &expr, &bounds);
+ if (scalar_subsumes(scalar_of(reg), &bounds) &&
+ !check_mul_overflow(abs64(expr.coef),
+ st->equations.syms[xreg_eq->sym].r64.size, &span))
+ return;
+ eq_env->jump_expr = expr;
+ eq_env->jump_bounds = *scalar_of(reg);
+ mark_sym_refined(env, xreg_eq->sym, 1);
+ propagate_bounds(env, st, false);
+ eq_env->jump_expr.coef = 0;
+}
+
+/* Apply all equations */
+void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
+{
+ struct bpf_xreg_equation *xreg_eq;
+ struct bpf_sym_equation *sym_eq;
+
+ st->equations.needs_refine = false;
+ for_each_xreg_equation(env, &st->equations, xreg_eq)
+ refine_sym_from_xreg(env, st, xreg_eq);
+ for_each_sym_equation(env, &st->equations, sym_eq)
+ refine_syms_by_equation(env, st, sym_eq, 0);
+ propagate_bounds(env, st, true);
+}
+
+static bool sym_has_xreg(struct bpf_verifier_env *env, const struct bpf_equations *eqs, u32 sym)
+{
+ const struct bpf_xreg_equation *xreg_eq;
+
+ if (over_limit(env))
+ return true;
+ for_each_xreg_equation(env, eqs, xreg_eq)
+ if (xreg_eq->sym == sym)
+ return true;
+ return false;
+}
+
+/*
+ * A truncation is a sum 'low bits = x - 2^n * high bits', see expr_low_bits(). Go back to x if
+ * only the low @bits bits matter, and no xreg holds the low bits. @all: also if an xreg does.
+ * w2 = w1; w2 += -1; w2 &= 255; the low 8 bits of r1 - 1
+ */
+static void skip_truncation(struct bpf_verifier_env *env, const struct bpf_equations *eqs,
+ struct expr *expr, u32 bits, bool all)
+{
+ const struct bpf_sym_equation *sym_eq = eqs->sym_equations + eqs->sym_equation_cnt;
+
+ if (over_limit(env))
+ return;
+ bpf_add_cost(env, eqs->sym_equation_cnt, BPF_COST_COPY);
+ while (sym_eq-- > eqs->sym_equations)
+ if (var_is_sym(expr->var) && sym_eq->sym[0] == var_symno(expr->var) &&
+ !is_op(sym_eq) && !((expr->coef * sym_eq->coef[2]) & low_bits(bits)) &&
+ (all || !sym_has_xreg(env, eqs, sym_eq->sym[0])))
+ *expr = expr_scale(rhs_expr(sym_eq, 0), expr->coef, expr->off);
+}
+
+/* "@a - @b @op 0" holds */
+static void refine_diff(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct expr *a, const struct expr *b, u8 op, bool jmp32)
+{
+ struct equation_pos pos = {};
+ struct scalar bounds, zero;
+ struct expr diff;
+
+ while (bpf_find_next_diff(env, &st->equations, a, b, &pos, &diff)) {
+ bpf_eval_expr(env, st, &diff, &bounds);
+ scalar_set_const(&zero, 0);
+ scalars_refine_cond_op(&bounds, &zero, op, jmp32);
+ if (!scalar_intersect(&bounds, &bounds, 64)) {
+ st->equations.infeasible = true;
+ return;
+ }
+ if (!var_is_sym(diff.var))
+ continue;
+ refine_sym(env, &st->equations, diff, bounds, 1);
+ propagate_bounds(env, st, false);
+ }
+}
+
+/* "@x != @low" holds. If @low is (u32)@x, then @x is above U32_MAX. */
+static void refine_carry(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ struct expr x, struct expr low, const struct bpf_reg_state *low_reg)
+{
+ struct expr untruncated = x;
+ struct scalar bounds;
+
+ if (reg_umax(low_reg) > U32_MAX || !var_is_sym(x.var))
+ return;
+ skip_truncation(env, &st->equations, &low, 32, true);
+ skip_truncation(env, &st->equations, &untruncated, 32, true);
+ if (low.var != untruncated.var || (u32)(low.coef - untruncated.coef) ||
+ (u32)(low.off - untruncated.off))
+ return;
+ scalar_set_unknown(&bounds);
+ bounds.r64 = cnum64_from_urange(BIT_ULL(32), U64_MAX);
+ refine_sym(env, &st->equations, x, bounds, 1);
+ propagate_bounds(env, st, false);
+}
+
+/* The jump compares two numbers, or the offsets of two pointers into one object */
+static bool compares_numbers(const struct bpf_verifier_state *st, const struct bpf_insn *insn)
+{
+ const struct bpf_reg_state *dst = &st->frame[st->curframe]->regs[insn->dst_reg];
+ const struct bpf_reg_state *src = &st->frame[st->curframe]->regs[insn->src_reg];
+
+ if (dst->type == SCALAR_VALUE)
+ return src->type == SCALAR_VALUE;
+ return BPF_CLASS(insn->code) == BPF_JMP && bpf_same_memory_origin(dst, src);
+}
+
+/*
+ * @st went through the jump at @idx, and its operands are refined. A jump that compares two
+ * numbers compares their difference with 0:
+ * end = p + sym0; ... if (p != end) sym0 != 0
+ */
+void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
+ bool taken)
+{
+ const struct bpf_insn *insn = &env->prog->insnsi[idx];
+ const struct bpf_reg_state *dst = &st->frame[st->curframe]->regs[insn->dst_reg];
+ const struct bpf_reg_state *src = &st->frame[st->curframe]->regs[insn->src_reg];
+ u8 op = taken ? BPF_OP(insn->code) : bpf_rev_opcode(BPF_OP(insn->code));
+ bool jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
+ bool src_is_reg = BPF_SRC(insn->code) == BPF_X, is_ptr;
+ s64 max = jmp32 ? S32_MAX : S64_MAX / 2, min = -max - 1;
+ struct expr a, b, untruncated[2];
+ struct cnum64 lim;
+
+ if (!env->eq_env)
+ return;
+ refine_from_reg(env, st, insn->dst_reg);
+ if (src_is_reg)
+ refine_from_reg(env, st, insn->src_reg);
+ if (!st->equations.xreg_equation_cnt)
+ return;
+ propagate_bounds(env, st, false);
+ if (!src_is_reg || BPF_OP(insn->code) == BPF_JSET || !compares_numbers(st, insn) ||
+ !bpf_xreg_expr(env, st, bpf_reg_to_xreg(st, insn->dst_reg), &a, &is_ptr) ||
+ !bpf_xreg_expr(env, st, bpf_reg_to_xreg(st, insn->src_reg), &b, &is_ptr))
+ return;
+ if (op == BPF_JEQ || op == BPF_JNE) {
+ jmp32 |= scalars_fit_in_bits(scalar_of(dst), scalar_of(src), 32);
+ } else {
+ /* the difference does not wrap. An object is not in the first page. */
+ if (is_ptr)
+ lim = cnum64_from_srange(-SZ_4K, BPF_MAX_VAR_OFF);
+ else if (op != bpf_signed_opcode(op))
+ lim = cnum64_from_srange(0, max * 2 + 1);
+ else
+ lim = cnum64_from_srange(min, max);
+ if (!cnum64_is_subset(lim, dst->r64) || !cnum64_is_subset(lim, src->r64))
+ return;
+ op = bpf_signed_opcode(op);
+ jmp32 = false;
+ }
+ refine_diff(env, st, &a, &b, op, jmp32);
+ if (op == BPF_JNE && !jmp32) {
+ refine_carry(env, st, a, b, src);
+ refine_carry(env, st, b, a, dst);
+ }
+ if (!jmp32)
+ return;
+ /* w1 += 1; if w1 != w2: the equation is for the number before the truncation */
+ untruncated[0] = a;
+ untruncated[1] = b;
+ skip_truncation(env, &st->equations, &untruncated[0], 32, true);
+ skip_truncation(env, &st->equations, &untruncated[1], 32, true);
+ if (!expr_equal(&untruncated[0], &a))
+ refine_diff(env, st, &untruncated[0], &b, op, true);
+ if (!expr_equal(&untruncated[1], &b))
+ refine_diff(env, st, &a, &untruncated[1], op, true);
+ if (!expr_equal(&untruncated[0], &a) && !expr_equal(&untruncated[1], &b))
+ refine_diff(env, st, &untruncated[0], &untruncated[1], op, true);
+}
+
static void print_operand(struct bpf_verifier_log *log, const char *prefix, struct expr expr)
{
if (expr.coef)
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index 26c100e715d..1cb2f358199 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -46,6 +46,9 @@ struct sym_scratch {
u32 use_cnt; /* equations that use it */
/* propagate_bounds(): the sweep that has to look at its equations */
u32 refined_sweep;
+ /* propagate_bounds(): not of this sym. The queue of refined syms. */
+ u32 queued_sym;
+ u32 first_use; /* see link_sym_equations() */
u32 new_symno;
bool determined, by_op;
const struct bpf_sym_equation *op; /* that makes it */
@@ -57,6 +60,9 @@ struct bpf_equation_env {
u32 *use_links;
u32 scratch_cap, use_link_cap, refined_cnt;
bool has_expr[2], is_ptr[2], equation_added, divided;
+ /* the operand of a jump and its bounds, see refine_from_reg() */
+ struct expr jump_expr;
+ struct scalar jump_bounds;
/* equation_join.c */
struct bpf_verifier_state join_state; /* it has equations and no frames */
struct join_phi *phis;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 199108c2226..033f6b42ab1 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -17468,6 +17468,7 @@ static bool is_safe_to_compute_dst_reg_range(const struct bpf_insn *insn,
}
switch (BPF_OP(insn->code)) {
+ case BPF_MOV:
case BPF_ADD:
case BPF_SUB:
case BPF_NEG:
@@ -17534,7 +17535,7 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins
}
/*
- * dst = dst <op> src: the transfer function of the ALU insn @insn, all but BPF_MOV.
+ * dst = dst <op> src: the transfer function of the ALU insn @insn.
* @src can't be @dst. BPF_NEG and BPF_END ignore its value: any valid scalar will do.
*
* WARNING: This function does calculations on 64-bit values, but the actual
@@ -17574,6 +17575,13 @@ void scalar_alu(struct scalar *dst, const struct scalar *src, const struct bpf_i
* tnum to calculate an approximation of the sign/unsigned bounds.
*/
switch (opcode) {
+ case BPF_MOV:
+ *dst = *src;
+ if (off && alu32)
+ coerce_subreg_to_size_sx(dst, off >> 3);
+ else if (off)
+ coerce_scalar_to_size_sx(dst, off >> 3);
+ break;
case BPF_ADD:
scalar_add(dst, dst, src);
break;
@@ -18322,7 +18330,6 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
}));
}
-static void scalars_refine_cond_op(struct scalar *a, struct scalar *b, u8 opcode, bool is_jmp32);
/*
* Learn more information about live branches by simulating refinement on both branches.
* scalars_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means
@@ -18593,6 +18600,18 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r
if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32)
return is_pkt_ptr_branch_taken(reg1, reg2, opcode);
+ /* Two pointers to one object have the order of their offsets */
+ if (reg1->type != SCALAR_VALUE && !is_jmp32 && opcode != BPF_JSET && env->bypass_spec_v1 &&
+ !(reg1->type & PTR_UNTRUSTED) && bpf_same_memory_origin(reg1, reg2)) {
+ /* an object can be at a low address, but not in the first page */
+ struct cnum64 lim = cnum64_from_srange(opcode == BPF_JEQ || opcode == BPF_JNE ?
+ -BPF_MAX_VAR_OFF : -SZ_4K, BPF_MAX_VAR_OFF);
+
+ if (!cnum64_is_subset(lim, reg1->r64) || !cnum64_is_subset(lim, reg2->r64))
+ return -1;
+ return is_scalar_branch_taken(env, reg1, reg2, bpf_signed_opcode(opcode), false);
+ }
+
if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
u64 val;
@@ -18662,7 +18681,7 @@ u8 bpf_rev_opcode(u8 opcode)
}
/* Refine range knowledge for <a> <op> <b> conditional operation. */
-static void scalars_refine_cond_op(struct scalar *a, struct scalar *b, u8 opcode, bool is_jmp32)
+void scalars_refine_cond_op(struct scalar *a, struct scalar *b, u8 opcode, bool is_jmp32)
{
struct tnum t;
u64 val;
@@ -18976,7 +18995,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
}
/* a == b (mod 2^N) implies a == b if both fit in uN or both fit in sN. N is @bits. */
-static bool scalars_fit_in_bits(const struct scalar *a, const struct scalar *b, u32 bits)
+bool scalars_fit_in_bits(const struct scalar *a, const struct scalar *b, u32 bits)
{
u64 umax = GENMASK_ULL(bits - 1, 0);
s64 smax = umax >> 1;
@@ -19223,6 +19242,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
err = refine_live_branch(env, insn, &env->true_reg1, &env->true_reg2);
if (err)
return err;
+ bpf_equations_jump(env, this_branch, *insn_idx, true);
if (env->log.level & BPF_LOG_LEVEL)
print_insn_state(env, this_branch, this_branch->curframe);
*insn_idx += insn->off;
@@ -19241,6 +19261,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
err = refine_live_branch(env, insn, &env->false_reg1, &env->false_reg2);
if (err)
return err;
+ bpf_equations_jump(env, this_branch, *insn_idx, false);
if (env->log.level & BPF_LOG_LEVEL)
print_insn_state(env, this_branch, this_branch->curframe);
return 0;
@@ -19331,6 +19352,9 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
insn->dst_reg);
return -EACCES;
}
+ /* last: the code above takes the operands of one branch for both */
+ bpf_equations_jump(env, this_branch, *insn_idx, false);
+ bpf_equations_jump(env, other_branch, *insn_idx, true);
if (env->log.level & BPF_LOG_LEVEL)
print_insn_state(env, this_branch, this_branch->curframe);
return 0;
@@ -20842,6 +20866,18 @@ static int check_held_in_leftover(struct bpf_verifier_env *env,
return held_cnt(st) > st->max_held_cnt;
}
+/*
+ * The cpu can speculate past the jump that the equations decide.
+ * The removal of dead code can delete an edge only from a conditional jump.
+ */
+static bool infeasible_state_ends(struct bpf_verifier_env *env)
+{
+ int from = env->cur_state->last_insn_idx;
+
+ return env->cur_state->equations.infeasible && env->bypass_spec_v1 && from >= 0 &&
+ bpf_insn_is_cond_jump(env->prog->insnsi[from].code);
+}
+
/* The current state came back to the header of the loop of @entry. Returns 1 if it ends here. */
static int next_trip(struct bpf_verifier_env *env, struct loop_stack_entry *entry,
bool *starts_leftover)
@@ -20924,11 +20960,16 @@ int bpf_flow_loop_enter(struct bpf_verifier_env *env, struct bpf_loop **backedge
bool pushed;
int err;
+ if (infeasible_state_ends(env))
+ return 1;
+
/* The callback came back to the helper that calls it, in the same trip. */
if (frame->callback_depth)
return enter_callback(env, true, backedge_loop);
loop_stack_pop(env);
+ if (infeasible_state_ends(env))
+ return 1;
err = loop_stack_push(env, &pushed);
if (err)
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (17 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 18/49] bpf: Refine all related numbers after a jump Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 8:26 ` sashiko-bot
2026-11-20 7:41 ` [PATCH bpf-next 20/49] bpf: Delete the equations that say nothing Alexei Starovoitov
` (29 subsequent siblings)
48 siblings, 2 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
An insn deletes the equations of the registers that it writes, see
def_regs. The result gets a new equation:
r2 = r1 r1 = sym0, r2 = sym0
r2 += 8 r2 = sym0 + 8
r2 <<= 2 r2 = 4 * sym0 + 32
r2 += r3 r2 = sym2, sum: sym2 = 4 * sym0 + sym1 + 32
r1 >>= 3 r1 = quot, sum: rem = sym0 - 8 * quot
w4 = w1 r4 = low, sum: low = sym0 - 2^32 * high
r1 ^= 5 r1 = sym3, op: sym3 = sym0 ^ 5
*(fp - 8) = r1 fp-8 = sym0, and a fill reads it back
call r1-r5 go to the frame of the callee
exit r0 goes to the frame of the caller
A number gets a sym only when a second xreg needs it.
A truncation is a sum for the low bits. Compilers test one truncation
and use another one:
w2 = w1; w2 += -1; w2 &= 255; if w2 > 99 goto out; use (u8)r1
expr_low_bits() looks for a number with the same low bits before it
makes a new sym. 'w1 <<= 24; w1 s>>= 24' is a truncation to 8 bits.
An op that reads only the low bits gets them from a truncation, also if
the test of the truncation was earlier:
w2 = w1; w2 &= 0xff; if w2 > 0xdf goto out; w1 &= 0xf0 0..0xd0
find_same_op() finds an op with the same operands. So two insns that
compute one number get one sym.
A call uses the equations of its arguments before it deletes them. The
largest offset and the largest size do not have to go together:
read(buf + off, 64 - off) r1 = buf + sym0, r2 = 64 - sym0
size_at_max_offset() gives the check the size that is left at the
largest offset. It can be 0 for a call that touches memory. So
check_stack_access_within_bounds() tests the range, not the size.
A speculative state gets no equations.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 9 +
kernel/bpf/equation.c | 535 +++++++++++++++++++++++++++++++++++
kernel/bpf/equation.h | 9 +
kernel/bpf/verifier.c | 44 ++-
4 files changed, 592 insertions(+), 5 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 297f19271ff..1a9adf126f9 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -2321,6 +2321,15 @@ void bpf_equation_env_free(struct bpf_verifier_env *env);
int bpf_equations_copy(struct bpf_verifier_env *env, struct bpf_equations *dst,
const struct bpf_equations *src);
void bpf_equations_free(struct bpf_equations *eqs);
+s64 bpf_equations_max_sum(struct bpf_verifier_env *env, int regno1, int regno2);
+void bpf_equations_before_insn(struct bpf_verifier_env *env);
+int bpf_equations_after_insn(struct bpf_verifier_env *env);
+void bpf_equations_forget_xreg(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ u64 xreg);
+void bpf_equations_spill(struct bpf_verifier_env *env, struct bpf_func_state *frame, int spi,
+ int regno);
+void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *frame, int spi,
+ int size);
void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
bool taken);
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index 16b50253a6e..5c9a3df8ce4 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -1039,6 +1039,541 @@ void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state
refine_diff(env, st, &untruncated[0], &untruncated[1], op, true);
}
+/* The largest sum of two arguments of a call: numbers, or offsets of pointers. Modulo 2^64. */
+s64 bpf_equations_max_sum(struct bpf_verifier_env *env, int regno1, int regno2)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct bpf_verifier_state *st = env->cur_state;
+ struct equation_pos pos = {};
+ struct expr neg, sum;
+ struct scalar bounds;
+ s64 max = S64_MAX;
+
+ if (!eq_env || regno1 < 0 || regno2 < 0 ||
+ (~eq_env->has_arg_expr & (BIT(regno1) | BIT(regno2))))
+ return max;
+ neg = expr_scale(eq_env->arg_expr[regno2], -1, 0);
+ while (bpf_find_next_diff(env, &st->equations, &eq_env->arg_expr[regno1], &neg, &pos,
+ &sum)) {
+ bpf_eval_expr(env, st, &sum, &bounds);
+ max = min(max, cnum64_smax(bounds.r64));
+ }
+ return max;
+}
+
+/* Insns write */
+
+void bpf_equations_forget_xreg(struct bpf_verifier_env *env,
+ struct bpf_verifier_state *st, u64 xreg)
+{
+ bpf_set_xreg_equation(env, &st->equations, xreg, NULL, false);
+}
+
+/* @to gets the number of @from */
+static void copy_xreg_equation(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ u64 to, u64 from)
+{
+ const struct bpf_xreg_equation *xreg_eq = bpf_find_xreg_equation(eqs, from);
+ struct expr expr;
+
+ if (!xreg_eq)
+ return bpf_set_xreg_equation(env, eqs, to, NULL, false);
+ expr = sym_expr(xreg_eq->coef, xreg_eq->sym, xreg_eq->off);
+ bpf_set_xreg_equation(env, eqs, to, &expr, xreg_eq->is_ptr);
+}
+
+/* The number in @xreg is copied to another xreg. It needs a sym if it is @live_after. */
+static bool assign_sym(struct bpf_verifier_env *env, u64 xreg, bool live_after, struct expr *expr,
+ bool *is_ptr)
+{
+ struct bpf_verifier_state *st = env->cur_state;
+
+ if (!bpf_xreg_expr(env, st, xreg, expr, is_ptr))
+ return false;
+ if (expr->var != xreg)
+ return true;
+ if (!live_after)
+ return false;
+ expr->var = bpf_new_sym(env, &st->equations, bpf_find_var_bounds(env, st, xreg, is_ptr));
+ bpf_set_xreg_equation(env, &st->equations, xreg, expr, *is_ptr);
+ return var_is_sym(expr->var);
+}
+
+static bool reg_is_live_after(struct bpf_verifier_env *env, u32 regno)
+{
+ return env->insn_aux_data[env->insn_idx + 1].live_regs_before & BIT(regno);
+}
+
+/* Register @regno is spilled */
+void bpf_equations_spill(struct bpf_verifier_env *env, struct bpf_func_state *frame, int spi,
+ int regno)
+{
+ struct bpf_verifier_state *st = env->cur_state;
+ u64 slot = bpf_xreg(frame->frameno, bpf_spi_to_xreg(spi));
+ struct expr expr;
+ bool is_ptr;
+
+ if (env->eq_env && regno >= 0 && !st->speculative &&
+ assign_sym(env, bpf_reg_to_xreg(st, regno), reg_is_live_after(env, regno),
+ &expr, &is_ptr))
+ bpf_set_xreg_equation(env, &st->equations, slot, &expr, is_ptr);
+ else
+ bpf_equations_forget_xreg(env, st, slot);
+}
+
+/* The low @size bytes of what is loaded are from the spilled register */
+void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *frame,
+ int spi, int size)
+{
+ if (!env->eq_env)
+ return;
+ env->eq_env->fill_xreg = bpf_xreg(frame->frameno, bpf_spi_to_xreg(spi));
+ env->eq_env->fill_bits = size * BITS_PER_BYTE;
+}
+
+/* Note the operands. Delete the equations of the registers that the insn writes. */
+void bpf_equations_before_insn(struct bpf_verifier_env *env)
+{
+ const struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];
+ u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
+ bool src_is_reg = BPF_SRC(insn->code) == BPF_X && op != BPF_END;
+ bool alu = class == BPF_ALU || class == BPF_ALU64;
+ struct bpf_verifier_state *st = env->cur_state;
+ struct bpf_equation_env *eq_env = env->eq_env;
+ u32 def = env->insn_aux_data[env->insn_idx].def_regs, fr = st->curframe, i, r;
+ struct bpf_equations *eqs = &st->equations;
+ bool is_ptr;
+
+ if (!eq_env)
+ return;
+ if (st->speculative || eqs->sym_cnt > 2 * eqs->clean_sym_cnt + MAX_BPF_REG)
+ bpf_equations_clean(env, st);
+ eq_env->insn_idx = env->insn_idx;
+ eq_env->fill_xreg = VAR_NONE;
+ eq_env->has_arg_expr = 0;
+ eq_env->equation_added = eq_env->divided = false;
+ for (i = 0; i < 2; i++) {
+ r = i ? insn->src_reg : insn->dst_reg;
+ eq_env->has_expr[i] = eq_env->is_ptr[i] = false;
+ if (!alu || st->speculative)
+ continue;
+ if (i && !src_is_reg) {
+ eq_env->operand_expr[1] = make_expr(0, 0, insn->imm);
+ scalar_set_const(&eq_env->operand_bounds[1], insn->imm);
+ eq_env->has_expr[1] = true;
+ } else if (r <= BPF_REG_FP && cur_regs(env)[r].type != NOT_INIT) {
+ eq_env->operand_bounds[i] = *scalar_of(&cur_regs(env)[r]);
+ eq_env->has_expr[i] =
+ assign_sym(env, bpf_reg_to_xreg(st, r), false,
+ &eq_env->operand_expr[i], &eq_env->is_ptr[i]);
+ }
+ }
+ if (class == BPF_JMP && op == BPF_EXIT && fr && !st->frame[fr]->in_callback_fn)
+ copy_xreg_equation(env, eqs, bpf_xreg(fr - 1, BPF_REG_0), bpf_xreg(fr, BPF_REG_0));
+ if (class == BPF_JMP && op == BPF_CALL) {
+ i = xreg_equation_pos(eqs, bpf_xreg(fr + 1, 0)) - eqs->xreg_equations;
+ eqs->normalized &= i == eqs->xreg_equation_cnt;
+ eqs->xreg_equation_cnt = i;
+ for (r = BPF_REG_1; r <= BPF_REG_5 && i; r++)
+ if (bpf_xreg_expr(env, st, bpf_xreg(fr, r), &eq_env->arg_expr[r], &is_ptr))
+ eq_env->has_arg_expr |= BIT(r);
+ }
+ for (r = 0; r < BPF_REG_FP && def >> r && eqs->xreg_equation_cnt; r++) {
+ if (!(def & BIT(r)))
+ continue;
+ /* the arguments are in the frame of the callee, if it gets one */
+ if (r && (bpf_pseudo_call(insn) || bpf_is_callx(insn)))
+ copy_xreg_equation(env, eqs, bpf_xreg(fr + 1, r), bpf_xreg(fr, r));
+ bpf_equations_forget_xreg(env, st, bpf_xreg(fr, r));
+ }
+}
+
+/* @expr = @divisor * quot + rem. Returns the expr of a new sym: @quot or @rem. */
+static bool add_divmod_equation(struct bpf_verifier_env *env, struct expr *expr, u64 divisor,
+ const struct scalar *quot, const struct scalar *rem, bool want_quot)
+{
+ struct bpf_equations *eqs = &env->cur_state->equations;
+ struct bpf_sym_equation sym_eq = { { -1, expr->coef, -divisor }, { 0, expr->var },
+ env->eq_env->insn_idx, expr->off };
+
+ sym_eq.sym[0] = bpf_new_sym(env, eqs, rem);
+ sym_eq.sym[2] = bpf_new_sym(env, eqs, quot);
+ *expr = sym_expr(1, sym_eq.sym[want_quot ? 2 : 0], 0);
+ env->eq_env->divided = true;
+ if (!bpf_add_sym_equation(env, eqs, sym_eq))
+ return false;
+ refine_syms_by_equation(env, env->cur_state,
+ &eqs->sym_equations[eqs->sym_equation_cnt - 1], 0);
+ for (i = 0; i < 3; i++)
+ if (sym_eq.coef[i])
+ env->eq_env->scratch[sym_eq.sym[i]].refined_sweep = 0;
+ return true;
+}
+
+/* Make @expr an expr of the left side of @sym_eq that has the same low @bits bits */
+static bool rewrite_expr_by_truncation(const struct bpf_sym_equation *sym_eq,
+ struct expr *expr, u32 bits)
+{
+ struct expr factor = make_expr(0, 0, expr->coef);
+
+ if (sym_eq->sym[1] != var_symno(expr->var) ||
+ !bpf_expr_div_exact(&factor, 0, sym_eq->coef[1]) ||
+ ((factor.off * sym_eq->coef[2]) & low_bits(bits)))
+ return false;
+ *expr = sym_expr(factor.off, sym_eq->sym[0], expr->off - factor.off * sym_eq->off);
+ return true;
+}
+
+static void set_operand(struct bpf_sym_equation *op, int i, const struct expr *expr)
+{
+ op->coef[i + 1] = expr->coef;
+ op->sym[i + 1] = expr->var;
+ *(i ? &op->off_src : &op->off) = expr->off;
+}
+
+/* How many low bits of its operands does the insn of @op read? */
+static u32 bits_read(struct bpf_verifier_env *env, const struct bpf_sym_equation *op)
+{
+ const struct bpf_insn *insn = &env->prog->insnsi[op->insn_idx];
+ u32 bits = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
+
+ if (BPF_OP(insn->code) == BPF_END)
+ return insn->imm;
+ if (BPF_OP(insn->code) == BPF_MOV)
+ return insn->off ?: bits;
+ if (BPF_OP(insn->code) == BPF_AND && !op->coef[2])
+ return min_t(u32, fls64(op->off_src), bits);
+ return bits;
+}
+
+static void op_skips_truncations(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ struct bpf_sym_equation *op)
+{
+ u32 i, bits = bits_read(env, op);
+ struct expr expr;
+
+ for (i = 0; i < 2 && bits < 64; i++) {
+ expr = rhs_expr(op, i);
+ if (!expr.coef)
+ continue;
+ skip_truncation(env, eqs, &expr, bits, false);
+ set_operand(op, i, &expr);
+ }
+}
+
+/*
+ * A test of @truncation tells more, if @op reads no other bits:
+ * r2 = r1; r2 &= 0xf8; r1 &= 0xff; if r1 > 8 goto out;
+ */
+static void read_from_truncation(struct bpf_verifier_env *env, struct bpf_sym_equation *op,
+ const struct bpf_sym_equation *truncation)
+{
+ struct expr expr;
+ int i;
+
+ for (i = 0; i < 2 && is_op(op) && truncation->sym[0] != op->sym[0]; i++) {
+ expr = rhs_expr(op, i);
+ if (expr.coef && rewrite_expr_by_truncation(truncation, &expr, bits_read(env, op)))
+ set_operand(op, i, &expr);
+ }
+}
+
+/* @res and @candidate have the same low @bits bits. If they are equal @expr gets @candidate. */
+static bool take_expr_if_fits(struct bpf_verifier_env *env, const struct scalar *res,
+ struct expr *expr, struct expr candidate, u32 bits)
+{
+ struct scalar bounds;
+
+ bpf_eval_expr(env, env->cur_state, &candidate, &bounds);
+ if (!scalars_fit_in_bits(&bounds, res, bits))
+ return false;
+ *expr = candidate;
+ return true;
+}
+
+/* @res is an extension of the low @bits bits of @expr. Make @expr the value of @res. */
+static bool expr_low_bits(struct bpf_verifier_env *env, struct scalar *res,
+ struct expr *expr, u32 bits)
+{
+ struct bpf_verifier_state *st = env->cur_state;
+ struct bpf_sym_equation *sym_eq;
+ struct scalar bounds;
+ struct expr operand;
+
+ if (bits == 64)
+ return true;
+ if (!(expr->coef & low_bits(bits)))
+ return false;
+ if (take_expr_if_fits(env, res, expr, *expr, bits))
+ return true;
+ operand = *expr;
+ skip_truncation(env, &st->equations, &operand, bits, true);
+ if (take_expr_if_fits(env, res, expr, operand, bits))
+ return true;
+ skip_truncation(env, &st->equations, expr, bits, false);
+ for_each_sum(env, &st->equations, sym_eq) {
+ operand = *expr;
+ if (rewrite_expr_by_truncation(sym_eq, &operand, bits) &&
+ take_expr_if_fits(env, res, expr, operand, bits))
+ return true;
+ }
+ scalar_set_unknown(&bounds);
+ if (!add_divmod_equation(env, expr, BIT_ULL(bits), &bounds, res, false))
+ return false;
+ /* a number has a range for 32 bits */
+ if (bits < 32)
+ for_each_sym_equation(env, &st->equations, sym_eq)
+ read_from_truncation(env, sym_eq, st->equations.sym_equations +
+ st->equations.sym_equation_cnt - 1);
+ return true;
+}
+
+/* The source goes into the equation of the result */
+static bool assign_sym_to_src(struct bpf_verifier_env *env)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ const struct bpf_insn *insn = &env->prog->insnsi[eq_env->insn_idx];
+
+ if (!eq_env->has_expr[1] && insn->src_reg != insn->dst_reg &&
+ reg_is_live_after(env, insn->src_reg))
+ eq_env->has_expr[1] =
+ assign_sym(env, bpf_reg_to_xreg(env->cur_state, insn->src_reg), true,
+ &eq_env->operand_expr[1], &eq_env->is_ptr[1]);
+ return eq_env->has_expr[1];
+}
+
+static bool alu_add(struct bpf_verifier_env *env, struct expr *expr, s64 sign, u32 bits)
+{
+ struct bpf_equations *eqs = &env->cur_state->equations;
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct expr *a = &eq_env->operand_expr[0], *b = &eq_env->operand_expr[1], neg;
+ struct scalar bounds;
+ struct equation_pos pos = {};
+ int i;
+
+ for (i = 0; i < 2; i++) {
+ if (i ? assign_sym_to_src(env) : eq_env->has_expr[0])
+ continue;
+ /* it is dead after the insn. Its range is the sum minus the other operand. */
+ if ((bits == 64 ? !~eq_env->operand_bounds[i].r64.size :
+ eq_env->operand_bounds[i].r32.size == U32_MAX) ||
+ !(i || assign_sym_to_src(env)) || !eq_env->operand_expr[!i].coef)
+ return false;
+ eq_env->operand_expr[i] =
+ make_expr(1,
+ bpf_new_sym(env, eqs, &eq_env->operand_bounds[i]), 0);
+ if (env->no_mem)
+ return false;
+ }
+ neg = expr_scale(*b, -sign, 0);
+ if (bpf_find_next_diff(env, eqs, a, &neg, &pos, expr))
+ return true;
+ scalar_add_scaled(&bounds, &eq_env->operand_bounds[0], &eq_env->operand_bounds[1], sign);
+ scalar_bounds_sync(&bounds);
+ *expr = make_expr(1, bpf_new_sym(env, eqs, &bounds), 0);
+ return bpf_add_sym_equation(env, eqs, (struct bpf_sym_equation){
+ { -1, a->coef, -neg.coef },
+ { expr->var, a->var, b->var }, eq_env->insn_idx, a->off - neg.off });
+}
+
+/* An insn that is a sum, a product with a constant, or the reverse of that */
+static bool alu_linear(struct bpf_verifier_env *env, const struct bpf_insn *insn,
+ struct scalar *res, struct expr *expr)
+{
+ u32 bits = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct tnum a = eq_env->operand_bounds[0].var_off;
+ struct tnum b = eq_env->operand_bounds[1].var_off;
+ u8 op = BPF_OP(insn->code);
+ struct scalar bounds;
+ struct expr untruncated;
+ u64 c;
+
+ if (op == BPF_MOV) {
+ if (insn->imm || !assign_sym_to_src(env))
+ return false;
+ *expr = eq_env->operand_expr[1];
+ return expr_low_bits(env, res, expr, insn->off ?: bits);
+ }
+ if (insn->off)
+ return false;
+ if ((op == BPF_OR || op == BPF_XOR) && !((a.value | a.mask) & (b.value | b.mask)))
+ op = BPF_ADD;
+ if (op == BPF_ADD || op == BPF_SUB)
+ return alu_add(env, expr, op == BPF_ADD ? 1 : -1, bits) &&
+ expr_low_bits(env, res, expr, bits);
+ if (!eq_env->has_expr[0])
+ return false;
+ if (op == BPF_MUL && !eq_env->operand_expr[0].coef && assign_sym_to_src(env))
+ swap(eq_env->operand_expr[0], eq_env->operand_expr[1]);
+ if (!eq_env->has_expr[1] || eq_env->operand_expr[1].coef)
+ return false;
+ *expr = eq_env->operand_expr[0];
+ c = eq_env->operand_expr[1].off & low_bits(bits);
+ switch (op) {
+ case BPF_NEG:
+ *expr = expr_scale(*expr, -1, 0);
+ return expr_low_bits(env, res, expr, bits);
+ case BPF_LSH:
+ if (c >= bits)
+ return false;
+ c = BIT_ULL(c);
+ fallthrough;
+ case BPF_MUL:
+ *expr = expr_scale(*expr, bits == 32 ? (s32)c : c, 0);
+ return expr_low_bits(env, res, expr, bits);
+ case BPF_AND:
+ if (c && !(c & (c + 1)))
+ return expr_low_bits(env, res, expr, min_t(u32, fls64(c), bits));
+ /* x & -8 == x - (x & 7) */
+ scalar_set_unknown(&bounds);
+ bounds.var_off = tnum_and(a, tnum_const(~c));
+ scalar_bounds_sync(&bounds);
+ return add_divmod_equation(env, expr, 1, res, &bounds, true);
+ case BPF_RSH:
+ case BPF_ARSH:
+ if (!c || c >= bits)
+ return false;
+ /* w1 <<= 24; w1 s>>= 24; */
+ untruncated = *expr;
+ if (expr->coef & low_bits(c))
+ skip_truncation(env, &env->cur_state->equations, &untruncated, bits, true);
+ if (!((untruncated.coef | untruncated.off) & low_bits(c))) {
+ *expr = make_expr(untruncated.coef >> c, untruncated.var,
+ (s64)untruncated.off >> c);
+ return expr_low_bits(env, res, expr, bits - c);
+ }
+ c = BIT_ULL(c);
+ fallthrough;
+ case BPF_DIV:
+ if (c < 2 || (bits == 32 && cnum64_umax(eq_env->operand_bounds[0].r64) >
+ (op == BPF_ARSH ? S32_MAX : U32_MAX)))
+ return false;
+ scalar_set_below(&bounds, c);
+ return add_divmod_equation(env, expr, c, res, &bounds, true);
+ }
+ return false;
+}
+
+/* The operands of an op possibly skipped their truncations, see op_skips_truncations() */
+static bool find_same_op(struct bpf_verifier_env *env, struct expr *expr)
+{
+ struct bpf_equations *eqs = &env->cur_state->equations;
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct bpf_sym_equation sym_eq = { .insn_idx = eq_env->insn_idx };
+ struct expr op[2];
+ int i;
+
+ if (bpf_find_op(env, eqs, eq_env->insn_idx, eq_env->operand_expr, expr))
+ return true;
+ for (i = 0; i < 2; i++)
+ set_operand(&sym_eq, i, &eq_env->operand_expr[i]);
+ op_skips_truncations(env, eqs, &sym_eq);
+ for (i = 0; i < 2; i++)
+ op[i] = rhs_expr(&sym_eq, i);
+ return bpf_find_op(env, eqs, eq_env->insn_idx, op, expr);
+}
+
+/*
+ * The same as read_from_truncation(), the test is before @op:
+ * r2 = r1; r2 &= 0xff; if r2 > 0xdf goto out; r1 &= 0xf0;
+ */
+static void refine_op_by_truncations(struct bpf_verifier_env *env,
+ const struct bpf_sym_equation *op)
+{
+ struct bpf_equations *eqs = &env->cur_state->equations;
+ struct bpf_sym_equation *sym_eq, copy;
+ struct scalar bounds;
+
+ if (bits_read(env, op) >= 32)
+ return;
+ for_each_sum(env, eqs, sym_eq) {
+ copy = *op;
+ read_from_truncation(env, ©, sym_eq);
+ if (memcmp(©, op, sizeof(copy)) && run_op(env, env->cur_state, ©, &bounds))
+ refine_bounds(eqs, &eqs->syms[op->sym[0]], &bounds);
+ }
+}
+
+/*
+ * A sum does not tell all about the result. A jump that refines the operands runs the op again:
+ * r2 = r1; r2 ^= 5; if r1 > 20 goto out;
+ */
+static bool alu_scalar(struct bpf_verifier_env *env, const struct bpf_insn *insn,
+ struct scalar *res, struct expr *expr)
+{
+ struct bpf_equations *eqs = &env->cur_state->equations;
+ struct bpf_equation_env *eq_env = env->eq_env;
+ bool ok;
+
+ if (BPF_OP(insn->code) == BPF_MOV) {
+ eq_env->operand_expr[0] = make_expr(0, 0, 0);
+ eq_env->has_expr[0] = !insn->imm;
+ }
+ if (eq_env->has_expr[0] && eq_env->has_expr[1] && find_same_op(env, expr))
+ return true;
+ ok = alu_linear(env, insn, res, expr);
+ if ((ok && !eq_env->divided) || !eq_env->has_expr[0] || !assign_sym_to_src(env))
+ return ok;
+ if (!ok)
+ *expr = make_expr(1, bpf_new_sym(env, eqs, res), 0);
+ if (!bpf_add_op(env, eqs, eq_env->insn_idx, eq_env->operand_expr, expr))
+ return true;
+ op_skips_truncations(env, eqs, &eqs->sym_equations[eqs->sym_equation_cnt - 1]);
+ refine_op_by_truncations(env, &eqs->sym_equations[eqs->sym_equation_cnt - 1]);
+ return true;
+}
+
+/* The insn is proven */
+int bpf_equations_after_insn(struct bpf_verifier_env *env)
+{
+ struct bpf_verifier_state *st = env->cur_state;
+ struct bpf_equation_env *eq_env = env->eq_env;
+ const struct bpf_insn *insn;
+ struct scalar *res, bounds;
+ bool is_ptr, ok, was_ptr;
+ struct expr expr;
+ u8 class, op;
+ u64 dst;
+
+ if (!eq_env)
+ return 0;
+ insn = &env->prog->insnsi[eq_env->insn_idx];
+ dst = bpf_reg_to_xreg(st, insn->dst_reg);
+ class = bpf_atomic_is_load_acq(insn) ? BPF_LDX : BPF_CLASS(insn->code);
+ op = BPF_OP(insn->code);
+ if ((class != BPF_ALU && class != BPF_ALU64 && class != BPF_LDX) || st->speculative)
+ goto out;
+ res = bpf_find_var_bounds(env, st, dst, &is_ptr);
+ if (!res || tnum_is_const(res->var_off))
+ goto out;
+ bounds = *res;
+ if (!is_ptr && scalar_align_bounds(&bounds))
+ *res = bounds;
+ else if (!is_ptr)
+ st->equations.infeasible = true;
+ if (class == BPF_LDX)
+ ok = eq_env->fill_xreg != VAR_NONE &&
+ assign_sym(env, eq_env->fill_xreg, true, &expr, &was_ptr) &&
+ was_ptr == is_ptr && expr_low_bits(env, res, &expr, eq_env->fill_bits);
+ else if (op == BPF_MOV ? is_ptr != eq_env->is_ptr[1] :
+ is_ptr != eq_env->is_ptr[0] + eq_env->is_ptr[1])
+ ok = false;
+ else if (is_ptr)
+ ok = class == BPF_ALU64 && alu_linear(env, insn, res, &expr);
+ else
+ ok = alu_scalar(env, insn, res, &expr);
+ if (!ok || (expr.coef && !var_is_sym(expr.var)))
+ goto out;
+ /* it was computed before */
+ eval_expr_nosync(env, st, &expr, &bounds);
+ if (!is_ptr)
+ refine_bounds(&st->equations, res, &bounds);
+ bpf_set_xreg_equation(env, &st->equations, dst, &expr, is_ptr);
+out:
+ return env->no_mem ? -ENOMEM : 0;
+}
+
static void print_operand(struct bpf_verifier_log *log, const char *prefix, struct expr expr)
{
if (expr.coef)
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index 1cb2f358199..db068a58c30 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -59,7 +59,16 @@ struct bpf_equation_env {
struct sym_scratch *scratch;
u32 *use_links;
u32 scratch_cap, use_link_cap, refined_cnt;
+ /* the insn that is checked and its operands */
+ s32 insn_idx;
+ struct expr operand_expr[2];
+ struct scalar operand_bounds[2];
bool has_expr[2], is_ptr[2], equation_added, divided;
+ /* the arguments of a call */
+ struct expr arg_expr[BPF_REG_5 + 1];
+ u8 has_arg_expr;
+ u64 fill_xreg;
+ u32 fill_bits;
/* the operand of a jump and its bounds, see refine_from_reg() */
struct expr jump_expr;
struct scalar jump_bounds;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 033f6b42ab1..cfcb20dda5e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3754,7 +3754,9 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
/* Mark slots affected by this stack write. */
for (i = 0; i < size; i++)
ss->slot_type[(slot - i) % BPF_REG_SIZE] = type;
+ return 0;
}
+ bpf_equations_spill(env, state, spi, value_regno);
return 0;
}
@@ -4016,6 +4018,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
/* coerce_scalar_to_size will adjust the boundaries. */
state->regs[dst_regno] = *reg;
+ bpf_equations_fill(env, reg_state, spi, size);
} else {
/* the low bytes are from the spill */
bool fill_exceeds_spill =
@@ -4023,6 +4026,9 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
int spill_cnt = 0, zero_cnt = 0;
+ if (fill_exceeds_spill)
+ bpf_equations_fill(env, reg_state, spi, spill_size);
+
for (i = 0; i < size; i++) {
type = stype[(slot - i) % BPF_REG_SIZE];
if (type == STACK_SPILL) {
@@ -4069,6 +4075,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
} else if (dst_regno >= 0) {
/* restore register state from stack */
state->regs[dst_regno] = *reg;
+ bpf_equations_fill(env, reg_state, spi, size);
} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
/* If dst_regno==-1, the caller is asking us whether
* it is acceptable to use this value as a SCALAR_VALUE
@@ -7043,9 +7050,9 @@ static int check_stack_access_within_bounds(
* Every stack access passes through here, including buffers passed to
* helpers and kfuncs and accesses into a caller's frame, so check the
* fastcall contract of the frame that owns the slots once for all of
- * them. Zero-sized accesses touch no stack and keep the rewrite enabled.
+ * them. An empty range touches no stack and keeps the rewrite enabled.
*/
- if (access_size)
+ if (min_off < max_off)
check_fastcall_stack_contract(env, state, env->insn_idx, min_off);
/* Note that there is no stack access with offset zero, so the needed stack
@@ -8053,6 +8060,28 @@ enum bpf_mem_size_failure {
BPF_MEM_SIZE_FAIL_SIZE,
};
+/*
+ * The check looks at the bytes from the smallest offset to the largest offset plus the size.
+ * The largest offset and the largest size do not have to go together:
+ * read(buf + off, 64 - off);
+ * So the result is not the size of the access. It is 0 for off = 0..64.
+ * The checked bytes of a packet pointer are relative to the pointer. It needs the largest size.
+ */
+static u32 size_at_max_offset(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg,
+ struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno)
+{
+ u32 type = base_type(mem_reg->type);
+ s64 end;
+
+ if ((type != PTR_TO_MAP_VALUE && type != PTR_TO_MEM && type != PTR_TO_STACK) ||
+ reg_smin(mem_reg) <= -BPF_MAX_VAR_OFF || reg_smax(mem_reg) >= BPF_MAX_VAR_OFF)
+ return reg_umax(size_reg);
+ end = bpf_equations_max_sum(env, reg_from_argno(mem_argno), reg_from_argno(size_argno));
+ end = clamp(end, reg_smax(mem_reg) + (s64)reg_umin(size_reg),
+ reg_smax(mem_reg) + (s64)reg_umax(size_reg));
+ return end - reg_smax(mem_reg);
+}
+
/* verify arguments to helpers or kfuncs consisting of a pointer and an access
* size.
*
@@ -8067,6 +8096,7 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
enum bpf_mem_size_failure *failure)
{
int err = 0;
+ u32 size;
if (failure)
*failure = BPF_MEM_SIZE_FAIL_NONE;
@@ -8110,11 +8140,12 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
goto size_error;
}
+ size = size_at_max_offset(env, mem_reg, size_reg, mem_argno, size_argno);
if (access_type & BPF_READ)
- err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
+ err = check_helper_mem_access(env, mem_reg, mem_argno, size,
BPF_READ, zero_size_allowed, meta, NULL);
if (!err && access_type & BPF_WRITE)
- err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
+ err = check_helper_mem_access(env, mem_reg, mem_argno, size,
BPF_WRITE, zero_size_allowed, meta, NULL);
if (err && failure)
*failure = BPF_MEM_SIZE_FAIL_MEMORY;
@@ -21079,6 +21110,8 @@ static int push_unwind_exit(struct bpf_verifier_env *env)
__mark_reg_const_zero(®s[BPF_REG_0]);
if (bpf_ret_reg_pair(env, cur_func(env)->subprogno))
__mark_reg_const_zero(®s[BPF_REG_2]);
+ bpf_equations_forget_xreg(env, branch, bpf_xreg(branch->curframe, BPF_REG_0));
+ bpf_equations_forget_xreg(env, branch, bpf_xreg(branch->curframe, BPF_REG_2));
return 0;
}
@@ -21343,8 +21376,9 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
if (err)
return err;
- err = do_check_insn(env, &do_print_state);
+ bpf_equations_before_insn(env);
fixups = insn_aux->fixups;
+ err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
if (nospec_ends_path(env, state, err)) {
/* Prevent this speculative path from ever reaching the
* insn that would have been unsafe to execute.
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns
2026-11-20 7:41 ` [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 8:26 ` sashiko-bot
1 sibling, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2711 bytes --]
> bpf: Make equations from ALU, spill, fill and call insns
TL;DR:
- Build breaks in equation.c: undefined clean_sym_cnt and
bpf_equations_clean() (added by next patch), hurting bisection
- Undeclared 'i' in add_divmod_equation(), also fixed only in the next patch
> diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
> --- a/kernel/bpf/equation.c
> +++ b/kernel/bpf/equation.c
[ ... ]
> +/* Note the operands. Delete the equations of the registers that the insn writes. */
> +void bpf_equations_before_insn(struct bpf_verifier_env *env)
> +{
[ ... ]
> + if (!eq_env)
> + return;
> + if (st->speculative || eqs->sym_cnt > 2 * eqs->clean_sym_cnt + MAX_BPF_REG)
> + bpf_equations_clean(env, st);
Does this build? struct bpf_equations has no clean_sym_cnt member at this
commit, and bpf_equations_clean() is neither declared nor defined anywhere,
so this should fail with "no member named clean_sym_cnt" and an implicit
declaration error for bpf_equations_clean().
kernel/bpf/Makefile builds equation.o under CONFIG_BPF_SYSCALL, so every
BPF-enabled config is affected and bisection breaks.
The next commit, a50a112fb866 ("bpf: Delete the equations that say
nothing"), adds 'u32 clean_sym_cnt;' to struct bpf_equations, the
prototype in bpf_verifier.h and the definition of bpf_equations_clean().
Should those move into this patch, or this call move into that one?
The commit message says "A speculative state gets no equations", but that
behavior lives entirely in the missing bpf_equations_clean(), so it is not
implemented by this patch alone.
[ ... ]
> +/* @expr = @divisor * quot + rem. Returns the expr of a new sym: @quot or @rem. */
> +static bool add_divmod_equation(struct bpf_verifier_env *env, struct expr *expr, u64 divisor,
> + const struct scalar *quot, const struct scalar *rem, bool want_quot)
> +{
> + struct bpf_equations *eqs = &env->cur_state->equations;
> + struct bpf_sym_equation sym_eq = { { -1, expr->coef, -divisor }, { 0, expr->var },
> + env->eq_env->insn_idx, expr->off };
[ ... ]
> + for (i = 0; i < 3; i++)
> + if (sym_eq.coef[i])
> + env->eq_env->scratch[sym_eq.sym[i]].refined_sweep = 0;
Where is 'i' declared here? The only locals in add_divmod_equation() are
eqs and sym_eq, and there is no file-scope 'i', so this looks like an
"'i' undeclared" compile error.
The same a50a112fb866 commit adds 'int i;' to this function. Could the
declaration be added in this patch so it builds on its own?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns
2026-11-20 7:41 ` [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
@ 2026-11-20 8:26 ` sashiko-bot
1 sibling, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:26 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 2 potential
issues to consider:
- [High] bpf: undeclared bpf_equations_clean() in
bpf_equations_before_insn()
- [High] bpf: undeclared loop variable i in add_divmod_equation()
--
commit 2ecfa437c292398a3fbabc9325d501e57c299c72
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Make equations from ALU, spill, fill and call insns
This commit records and updates verifier equations across ALU operations,
stack spills and fills, and subprogram or helper calls, and uses argument
equations to bound memory access sizes at the maximum pointer offset.
> diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
> index 16b50253a6e10..5c9a3df8ce428 100644
> --- a/kernel/bpf/equation.c
> +++ b/kernel/bpf/equation.c
> @@ -1039,6 +1039,541 @@ void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state
[ ... ]
> +/* Note the operands. Delete the equations of the registers that the insn writes. */
> +void bpf_equations_before_insn(struct bpf_verifier_env *env)
> +{
> + const struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];
> + u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
> + bool src_is_reg = BPF_SRC(insn->code) == BPF_X && op != BPF_END;
> + bool alu = class == BPF_ALU || class == BPF_ALU64;
> + struct bpf_verifier_state *st = env->cur_state;
> + struct bpf_equation_env *eq_env = env->eq_env;
> + u32 def = env->insn_aux_data[env->insn_idx].def_regs, fr = st->curframe, i, r;
> + struct bpf_equations *eqs = &st->equations;
> + bool is_ptr;
> +
> + if (!eq_env)
> + return;
> + if (st->speculative || eqs->sym_cnt > 2 * eqs->clean_sym_cnt + MAX_BPF_REG)
> + bpf_equations_clean(env, st);
[Severity: High]
Does this break the build with CONFIG_BPF_SYSCALL=y? At this commit,
struct bpf_equations has no clean_sym_cnt member and bpf_equations_clean()
is not declared or defined until commit "bpf: Delete the equations that
say nothing" later in the series, breaking git bisect.
[ ... ]
> +/* @expr = @divisor * quot + rem. Returns the expr of a new sym: @quot or @rem. */
> +static bool add_divmod_equation(struct bpf_verifier_env *env, struct expr *expr, u64 divisor,
> + const struct scalar *quot, const struct scalar *rem, bool want_quot)
> +{
> + struct bpf_equations *eqs = &env->cur_state->equations;
> + struct bpf_sym_equation sym_eq = { { -1, expr->coef, -divisor }, { 0, expr->var },
> + env->eq_env->insn_idx, expr->off };
> +
> + sym_eq.sym[0] = bpf_new_sym(env, eqs, rem);
> + sym_eq.sym[2] = bpf_new_sym(env, eqs, quot);
> + *expr = sym_expr(1, sym_eq.sym[want_quot ? 2 : 0], 0);
> + env->eq_env->divided = true;
> + if (!bpf_add_sym_equation(env, eqs, sym_eq))
> + return false;
> + refine_syms_by_equation(env, env->cur_state,
> + &eqs->sym_equations[eqs->sym_equation_cnt - 1], 0);
> + for (i = 0; i < 3; i++)
> + if (sym_eq.coef[i])
> + env->eq_env->scratch[sym_eq.sym[i]].refined_sweep = 0;
> + return true;
> +}
[Severity: High]
Is the loop variable i declared in add_divmod_equation()? Without a local
declaration, this fails to compile until commit "bpf: Delete the equations
that say nothing" adds int i later in the series.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=19
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 20/49] bpf: Delete the equations that say nothing
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (18 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 19/49] bpf: Make equations from ALU, spill, fill and call insns Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` bot+bpf-ci
2026-11-20 7:41 ` [PATCH bpf-next 21/49] bpf: Compare states by join_key, then by value Alexei Starovoitov
` (28 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Insns add syms, and registers die. Without a clean-up the table grows,
and two states with the same facts have tables that do not compare.
if then
sym0 is 5 put 5 into the equations
sym1 = sym0 + 8 replace sym1
r1 = sym0, only use delete, the range of r1 says the same
no xreg holds sym0 eliminate it:
p = sym0 + len, sym0 = 8 * trip + sym1 -> p = 8 * trip + sym2
The result can be a second equation for the same sum, often with a
smaller range than the guess of a join. refine_same_sum() gives the
range to the first equation and deletes the second one. Two sums can
also become the same when a sym is replaced.
The new range needs a sym to hold it. If the equation has no sym with
one use, the sym of the deleted equation holds it:
x = m + d, m = 2 * k + 41 * j, k is dead -> x = 41 * j + d'
normalize_equations() also sets the order: each equation has at most
one sym that the xregs and the earlier equations do not determine. A
later patch needs that order to check a table in one walk.
A state keeps 2 * SUMS_PER_XREG sym equations for each xreg equation.
That cuts a chain such as 'x += k; x += k; ...'. A pass that is over
its budget drops all equations.
A state without an equation has more values. So it is always safe to
delete an equation.
The clean-up runs when a state is cleaned, and when the number of syms
doubles.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 2 +
kernel/bpf/equation.c | 426 +++++++++++++++++++++++++++++++++++
kernel/bpf/equation.h | 3 +
kernel/bpf/states.c | 1 +
4 files changed, 432 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 1a9adf126f9..d4159384821 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -555,6 +555,7 @@ struct bpf_equations {
bool normalized; /* see normalize_equations() */
bool infeasible; /* the equations cannot hold: the program does not get here */
bool needs_refine; /* widened: the equations are not applied, see bpf_flow_next() */
+ u32 clean_sym_cnt; /* after the last bpf_equations_clean() */
};
struct bpf_verifier_state {
@@ -2321,6 +2322,7 @@ void bpf_equation_env_free(struct bpf_verifier_env *env);
int bpf_equations_copy(struct bpf_verifier_env *env, struct bpf_equations *dst,
const struct bpf_equations *src);
void bpf_equations_free(struct bpf_equations *eqs);
+void bpf_equations_clean(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
s64 bpf_equations_max_sum(struct bpf_verifier_env *env, int regno1, int regno2);
void bpf_equations_before_insn(struct bpf_verifier_env *env);
int bpf_equations_after_insn(struct bpf_verifier_env *env);
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index 5c9a3df8ce4..479cb1466e1 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -640,6 +640,7 @@ static bool refine_ptr(struct bpf_verifier_env *env, struct bpf_verifier_state *
return true;
}
+/* All states share the marks. If propagate_bounds() does not follow, clear the mark. */
static void mark_sym_refined(struct bpf_verifier_env *env, u32 sym, u32 sweep)
{
struct bpf_equation_env *eq_env = env->eq_env;
@@ -1195,6 +1196,7 @@ static bool add_divmod_equation(struct bpf_verifier_env *env, struct expr *expr,
struct bpf_equations *eqs = &env->cur_state->equations;
struct bpf_sym_equation sym_eq = { { -1, expr->coef, -divisor }, { 0, expr->var },
env->eq_env->insn_idx, expr->off };
+ int i;
sym_eq.sym[0] = bpf_new_sym(env, eqs, rem);
sym_eq.sym[2] = bpf_new_sym(env, eqs, quot);
@@ -1574,6 +1576,430 @@ int bpf_equations_after_insn(struct bpf_verifier_env *env)
return env->no_mem ? -ENOMEM : 0;
}
+/* Clean up */
+
+static bool sym_has_one_use(const struct sym_scratch *scratch, u32 sym)
+{
+ return sym != NO_SYM && scratch[sym].use_cnt == 1;
+}
+
+static void count_sym_uses(struct bpf_verifier_env *env, const struct bpf_equations *eqs)
+{
+ struct sym_scratch *scratch = env->eq_env->scratch;
+ const struct bpf_xreg_equation *xreg_eq;
+ const struct bpf_sym_equation *sym_eq;
+ u32 i;
+
+ bpf_add_cost(env, eqs->sym_cnt, BPF_COST_COPY);
+ memset(scratch, 0, eqs->sym_cnt * sizeof(*scratch));
+ for_each_xreg_equation(env, eqs, xreg_eq)
+ if (xreg_eq->coef)
+ scratch[xreg_eq->sym].use_cnt++;
+ for_each_sym_equation(env, eqs, sym_eq) {
+ for (i = 0; i < 3; i++)
+ if (sym_eq->coef[i])
+ scratch[sym_eq->sym[i]].use_cnt++;
+ if (is_op(sym_eq))
+ scratch[sym_eq->sym[0]].op = sym_eq;
+ }
+}
+
+static void replace_operand(struct bpf_sym_equation *sym_eq, u32 sym, const struct expr *new_expr)
+{
+ struct expr expr;
+ int i;
+
+ for (i = 0; i < 2; i++) {
+ if (sym_eq->sym[i + 1] != sym)
+ continue;
+ expr = expr_scale(*new_expr, sym_eq->coef[i + 1], rhs_expr(sym_eq, i).off);
+ set_operand(sym_eq, i, &expr);
+ }
+ if (sym_eq->sym[0] == sym || sym_eq->sym[0] == var_symno(new_expr->var))
+ sym_eq->sym[0] = NO_SYM;
+}
+
+/* Replace @sym with @expr */
+static bool substitute_sym(struct bpf_verifier_env *env, struct bpf_equations *eqs, u32 sym,
+ const struct expr *expr, const struct bpf_sym_equation *def_equation)
+{
+ struct bpf_xreg_equation *xreg_eq;
+ struct bpf_sym_equation *sym_eq, new;
+ bool changed = false;
+ int i;
+
+ if (over_limit(env))
+ return false;
+ for_each_xreg_equation(env, eqs, xreg_eq) {
+ if (xreg_eq->sym != sym)
+ continue;
+ xreg_eq->off += xreg_eq->coef * expr->off;
+ xreg_eq->coef *= expr->coef;
+ xreg_eq->sym = expr->var;
+ }
+ for_each_sym_equation(env, eqs, sym_eq) {
+ i = find_sym_pos(sym_eq, sym);
+ if (i == 3 || sym_eq == def_equation || sym_eq->sym[0] == NO_SYM)
+ continue;
+ if (is_op(sym_eq)) {
+ replace_operand(sym_eq, sym, expr);
+ continue;
+ }
+ new = *sym_eq;
+ new.off += new.coef[i] * expr->off;
+ new.coef[i] *= expr->coef;
+ new.sym[i] = expr->var;
+ if (isolate_sym(&new))
+ *sym_eq = new;
+ else if (i || !expr->coef || find_sym_pos(sym_eq, expr->var) < 3)
+ sym_eq->sym[0] = NO_SYM;
+ changed = true;
+ }
+ return changed;
+}
+
+/* The position of the sym that only @sym_eq uses. -1 if there is none. */
+static int find_one_use_sym(const struct sym_scratch *scratch,
+ const struct bpf_sym_equation *sym_eq)
+{
+ int i;
+
+ for (i = 2; i >= 0 && !sym_has_one_use(scratch, sym_eq->sym[i]); i--)
+ ;
+ return i;
+}
+
+/* Another equation is for the same sum as @sym_eq. It gets the range of the sym at @one_use. */
+static bool refine_same_sum(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ const struct bpf_sym_equation *sym_eq, int one_use)
+{
+ int i = !one_use, j = 2 - (one_use == 2);
+ struct expr a = sym_expr(sym_eq->coef[i], sym_eq->sym[i], sym_eq->off), diff;
+ struct expr b = sym_expr(-sym_eq->coef[j], sym_eq->sym[j], 0);
+ struct equation_pos pos = {};
+ struct scalar bounds;
+
+ /* the other equation can be a wrong guess, and then both are lost */
+ if (eqs == &env->eq_env->join_state.equations)
+ return false;
+ while (b.coef && bpf_find_next_diff(env, eqs, &a, &b, &pos, &diff)) {
+ if (!is_plus_minus_one(diff.coef) || var_symno(diff.var) == NO_SYM ||
+ var_symno(diff.var) == sym_eq->sym[one_use])
+ continue;
+ scalar_scale(&bounds, &eqs->syms[sym_eq->sym[one_use]], -sym_eq->coef[one_use]);
+ refine_sym(env, eqs, diff, bounds, 1);
+ eqs->sym_equations[pos.next - 1].insn_idx = -1;
+ return true;
+ }
+ return false;
+}
+
+/*
+ * No xreg holds the sym at @pos of @def_equation any more. Eliminate it from the equations:
+ * p' = sym0 + len, sym0 = 8 * trip + sym1 -> p' = 8 * trip + sym2, sym2 is sym1 + len
+ */
+static bool eliminate_sym(struct bpf_verifier_env *env, struct bpf_equations *eqs,
+ struct bpf_sym_equation *def_equation, int pos)
+{
+ struct sym_scratch *scratch = env->eq_env->scratch;
+ int one_use = find_one_use_sym(scratch, def_equation), i, j, k;
+ struct bpf_sym_equation *sym_eq, def = *def_equation;
+ struct scalar bounds, rest, *s;
+ bool spare = def.insn_idx < 0;
+ u32 one_use_sym;
+ s64 factor;
+
+ if (one_use < 0 || one_use == pos || !is_plus_minus_one(def.coef[pos]))
+ return false;
+ def_equation->sym[0] = NO_SYM;
+ rest = eqs->syms[def.sym[one_use]];
+ for_each_sum(env, eqs, sym_eq) {
+ i = find_sym_pos(sym_eq, def.sym[pos]);
+ if (i == 3 || sym_eq == def_equation)
+ continue;
+ j = find_one_use_sym(scratch, sym_eq);
+ for (k = 0; k < 3 && j < 0 && spare; k++)
+ if (k != i && sym_eq->coef[k] && !scratch[sym_eq->sym[k]].determined)
+ j = k;
+ if (j < 0)
+ continue;
+ bpf_add_cost(env, 4, BPF_COST_XREG);
+ factor = -sym_eq->coef[i] * def.coef[pos];
+ scalar_scale(&bounds, &eqs->syms[sym_eq->sym[j]], sym_eq->coef[j]);
+ /* no xreg gives the value of that sym. The sym of @def takes its place. */
+ if (!sym_has_one_use(scratch, sym_eq->sym[j])) {
+ sym_eq->sym[j] = def.sym[one_use];
+ spare = false;
+ }
+ s = &eqs->syms[sym_eq->sym[j]];
+ scalar_add_scaled(s, &bounds, &rest, factor * def.coef[one_use]);
+ scalar_bounds_sync(s);
+ sym_eq->coef[j] = 1;
+ sym_eq->coef[i] = factor * def.coef[3 - pos - one_use];
+ sym_eq->sym[i] = def.sym[3 - pos - one_use];
+ sym_eq->off += factor * def.off;
+ sym_eq->insn_idx = -1;
+ /* a sym with one use is the last choice for the left side */
+ swap(sym_eq->coef[j], sym_eq->coef[2]);
+ swap(sym_eq->sym[j], sym_eq->sym[2]);
+ one_use_sym = sym_eq->sym[2];
+ if (!isolate_sym(sym_eq) ||
+ refine_same_sum(env, eqs, sym_eq, find_sym_pos(sym_eq, one_use_sym)))
+ sym_eq->sym[0] = NO_SYM;
+ }
+ return true;
+}
+
+/* No xreg holds @sym, no jump tested it. An op made it of the syms of @sum: w1 += 1; w1 += 1; */
+static bool only_op_is_left(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct bpf_sym_equation *sum, u32 sym)
+{
+ const struct sym_scratch *scratch = &env->eq_env->scratch[sym];
+ const struct bpf_sym_equation *op = scratch->op;
+ struct scalar bounds;
+
+ return op && scratch->use_cnt == 2 && (!op->coef[1] || find_sym_pos(sum, op->sym[1]) < 3) &&
+ (!op->coef[2] || find_sym_pos(sum, op->sym[2]) < 3) &&
+ run_op(env, st, op, &bounds) && scalar_subsumes(&st->equations.syms[sym], &bounds);
+}
+
+static bool equation_is_useless(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct bpf_sym_equation *sym_eq)
+{
+ const struct sym_scratch *scratch = env->eq_env->scratch;
+ int i, sym_cnt = 0, one_use_cnt = 0;
+
+ if (is_op(sym_eq))
+ return sym_has_one_use(scratch, sym_eq->sym[0]) ||
+ (!sym_eq->coef[1] && !sym_eq->coef[2]);
+ for (i = 0; i < 3; i++) {
+ sym_cnt += sym_eq->coef[i] && !only_op_is_left(env, st, sym_eq, sym_eq->sym[i]);
+ if (!sym_has_one_use(scratch, sym_eq->sym[i]))
+ continue;
+ one_use_cnt++;
+ if ((sym_eq->coef[i] & 1) && scalar_is_unknown(&st->equations.syms[sym_eq->sym[i]]))
+ return true;
+ }
+ return one_use_cnt >= sym_cnt - 1;
+}
+
+static void forget_useless_equations(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
+{
+ struct bpf_equations *eqs = &st->equations;
+ struct sym_scratch *scratch = env->eq_env->scratch;
+ struct bpf_xreg_equation *xreg_eq, *next;
+ struct bpf_sym_equation *sym_eq;
+ bool more;
+ int i;
+
+ do {
+ more = false;
+ for_each_sym_equation(env, eqs, sym_eq) {
+ if (sym_eq->sym[0] == NO_SYM || !equation_is_useless(env, st, sym_eq))
+ continue;
+ for (i = 0; i < 3; i++)
+ more |= sym_eq->coef[i] && --scratch[sym_eq->sym[i]].use_cnt == 1;
+ sym_eq->sym[0] = NO_SYM;
+ }
+ } while (more && !over_limit(env));
+ forget_sym_equations(env, eqs, S32_MIN);
+ next = eqs->xreg_equations;
+ for_each_xreg_equation(env, eqs, xreg_eq) {
+ if (xreg_eq->coef && !sym_has_one_use(scratch, xreg_eq->sym))
+ *next++ = *xreg_eq;
+ else if (xreg_eq->coef)
+ scratch[xreg_eq->sym].use_cnt = 0;
+ }
+ eqs->xreg_equation_cnt = next - eqs->xreg_equations;
+}
+
+/* The sym that @sym_eq determines. 3: all are determined. -1: more than one is not. */
+static int find_undetermined_sym(const struct sym_scratch *scratch,
+ const struct bpf_sym_equation *sym_eq)
+{
+ int i, pos = 3;
+
+ for (i = 0; i < 3; i++) {
+ if (!sym_eq->coef[i] || scratch[sym_eq->sym[i]].determined)
+ continue;
+ if (pos < 3 || (i && is_op(sym_eq)))
+ return -1;
+ pos = i;
+ }
+ return pos;
+}
+
+/* Only an op tells a sym of @sym_eq */
+static bool leans_on_op(const struct sym_scratch *scratch, const struct bpf_sym_equation *sym_eq)
+{
+ int i;
+
+ for (i = 0; i < 3; i++)
+ if (sym_eq->coef[i] && scratch[sym_eq->sym[i]].by_op)
+ return true;
+ return is_op(sym_eq);
+}
+
+/* Delete what says nothing. Put the rest in the order that check_equations() needs. */
+static void normalize_equations(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
+{
+ struct sym_scratch *scratch = env->eq_env->scratch;
+ struct bpf_equations *eqs = &st->equations;
+ struct bpf_xreg_equation *xreg_eq;
+ struct bpf_sym_equation *sym_eq, *done, *sweep_start, *end;
+ bool substituted = false;
+ struct expr expr;
+ u32 i, cnt;
+ int pos;
+
+ if (eqs->normalized)
+ return;
+ for (i = 0; i < eqs->sym_cnt; i++) {
+ expr = make_expr(0, 0, eqs->syms[i].var_off.value);
+ if (tnum_is_const(eqs->syms[i].var_off))
+ substituted |= substitute_sym(env, eqs, i, &expr, NULL);
+ }
+ /* see read_from_truncation() */
+ for_each_sym_equation(env, eqs, sym_eq)
+ if (is_op(sym_eq) && bits_read(env, sym_eq) < 32)
+ op_skips_truncations(env, eqs, sym_eq);
+again:
+ if (over_limit(env))
+ eqs->xreg_equation_cnt = eqs->sym_equation_cnt = 0;
+ forget_sym_equations(env, eqs, S32_MIN);
+ count_sym_uses(env, eqs);
+ for_each_sum(env, eqs, sym_eq) {
+ if (sym_eq->coef[2] || sym_eq->sym[0] == NO_SYM)
+ continue;
+ refine_syms_by_equation(env, st, sym_eq, 0);
+ expr = rhs_expr(sym_eq, 0);
+ /* sym0 = 2 * sym1, the only use of sym1: it shows only the low bits of sym0 */
+ if (!is_plus_minus_one(expr.coef) && sym_has_one_use(scratch, sym_eq->sym[1]))
+ continue;
+ substitute_sym(env, eqs, sym_eq->sym[0], &expr, sym_eq);
+ substituted = true;
+ }
+ if (substituted) {
+ forget_sym_equations(env, eqs, S32_MIN);
+ count_sym_uses(env, eqs);
+ /* two sums can be the same now. What an insn made keeps its form. */
+ for_each_sum(env, eqs, sym_eq) {
+ pos = find_one_use_sym(scratch, sym_eq);
+ if (pos >= 0 && sym_eq->insn_idx < 0 &&
+ refine_same_sum(env, eqs, sym_eq, pos))
+ sym_eq->sym[0] = NO_SYM;
+ }
+ forget_sym_equations(env, eqs, S32_MIN);
+ count_sym_uses(env, eqs);
+ substituted = false;
+ }
+ forget_useless_equations(env, st);
+
+ for_each_xreg_equation(env, eqs, xreg_eq)
+ scratch[xreg_eq->sym].determined = true;
+ /* The equations near the xregs go first. Cut a long chain: x += k; x += k; ... */
+ done = eqs->sym_equations;
+ end = eqs->sym_equations +
+ min(eqs->sym_equation_cnt, 2 * SUMS_PER_XREG * eqs->xreg_equation_cnt);
+ do {
+ bpf_add_cost(env, eqs->sym_equation_cnt - (done - eqs->sym_equations),
+ BPF_COST_COPY);
+ sweep_start = done;
+ for (sym_eq = done;
+ sym_eq < eqs->sym_equations + eqs->sym_equation_cnt && done < end; sym_eq++) {
+ pos = find_undetermined_sym(scratch, sym_eq);
+ /* sum += i & 1: go around the old sum, see eliminate_sym() */
+ if (pos < 0 || (pos % 3 && sym_eq->insn_idx >= 0 &&
+ !sym_has_one_use(scratch, sym_eq->sym[pos]) &&
+ leans_on_op(scratch, sym_eq)))
+ continue;
+ if (sym_eq != done)
+ swap(*sym_eq, *done);
+ done++;
+ }
+ for (sym_eq = sweep_start; sym_eq < done; sym_eq++) {
+ pos = find_undetermined_sym(scratch, sym_eq);
+ if (pos == 3)
+ continue;
+ scratch[sym_eq->sym[pos]].by_op = leans_on_op(scratch, sym_eq);
+ scratch[sym_eq->sym[pos]].determined = true;
+ }
+ } while (sweep_start < done && done < end && !over_limit(env));
+ /* Nothing refines the operands of the other ops. The other sums need a sym with one use. */
+ for (sym_eq = done; sym_eq < eqs->sym_equations + eqs->sym_equation_cnt; sym_eq++)
+ for (i = 0; i < 3 && !is_op(sym_eq) && find_one_use_sym(scratch, sym_eq) < 0; i++)
+ if (sym_eq->coef[i])
+ scratch[sym_eq->sym[i]].new_symno = scratch[sym_eq->sym[i]].by_op;
+ cnt = 0;
+ for_each_sym_equation(env, eqs, sym_eq) {
+ if (!is_op(sym_eq) || (sym_eq < done && !scratch[sym_eq->sym[0]].new_symno))
+ continue;
+ sym_eq->sym[0] = NO_SYM;
+ cnt++;
+ }
+ if (cnt)
+ goto again;
+ /* the equations that are left lead through unknown syms. The last insn goes first. */
+ for (cnt = 0; cnt < 2 && done < end; cnt++)
+ for (sym_eq = eqs->sym_equations + eqs->sym_equation_cnt; sym_eq-- > done;)
+ for (i = 0; i < 3 && (cnt || sym_eq->insn_idx >= 0); i++)
+ if (sym_eq->coef[i] &&
+ !scratch[sym_eq->sym[i]].determined &&
+ eliminate_sym(env, eqs, sym_eq, i))
+ goto again;
+ if (done != eqs->sym_equations + eqs->sym_equation_cnt) {
+ eqs->sym_equation_cnt = done - eqs->sym_equations;
+ goto again;
+ }
+
+ eqs->normalized = true;
+ for (i = 0, cnt = 0; i < eqs->sym_cnt; i++) {
+ scratch[i].refined_sweep = 0;
+ if (!scratch[i].use_cnt)
+ continue;
+ if (cnt != i)
+ eqs->syms[cnt] = eqs->syms[i];
+ scratch[i].new_symno = cnt++;
+ }
+ if (cnt == eqs->sym_cnt)
+ return;
+ eqs->sym_cnt = cnt;
+ for_each_xreg_equation(env, eqs, xreg_eq)
+ xreg_eq->sym = scratch[xreg_eq->sym].new_symno;
+ for_each_sym_equation(env, eqs, sym_eq)
+ for (i = 0; i < 3; i++)
+ if (sym_eq->coef[i])
+ sym_eq->sym[i] = scratch[sym_eq->sym[i]].new_symno;
+}
+
+/* Dead xregs are cleared */
+void bpf_equations_clean(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
+{
+ struct bpf_xreg_equation *xreg_eq, *next = st->equations.xreg_equations;
+ struct bpf_equations *eqs = &st->equations;
+ const struct scalar *s;
+
+ if (!env->eq_env)
+ return;
+ if (st->speculative)
+ eqs->xreg_equation_cnt = 0;
+ for_each_xreg_equation(env, eqs, xreg_eq) {
+ s = equation_xreg_bounds(env, st, xreg_eq);
+ if (s && !tnum_is_const(s->var_off)) {
+ *next++ = *xreg_eq;
+ continue;
+ }
+ refine_sym_from_xreg(env, st, xreg_eq);
+ env->eq_env->scratch[xreg_eq->sym].refined_sweep = 0;
+ }
+ eqs->normalized &= next == xreg_eq;
+ eqs->xreg_equation_cnt = next - eqs->xreg_equations;
+ if (!eqs->xreg_equation_cnt)
+ eqs->sym_equation_cnt = eqs->sym_cnt = 0;
+ normalize_equations(env, st);
+ eqs->clean_sym_cnt = eqs->sym_cnt;
+}
+
static void print_operand(struct bpf_verifier_log *log, const char *prefix, struct expr expr)
{
if (expr.coef)
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index db068a58c30..35efc6d86a6 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -9,6 +9,9 @@
#define VAR_NONE U64_MAX
#define NO_SYM U32_MAX
+/* For each xreg a state keeps twice that many sums and ops in total, see normalize_equations() */
+#define SUMS_PER_XREG 2
+
/* coef * var + off. A constant has no var and coef == 0. */
struct expr {
s64 coef;
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index df1866c2b55..bb8fa008a8a 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -224,6 +224,7 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
__clean_func_state(env, st->frame[i], live_regs, i);
}
+ bpf_equations_clean(env, st);
return bpf_renumber_ids(env, st);
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 20/49] bpf: Delete the equations that say nothing
2026-11-20 7:41 ` [PATCH bpf-next 20/49] bpf: Delete the equations that say nothing Alexei Starovoitov
@ 2026-11-20 8:25 ` bot+bpf-ci
0 siblings, 0 replies; 79+ messages in thread
From: bot+bpf-ci @ 2026-11-20 8:25 UTC (permalink / raw)
To: alexei.starovoitov, bpf
Cc: daniel, andrii, eddyz87, memxor, ast, andrii, daniel, martin.lau,
eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 3225 bytes --]
> bpf: Delete the equations that say nothing
TL;DR:
- Previous commit likely fails to build, using clean_sym_cnt and
bpf_equations_clean() added only here, breaking bisection
- Previous commit likely also lacks the "int i;" declaration in
add_divmod_equation()
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
> @@ -555,6 +555,7 @@ struct bpf_equations {
> bool normalized; /* see normalize_equations() */
> bool infeasible; /* the equations cannot hold: the program does not get here */
> bool needs_refine; /* widened: the equations are not applied, see bpf_flow_next() */
> + u32 clean_sym_cnt; /* after the last bpf_equations_clean() */
> };
[ ... ]
> @@ -2321,6 +2322,7 @@ void bpf_equation_env_free(struct bpf_verifier_env *env);
> int bpf_equations_copy(struct bpf_verifier_env *env, struct bpf_equations *dst,
> const struct bpf_equations *src);
> void bpf_equations_free(struct bpf_equations *eqs);
> +void bpf_equations_clean(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
> s64 bpf_equations_max_sum(struct bpf_verifier_env *env, int regno1, int regno2);
Does the previous commit in the series, 0462c546cfe6 ("bpf: Make equations
from ALU, spill, fill and call insns"), build without this field and this
prototype?
bpf_equations_before_insn() at that commit already does:
if (st->speculative || eqs->sym_cnt > 2 * eqs->clean_sym_cnt + MAX_BPF_REG)
bpf_equations_clean(env, st);
but neither clean_sym_cnt nor bpf_equations_clean() exists until this
commit, and the code is not under an #if. With CONFIG_BPF_SYSCALL=y,
kernel/bpf/equation.c should fail there with "no member named
clean_sym_cnt" and an implicit declaration of bpf_equations_clean(), which
breaks bisection.
Could the clean_sym_cnt check and the bpf_equations_clean() call be moved
into this commit, or the field, prototype and function into a commit before
0462c546cfe6?
[ ... ]
> diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
> --- a/kernel/bpf/equation.c
> +++ b/kernel/bpf/equation.c
[ ... ]
> @@ -1195,6 +1196,7 @@ static bool add_divmod_equation(struct bpf_verifier_env *env, struct expr *expr,
> struct bpf_equations *eqs = &env->cur_state->equations;
> struct bpf_sym_equation sym_eq = { { -1, expr->coef, -divisor }, { 0, expr->var },
> env->eq_env->insn_idx, expr->off };
> + int i;
>
> sym_eq.sym[0] = bpf_new_sym(env, eqs, rem);
> sym_eq.sym[2] = bpf_new_sym(env, eqs, quot);
Is this declaration of i meant to be part of the previous commit?
At 0462c546cfe6 add_divmod_equation() already has:
for (i = 0; i < 3; i++)
if (sym_eq.coef[i])
env->eq_env->scratch[sym_eq.sym[i]].refined_sweep = 0;
but i is not declared anywhere in the function, so that commit should not
compile. Moving "int i;" into 0462c546cfe6 would let every prefix of the
series build.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44698144269
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 21/49] bpf: Compare states by join_key, then by value
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (19 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 20/49] bpf: Delete the equations that say nothing Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:22 ` sashiko-bot
2026-11-20 7:41 ` [PATCH bpf-next 22/49] bpf: Continue a pass after an unproven insn Alexei Starovoitov
` (27 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The compare and the join of two states start with the same question:
can they be one state at all? Each had its own list of fields. Put the
fields in one group, the join_key:
what is held references, locks, dynptrs, iterators, irq flags
where it is the call chain, the instance of the function
which trip the peeled trip, or the leftover
The group is compared and hashed as bytes, so it has no padding.
States with two join_keys never meet. bpf_state_subsumes() looks only
at values.
regsafe() had a case for each type. All say the same: the type and the
object are equal, and the number is within. bpf_same_type_and_object()
answers the first part, also for the join.
Two rules are new. A later patch joins such values:
- Pointers to the packet are compared by the bytes that are left:
smin smax range
|-- offsets ---|-- bytes left ---|
Pointers that share an id have to keep their distance. The pairs of
ids are in env->pkt_pairs, which grows on demand.
- stacksafe() knows the bytes that a join makes. An unreadable spill
subsumes a written byte, misc subsumes a spilled number, zero
subsumes a spilled 0.
This also fixes a leak in the old code. A store of a pointer at a
variable stack offset makes a number of it. With the mitigations off
that needed no CAP_PERFMON. Reject it: a join of two offsets makes such
a store too.
This replaces refsafe(), regs_exact() and stack_arg_safe().
async_entry_cnt goes away: states of an async callback are joined.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 105 +++++++++---
kernel/bpf/states.c | 323 +++++++++++++++++++++++++----------
kernel/bpf/verifier.c | 15 +-
3 files changed, 324 insertions(+), 119 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index d4159384821..7b1cc56cd73 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -395,7 +395,7 @@ struct bpf_reference_state {
struct bpf_retval_range {
s32 minval;
s32 maxval;
- bool return_32bit;
+ u32 return_32bit; /* no padding, see bpf_same_join_key() */
};
/*
@@ -457,6 +457,8 @@ struct loop_stack_entry {
*/
struct bpf_func_state {
struct bpf_reg_state regs[MAX_BPF_REG];
+ /* equal in states that have a join, see bpf_same_join_key() */
+ struct_group(join_key,
/* index of call instruction that called into this func */
int callsite;
/* stack frame number of this function state from pov of
@@ -464,32 +466,36 @@ struct bpf_func_state {
* 0 = main function, 1 = first callee.
*/
u32 frameno;
- /*
- * Unique diagnostic identity for this function invocation. Frame depth is
- * reused after returns, while this ID is preserved across state clones.
- */
- u32 diag_frame_id;
- /* parent_id of the args that are gone when this frame returns */
- u32 frame_id;
/* subprog number == index within subprog_info
* zero == main subprog
*/
u32 subprogno;
/*
- * The frame of an async callback has the count of frame 0 of the state
- * that sets the callback plus one. Frames that differ in it are not joined.
+ * What was held at the entry of the frame and is still held. For references these are
+ * the first entry_refs of refs[]. What is released and acquired again does not count.
+ * This frame must release all the rest.
*/
- u32 async_entry_cnt;
+ u32 entry_refs;
+ u32 entry_rcu_locks;
+ u32 entry_preempt_locks;
struct bpf_retval_range callback_ret_range;
bool in_callback_fn;
bool in_async_callback_fn;
bool in_exception_callback_fn;
- bool no_stack_arg_load;
/*
* The helper invokes this callback under rcu_read_lock(), so the
* callback and its callees run in an RCU read-side critical section.
*/
bool in_rcu_callback_fn;
+ /* parent_id of the args that are gone when this frame returns */
+ u32 frame_id;
+ );
+ bool no_stack_arg_load;
+ /*
+ * Unique diagnostic identity for this function invocation. Frame depth is
+ * reused after returns, while this ID is preserved across state clones.
+ */
+ u32 diag_frame_id;
/* For callback calling functions that limit number of possible
* callback executions (e.g. bpf_loop) keeps track of current
* simulated iteration number.
@@ -508,14 +514,6 @@ struct bpf_func_state {
* subprogram (loops never cross subprogram boundaries).
*/
u32 loop_stack_cnt;
- /*
- * How much of what was held when the frame was entered is held ever since. For
- * references that is the first entry_refs of refs[]. What is released and taken again
- * doesn't count. More than that is for this frame to release.
- */
- u32 entry_refs;
- u32 entry_rcu_locks;
- u32 entry_preempt_locks;
/* The following fields should be last. See copy_func_state() */
/* The state of the stack. Each element of the array describes BPF_REG_SIZE
@@ -565,7 +563,7 @@ struct bpf_verifier_state {
/* Acquired reference states */
struct bpf_reference_state *refs;
u32 insn_idx;
- u32 curframe;
+ bool speculative;
/* an xreg has no value: unreachable. Only a path-sensitive run drops such a state. */
bool infeasible;
/* an error only in a leftover, see check_held_in_leftover() */
@@ -574,6 +572,8 @@ struct bpf_verifier_state {
/* the loops that made an xreg of data_deps imprecise, see bpf_loop.suspect_bit */
u64 suspect_loops;
+ /* equal in states that have a join, see bpf_same_join_key(). No holes. */
+ struct_group(join_key,
struct_group(held,
void *active_lock_ptr;
u32 acquired_refs;
@@ -582,9 +582,8 @@ struct bpf_verifier_state {
u32 active_rcu_locks;
u32 active_irq_id;
u32 active_lock_id;
-
- bool speculative;
);
+ u32 curframe;
bool in_sleepable;
/* a global function or an extension, and what it calls: exit of frame 0 is not the end */
bool callers_unknown;
@@ -600,6 +599,9 @@ struct bpf_verifier_state {
* state at its entry came from, see bpf_flow_call().
*/
u32 flow_entry;
+ );
+ /* Not zero in a function that is verified on its own, see bpf_flow_call() */
+ u32 instance_id;
/* The insn that the state comes from, -1 if no instructions have been executed yet. */
u32 last_insn_idx;
@@ -654,6 +656,16 @@ static inline bool bpf_stack_args_are_incoming(const struct bpf_func_state *fram
return !frame->frameno && !frame->no_stack_arg_load;
}
+/* Stack argument @i of @frame to compare with @other or NULL: incoming ones are not outgoing */
+static inline const struct bpf_reg_state *
+bpf_stack_arg(const struct bpf_func_state *frame, const struct bpf_func_state *other, u32 i)
+{
+ if (i >= frame->out_stack_arg_cnt ||
+ (bpf_stack_args_are_incoming(frame) && !bpf_stack_args_are_incoming(other)))
+ return NULL;
+ return &frame->stack_arg_regs[i];
+}
+
/* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */
#define bpf_for_each_spilled_reg(iter, frame, reg, mask) \
for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask); \
@@ -1190,6 +1202,16 @@ static inline bool bpf_id_is_xreg(u32 id)
return id & 2;
}
+/* pointers to the packet with id @old are those with id @cur moved by @delta */
+struct bpf_pkt_pair {
+ u32 old;
+ u32 cur;
+ u64 delta;
+ /* for a join */
+ u32 type;
+ u32 id;
+};
+
struct bpf_idmap {
u32 cnt;
struct bpf_id_pair map[BPF_ID_MAP_SIZE];
@@ -1289,6 +1311,7 @@ struct bpf_verifier_env {
bool path_sensitive; /* no joins, see run_path_sensitive() */
u64 cost, mem; /* see bpf_check_limits() */
u64 max_cost, max_mem;
+ u64 hash_seed; /* see bpf_hash_join_key() */
u64 pass_max_cost; /* see bpf_undo_precision() */
bool aborted; /* a thaw does not take it back */
bool no_mem; /* a table cannot grow, see bpf_check_limits() */
@@ -1363,6 +1386,8 @@ struct bpf_verifier_env {
} *callx_edges;
struct bpf_index callx_index;
struct bpf_idmap idmap_scratch;
+ struct bpf_pkt_pair *pkt_pairs; /* of the two states that are compared or joined */
+ u32 pkt_pair_cnt, pkt_pair_cap;
struct {
int *insn_state;
int *insn_stack;
@@ -1593,9 +1618,15 @@ void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
-bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
- struct bpf_verifier_state *cur);
-
+bool bpf_add_pkt_pair(struct bpf_verifier_env *env, const struct bpf_pkt_pair *pair);
+bool bpf_state_subsumes(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *cur);
+bool bpf_scalar_subsumes(const struct bpf_reg_state *old, const struct bpf_reg_state *cur);
+bool bpf_same_type_and_object(struct bpf_verifier_env *env, const struct bpf_reg_state *a,
+ const struct bpf_reg_state *b);
+bool bpf_same_join_key(const struct bpf_verifier_state *a, const struct bpf_verifier_state *b);
+u64 bpf_hash64(const void *p, size_t len, u64 hash);
+u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
/*
@@ -1696,7 +1727,7 @@ static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env,
}
/* Return IP for a given frame in a call stack */
-static inline u32 bpf_frame_insn_idx(struct bpf_verifier_state *st, u32 frame)
+static inline u32 bpf_frame_insn_idx(const struct bpf_verifier_state *st, u32 frame)
{
return frame == st->curframe
? st->insn_idx
@@ -2349,12 +2380,25 @@ static inline u8 bpf_signed_opcode(u8 op)
return op == BPF_JGT || op == BPF_JGE ? op + BPF_JSGT - BPF_JGT : op;
}
+/* True if pkt_bytes_left() is valid */
+static inline bool pkt_has_bytes_left(const struct bpf_reg_state *reg)
+{
+ return reg->range >= 0 && reg_smin(reg) >= -MAX_PACKET_OFF &&
+ reg_smax(reg) <= MAX_PACKET_OFF;
+}
+
/* How many bytes are checked from the start of the packet, wherever the pointer is */
static inline int pkt_min_len(const struct bpf_reg_state *reg)
{
return reg->r64.size ? reg->pkt_min_len : max(reg->range, reg->pkt_min_len);
}
+/* How many bytes are checked after a pointer to the packet */
+static inline s64 pkt_bytes_left(const struct bpf_reg_state *reg)
+{
+ return reg->range - reg_smax(reg);
+}
+
/* The loop that the frame is in, or NULL */
static inline struct loop_stack_entry *bpf_top_loop(const struct bpf_func_state *frame)
{
@@ -2370,4 +2414,11 @@ static inline u32 bpf_xreg_cnt(const struct bpf_verifier_state *st)
return nr;
}
+static inline bool bpf_slot_is_held(const struct bpf_stack_state *slot)
+{
+ u8 t = slot->slot_type[BPF_REG_SIZE - 1];
+
+ return t == STACK_DYNPTR || t == STACK_ITER || t == STACK_IRQ_FLAG;
+}
+
#endif /* _LINUX_BPF_VERIFIER_H */
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index bb8fa008a8a..174b1a613fb 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -4,6 +4,7 @@
#include <linux/bpf_verifier.h>
#include <linux/cnum.h>
#include <linux/filter.h>
+#include <linux/siphash.h>
/* check %cur's range satisfies %old's */
static bool range_within(const struct scalar *old, const struct scalar *cur)
@@ -39,6 +40,11 @@ bool scalar_subsumes(const struct scalar *old, const struct scalar *cur)
return range_within(old, cur) && tnum_in(old->var_off, cur->var_off);
}
+bool bpf_scalar_subsumes(const struct bpf_reg_state *old, const struct bpf_reg_state *cur)
+{
+ return scalar_subsumes(scalar_of(old), scalar_of(cur));
+}
+
static void renumber_id(struct bpf_verifier_env *env, u32 *id, u32 new_id)
{
struct bpf_idmap *map = &env->idmap_scratch;
@@ -116,6 +122,51 @@ int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st
return -E2BIG;
}
+bool bpf_add_pkt_pair(struct bpf_verifier_env *env, const struct bpf_pkt_pair *pair)
+{
+ if (!bpf_grow_array(env, &env->pkt_pairs, &env->pkt_pair_cap, env->pkt_pair_cnt + 1,
+ sizeof(*pair)))
+ return false;
+ env->pkt_pairs[env->pkt_pair_cnt++] = *pair;
+ return true;
+}
+
+/* Packet pointers that share an id in the old state share one in the new state, as far apart */
+static bool check_pkt_ids(struct bpf_verifier_env *env, const struct bpf_reg_state *rold,
+ const struct bpf_reg_state *rcur)
+{
+ struct bpf_pkt_pair pair = { rold->id, rcur->id, rcur->r64.base - rold->r64.base };
+ const struct bpf_pkt_pair *p = env->pkt_pairs;
+
+ if (!rold->id)
+ return !rcur->id;
+ bpf_add_cost(env, env->pkt_pair_cnt, BPF_COST_SCAN);
+ for (; p < env->pkt_pairs + env->pkt_pair_cnt; p++)
+ if (p->old == pair.old)
+ return p->cur == pair.cur && p->delta == pair.delta;
+ return bpf_add_pkt_pair(env, &pair);
+}
+
+/*
+ * The bytes that are checked for @rold are checked for @rcur. This matters even if the range
+ * is below the offset: the program can move the pointer back.
+ */
+static bool pkt_range_safe(const struct bpf_reg_state *rold, const struct bpf_reg_state *rcur)
+{
+ if (pkt_min_len(rold) > pkt_min_len(rcur))
+ return false;
+ /* the range tells no more than pkt_min_len() */
+ if (!rold->range || rold->range == pkt_min_len(rold))
+ return true;
+ /* AT_PKT_END or BEYOND_PKT_END */
+ if (rold->range < 0 || rcur->range < 0)
+ return rold->range == rcur->range;
+ if (pkt_has_bytes_left(rold) && pkt_has_bytes_left(rcur))
+ return pkt_bytes_left(rold) <= pkt_bytes_left(rcur);
+ return rold->range <= rcur->range && rold->r64.base == rcur->r64.base &&
+ rold->r64.size == rcur->r64.size;
+}
+
static void __clean_func_state(struct bpf_verifier_env *env,
struct bpf_func_state *st,
u16 live_regs, int frame)
@@ -228,6 +279,50 @@ int bpf_clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
return bpf_renumber_ids(env, st);
}
+/*
+ * Values of one type and object differ only in a number: the value of a scalar, the offset
+ * of a pointer.
+ *
+ * Types have to match exactly, including their modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc).
+ * A pointer is not a number: numbers are allowed to leak while pointers are not. A pointer is
+ * not its MAYBE_NULL variant: a NULL check changes all xregs with the same id.
+ */
+bool bpf_same_type_and_object(struct bpf_verifier_env *env, const struct bpf_reg_state *a,
+ const struct bpf_reg_state *b)
+{
+ if (a->type != b->type)
+ return false;
+ if (a->type == SCALAR_VALUE || a->type == NOT_INIT || type_is_pkt_pointer(a->type) ||
+ base_type(a->type) == PTR_TO_ARENA)
+ return true;
+ if (!reg_object_equal(a, b))
+ return false;
+ /* Only what an insn can make: see check_stack_access_for_ptr_arithmetic() */
+ if (bpf_reg_has_range(a) && (a->type != PTR_TO_STACK || env->bypass_spec_v1))
+ return true;
+ return scalar_equal(scalar_of(a), scalar_of(b));
+}
+
+/* Returns true if (rold safe implies rcur safe) */
+static bool regsafe(struct bpf_verifier_env *env, const struct bpf_reg_state *rold,
+ const struct bpf_reg_state *rcur)
+{
+ bpf_add_cost(env, 1, BPF_COST_XREG);
+ if (rold->type == NOT_INIT)
+ /* nothing can be done with it */
+ return true;
+ if (!rcur || !bpf_same_type_and_object(env, rold, rcur))
+ return false;
+ /* The offset is not tracked. */
+ if (base_type(rold->type) == PTR_TO_ARENA)
+ return true;
+ if (type_is_pkt_pointer(rold->type) &&
+ (!pkt_range_safe(rold, rcur) || !check_pkt_ids(env, rold, rcur)))
+ return false;
+ /* new val must satisfy old val knowledge */
+ return bpf_scalar_subsumes(rold, rcur);
+}
+
static struct bpf_reg_state unbound_reg;
static __init int unbound_reg_init(void)
@@ -272,7 +367,7 @@ static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,
}
static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
- struct bpf_func_state *cur, struct bpf_idmap *idmap)
+ struct bpf_func_state *cur)
{
int i, spi;
@@ -287,6 +382,7 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
int im = i % BPF_REG_SIZE;
u8 old_type;
+ bpf_add_cost(env, 1, BPF_COST_SCAN);
spi = i / BPF_REG_SIZE;
old_slot = bpf_stack_slot(old, spi);
old_type = old_slot->slot_type[im];
@@ -295,6 +391,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
if (old_type == STACK_INVALID || old_type == STACK_POISON)
continue;
+ /* it can be filled into a register that is unreadable */
+ if (old_type == STACK_SPILL && old_slot->spilled_ptr.type == NOT_INIT &&
+ (env->allow_uninit_stack ||
+ (cur_slot && cur_slot->slot_type[im] != STACK_INVALID &&
+ cur_slot->slot_type[im] != STACK_POISON)))
+ continue;
+
if (env->allow_uninit_stack && old_type == STACK_MISC)
continue;
@@ -320,7 +423,7 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
old_reg = scalar_reg_for_stack(env, old_slot, im);
cur_reg = scalar_reg_for_stack(env, cur_slot, im);
if (old_reg && cur_reg) {
- if (!regsafe(env, old_reg, cur_reg, idmap))
+ if (!regsafe(env, old_reg, cur_reg))
return false;
i += (im == 0 ? BPF_REG_SIZE - 1 : 3);
continue;
@@ -331,7 +434,12 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
* it will be safe with zero-initialized stack.
* The opposite is not true
*/
- if (old_type == STACK_MISC && cur_slot->slot_type[im] == STACK_ZERO)
+ if (old_type == STACK_MISC &&
+ (cur_slot->slot_type[im] == STACK_ZERO ||
+ is_spilled_scalar_after(cur_slot, im)))
+ continue;
+ if (old_type == STACK_ZERO && is_spilled_scalar_after(cur_slot, im) &&
+ bpf_register_is_null(&cur_slot->spilled_ptr))
continue;
if (old_type != cur_slot->slot_type[im])
/*
@@ -355,34 +463,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
* (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
* such verifier states are not equivalent.
*/
- if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr, idmap))
+ if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr))
return false;
break;
case STACK_DYNPTR:
- old_reg = &old_slot->spilled_ptr;
- cur_reg = &cur_slot->spilled_ptr;
- if (old_reg->dynptr.type != cur_reg->dynptr.type ||
- old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot ||
- !check_ids(old_reg->id, cur_reg->id, idmap) ||
- !check_ids(old_reg->parent_id, cur_reg->parent_id, idmap))
- return false;
- break;
case STACK_ITER:
- old_reg = &old_slot->spilled_ptr;
- cur_reg = &cur_slot->spilled_ptr;
- if (old_reg->type != cur_reg->type ||
- old_reg->iter.btf != cur_reg->iter.btf ||
- old_reg->iter.btf_id != cur_reg->iter.btf_id ||
- old_reg->iter.state != cur_reg->iter.state ||
- old_reg->iter.loops != cur_reg->iter.loops ||
- !check_ids(old_reg->id, cur_reg->id, idmap))
- return false;
- break;
case STACK_IRQ_FLAG:
- old_reg = &old_slot->spilled_ptr;
- cur_reg = &cur_slot->spilled_ptr;
- if (!check_ids(old_reg->id, cur_reg->id, idmap) ||
- old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class)
+ if (!reg_object_equal(&old_slot->spilled_ptr, &cur_slot->spilled_ptr))
return false;
break;
case STACK_MISC:
@@ -398,6 +485,114 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
return true;
}
+static const struct bpf_stack_state *held_slot(const struct bpf_func_state *f, u32 i)
+{
+ return i < bpf_stack_nr_slots(f) && bpf_slot_is_held(bpf_stack_slot(f, i)) ?
+ bpf_stack_slot(f, i) : NULL;
+}
+
+static bool same_refs(const struct bpf_verifier_state *a, const struct bpf_verifier_state *b)
+{
+ u32 i;
+
+ for (i = 0; i < a->acquired_refs; i++)
+ if (a->refs[i].type != b->refs[i].type || a->refs[i].id != b->refs[i].id ||
+ memcmp(&a->refs[i].ptr, &b->refs[i].ptr, sizeof(u64)))
+ return false;
+ return true;
+}
+
+/*
+ * The join_key of a state is what has no join:
+ * - what is held: references, locks, dynptrs, iterators, irq flags
+ * - the call chain and the instance
+ * - the peeled trip
+ */
+bool bpf_same_join_key(const struct bpf_verifier_state *a, const struct bpf_verifier_state *b)
+{
+ const struct bpf_stack_state *sa, *sb;
+ const struct loop_stack_entry *ea, *eb;
+ const struct bpf_func_state *fa, *fb;
+ u32 fr, i;
+
+ if (memcmp(&a->join_key, &b->join_key, sizeof(a->join_key)) ||
+ a->instance_id != b->instance_id || !same_refs(a, b))
+ return false;
+ for (fr = 0; fr <= a->curframe; fr++) {
+ fa = a->frame[fr];
+ fb = b->frame[fr];
+ if (memcmp(&fa->join_key, &fb->join_key, sizeof(fa->join_key)) ||
+ fa->loop_stack_cnt != fb->loop_stack_cnt)
+ return false;
+ for (i = 0; i < fa->loop_stack_cnt; i++) {
+ ea = &fa->loop_stack[i];
+ eb = &fb->loop_stack[i];
+ if (ea->loop != eb->loop || ea->peeled != eb->peeled ||
+ (ea->peeled && ea->trip != eb->trip))
+ return false;
+ }
+ for (i = 0; i < max(bpf_stack_nr_slots(fa), bpf_stack_nr_slots(fb)); i++) {
+ sa = held_slot(fa, i);
+ sb = held_slot(fb, i);
+ if (!sa != !sb || (sa && (sa->slot_type[0] != sb->slot_type[0] ||
+ !reg_object_equal(&sa->spilled_ptr,
+ &sb->spilled_ptr))))
+ return false;
+ }
+ }
+ return true;
+}
+
+u64 bpf_hash64(const void *p, size_t len, u64 hash)
+{
+ siphash_key_t key = {{ hash }};
+
+ return siphash(p, len, &key);
+}
+
+/*
+ * The hash of the join_key and of the insn. The seed is random, so a program cannot select
+ * states that collide. Only a collision can change a result.
+ */
+u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st)
+{
+ u64 hash = bpf_hash64(&st->join_key, sizeof(st->join_key), env->hash_seed +
+ ((u64)st->instance_id << 32 | st->insn_idx));
+ u32 fr, i, trip;
+ const struct bpf_stack_state *slot;
+ const struct loop_stack_entry *e;
+ const struct bpf_func_state *f;
+
+ for (i = 0; i < st->acquired_refs; i++)
+ hash = bpf_hash64(&st->refs[i].ptr, sizeof(u64), hash + st->refs[i].type);
+ for (fr = 0; fr <= st->curframe; fr++) {
+ f = st->frame[fr];
+ hash = bpf_hash64(&f->join_key, sizeof(f->join_key), hash);
+ for (i = 0, e = f->loop_stack; i < f->loop_stack_cnt; i++, e++) {
+ trip = e->peeled ? e->trip : -1;
+ hash = bpf_hash64(&trip, sizeof(trip), hash);
+ }
+ for (i = 0; i < bpf_stack_nr_slots(f); i++) {
+ slot = held_slot(f, i);
+ if (!slot)
+ continue;
+ hash = bpf_hash64(&slot->spilled_ptr,
+ offsetof(struct bpf_reg_state, var_off), hash + i);
+ }
+ }
+ return hash;
+}
+
+static bool loop_stack_safe(struct bpf_func_state *old, struct bpf_func_state *cur)
+{
+ u32 i;
+
+ for (i = 0; i < old->loop_stack_cnt; i++)
+ if (!trip_index_safe(&old->loop_stack[i], &cur->loop_stack[i]))
+ return false;
+ return true;
+}
+
/* compare two verifier states
*
* The state that gets to an insn doesn't have to be looked at when there is
@@ -418,56 +613,30 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
u16 i;
- if (old->callback_depth > cur->callback_depth)
- return false;
-
- if (!old->no_stack_arg_load && cur->no_stack_arg_load)
- return false;
-
- if (memcmp(&old->callback_ret_range, &cur->callback_ret_range,
- sizeof(old->callback_ret_range)))
- return false;
-
- if (old->in_rcu_callback_fn != cur->in_rcu_callback_fn)
- return false;
-
- /* what the frame did to what its callers hold is a part of what it tells them */
- if (old->entry_refs != cur->entry_refs ||
- old->entry_rcu_locks != cur->entry_rcu_locks ||
- old->entry_preempt_locks != cur->entry_preempt_locks)
- return false;
-
- if (!check_ids(old->frame_id, cur->frame_id, &env->idmap_scratch))
+ if (old->callback_depth > cur->callback_depth ||
+ (!old->no_stack_arg_load && cur->no_stack_arg_load) ||
+ !loop_stack_safe(old, cur))
return false;
for (i = 0; i < MAX_BPF_REG; i++)
- if (((1 << i) & live_regs) &&
- !regsafe(env, &old->regs[i], &cur->regs[i], &env->idmap_scratch))
+ if ((live_regs & BIT(i)) && !regsafe(env, &old->regs[i], &cur->regs[i]))
return false;
- if (!stacksafe(env, old, cur, &env->idmap_scratch))
- return false;
-
- if (!stack_arg_safe(env, old, cur, &env->idmap_scratch))
- return false;
-
- if (!loop_stack_safe(env, old, cur))
- return false;
+ /* outgoing stack args are a part of the state and must agree */
+ for (i = 0; i < old->out_stack_arg_cnt; i++)
+ if (!regsafe(env, &old->stack_arg_regs[i], bpf_stack_arg(cur, old, i)))
+ return false;
- return true;
+ return stacksafe(env, old, cur);
}
-/* Everything that @cur describes is described by @old. */
-bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state *old,
- struct bpf_verifier_state *cur)
+/* @old describes all that @cur describes. They have the same join_key. */
+bool bpf_state_subsumes(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *cur)
{
- u32 insn_idx;
int i;
- if (old->curframe != cur->curframe)
- return false;
-
- reset_idmap_scratch(env);
+ env->pkt_pair_cnt = 0;
/* Verification state from speculative execution simulation
* must never stand for a non-speculative execution one.
@@ -475,28 +644,10 @@ bool bpf_state_subsumes(struct bpf_verifier_env *env, struct bpf_verifier_state
if (old->speculative && !cur->speculative)
return false;
- if (old->callers_unknown != cur->callers_unknown)
- return false;
-
- if (old->in_sleepable != cur->in_sleepable)
- return false;
-
- if (old->unwound != cur->unwound || old->changed_pkt_data != cur->changed_pkt_data)
- return false;
-
- if (!refsafe(old, cur, &env->idmap_scratch))
- return false;
-
- /* for states to be equal callsites have to be the same
- * and all frame states need to be equivalent
- */
- for (i = 0; i <= old->curframe; i++) {
- insn_idx = bpf_frame_insn_idx(old, i);
- if (old->frame[i]->callsite != cur->frame[i]->callsite)
- return false;
- if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx))
+ for (i = 0; i <= old->curframe; i++)
+ if (!func_states_equal(env, old->frame[i], cur->frame[i],
+ bpf_frame_insn_idx(old, i)))
return false;
- }
return true;
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index cfcb20dda5e..01c492e0242 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -29,6 +29,7 @@
#include <linux/keyctl.h>
#include <linux/btf_ids.h>
#include <linux/poison.h>
+#include <linux/random.h>
#include <linux/module.h>
#include <linux/cpumask.h>
#include <linux/cnum.h>
@@ -1816,16 +1817,12 @@ int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_st
return err;
bpf_add_cost(env, bpf_xreg_cnt(src) + src->acquired_refs, BPF_COST_COPY);
dst_state->speculative = src->speculative;
- dst_state->in_sleepable = src->in_sleepable;
- dst_state->callers_unknown = src->callers_unknown;
- dst_state->unwound = src->unwound;
- dst_state->changed_pkt_data = src->changed_pkt_data;
dst_state->flow_entry = src->flow_entry;
dst_state->infeasible = src->infeasible;
dst_state->max_held_cnt = src->max_held_cnt;
dst_state->suspect_loops = src->suspect_loops;
+ dst_state->join_key = src->join_key;
dst_state->insn_idx = src->insn_idx;
- dst_state->curframe = src->curframe;
dst_state->last_insn_idx = src->last_insn_idx;
for (i = 0; i <= src->curframe; i++) {
dst = dst_state->frame[i];
@@ -3796,6 +3793,11 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
max_off = reg_smax(ptr_reg) + off + size;
if (value_regno >= 0)
value_reg = &cur->regs[value_regno];
+ /* no slot remembers the pointer, a load gets it as a number */
+ if (value_reg && __is_pointer_value(env->allow_ptr_leaks, value_reg)) {
+ verbose(env, "R%d leaks addr into stack\n", value_regno);
+ return -EACCES;
+ }
if ((value_reg && bpf_register_is_null(value_reg)) ||
(!value_reg && is_bpf_st_mem(insn) && insn->imm == 0))
writing_zero = true;
@@ -11418,7 +11420,6 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
if (is_async_cb_sleepable(env, insn))
env->sleepable_async_cb = true;
callee = async_cb->frame[0];
- callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1;
/* Convert bpf_timer_set_callback() args into timer callback args */
err = set_callee_state_cb(env, caller, callee, insn_idx);
@@ -24736,6 +24737,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
env->max_cost = BPF_MAX_COST >> (is_priv ? 0 : BPF_LIMITS_SHIFT_UNPRIV);
env->pass_max_cost = U64_MAX;
env->max_mem = BPF_MAX_MEM >> (is_priv ? 0 : BPF_LIMITS_SHIFT_UNPRIV);
+ env->hash_seed = get_random_u64();
env->signature = attr->signature;
/* user could have requested verbose verifier output
@@ -25083,6 +25085,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
kvfree(env->cfg.insn_postorder);
kvfree(env->succ);
kvfree(env->gotox_tmp_buf);
+ kvfree(env->pkt_pairs);
kvfree(env->callx_edges);
kvfree(env->callx_index.slots);
kvfree(env->func_ptrs);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 21/49] bpf: Compare states by join_key, then by value
2026-11-20 7:41 ` [PATCH bpf-next 21/49] bpf: Compare states by join_key, then by value Alexei Starovoitov
@ 2026-11-20 8:22 ` sashiko-bot
0 siblings, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:22 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 2 potential
issues to consider:
- [High] bpf: pointer leak via atomic RMW on unreadable spill in
stacksafe()
- [Medium] bpf: 8-byte overread of 4-byte refs[i].ptr on 32-bit in
same_refs()
--
commit e3beba5c290061bf6cb6bccd8d16009036af5591
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Compare states by join_key, then by value
This commit groups non-joinable verifier state fields into join_key structs
for bytewise comparison and hashing, and simplifies register and stack
subsumption checks in states.c.
> diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
> index bb8fa008a8a5a..174b1a613fb98 100644
> --- a/kernel/bpf/states.c
> +++ b/kernel/bpf/states.c
[ ... ]
> @@ -295,6 +391,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
> if (old_type == STACK_INVALID || old_type == STACK_POISON)
> continue;
>
> + /* it can be filled into a register that is unreadable */
> + if (old_type == STACK_SPILL && old_slot->spilled_ptr.type == NOT_INIT &&
> + (env->allow_uninit_stack ||
> + (cur_slot && cur_slot->slot_type[im] != STACK_INVALID &&
> + cur_slot->slot_type[im] != STACK_POISON)))
> + continue;
[Severity: High]
Can this leak a spilled pointer when !env->allow_ptr_leaks?
This lets an unreadable spill (STACK_SPILL with spilled_ptr.type == NOT_INIT)
in old subsume a spilled pointer in cur. If the subsuming path executes a
64-bit atomic RMW on that stack slot (such as lock *(u64 *)(r10 - 8) += r2),
check_atomic_rmw() checks the read with dst_regno == -1:
kernel/bpf/verifier.c:check_stack_read_fixed_off() {
...
} else if (dst_regno >= 0) {
/* restore register state from stack */
state->regs[dst_regno] = *reg;
bpf_equations_fill(env, reg_state, spi, size);
} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
...
return -EACCES;
}
...
}
Because is_pointer_regtype(NOT_INIT) is false, __is_pointer_value() returns
false for the unreadable spill, and the atomic write step scrubs the slot to
STACK_MISC. A subsequent 64-bit load from the slot then reads the pointer
from the subsumed path as a SCALAR_VALUE.
[ ... ]
> @@ -398,6 +485,114 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
> return true;
> }
>
> +static const struct bpf_stack_state *held_slot(const struct bpf_func_state *f, u32 i)
> +{
> + return i < bpf_stack_nr_slots(f) && bpf_slot_is_held(bpf_stack_slot(f, i)) ?
> + bpf_stack_slot(f, i) : NULL;
> +}
> +
> +static bool same_refs(const struct bpf_verifier_state *a, const struct bpf_verifier_state *b)
> +{
> + u32 i;
> +
> + for (i = 0; i < a->acquired_refs; i++)
> + if (a->refs[i].type != b->refs[i].type || a->refs[i].id != b->refs[i].id ||
> + memcmp(&a->refs[i].ptr, &b->refs[i].ptr, sizeof(u64)))
> + return false;
> + return true;
> +}
[ ... ]
> +u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st)
> +{
> + u64 hash = bpf_hash64(&st->join_key, sizeof(st->join_key), env->hash_seed +
> + ((u64)st->instance_id << 32 | st->insn_idx));
> + u32 fr, i, trip;
> + const struct bpf_stack_state *slot;
> + const struct loop_stack_entry *e;
> + const struct bpf_func_state *f;
> +
> + for (i = 0; i < st->acquired_refs; i++)
> + hash = bpf_hash64(&st->refs[i].ptr, sizeof(u64), hash + st->refs[i].type);
[Severity: Medium]
On 32-bit architectures, does passing &refs[i].ptr with sizeof(u64) to
memcmp() in same_refs() and bpf_hash64() in bpf_hash_join_key() overread the
4-byte void *ptr union member?
Although the enclosing anonymous union in struct bpf_reference_state is 8
bytes wide because of struct { int parent_id; u32 ctx_arg; }, &refs[i].ptr has
type void * const * and a 4-byte sub-object size on 32-bit targets, which
triggers -Wstringop-overread and CONFIG_FORTIFY_SOURCE read-overflow checks.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=21
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 22/49] bpf: Continue a pass after an unproven insn
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (20 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 21/49] bpf: Compare states by join_key, then by value Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 23/49] bpf: Find the xregs that an unproven insn depends on Alexei Starovoitov
` (26 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A pass stops at its first error. A program with many insns that need
precision then takes one pass for each insn. Let the pass find all of
them:
r0 = *(u64 *)(r2 + 0) unproven; r0 = unknown, from_unproven
r4 = *(u8 *)(r0 + 0) unproven because of r0: not a new error
r5 = *(u8 *)(r7 + 0) unproven: noted too
skip_unproven_insn() continues as if the insn is proven. Such a pass
accepts nothing: env->err keeps the first error, and the log ends there.
If the insn is a call that changes what is held, the path ends.
bpf_note_unproven_insn() notes what the next pass needs: the insn, the
calls that lead to it, and the suspect loops of the state. A loop is a
suspect if it made a value imprecise. The pass ends early when all
unproven insns of the last pass are unproven again.
Two errors have no state that caused them:
fixups differ The first state is gone. The insn remembers the
suspect loops of its states. If a speculative
state was first, the insn gets a barrier.
check_after_pass() bpf_find_unproven_after_pass() takes the calls
and the insns that set the depth of a stack.
Small fixes that this needs:
- A join can make an unset stack argument: -EINVAL, not -EFAULT.
- An unproven call does not mark a global function as unreliable.
- The log does not skip a slot if it starts at the main dynptr slot.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 24 +++-
kernel/bpf/Makefile | 1 +
kernel/bpf/log.c | 3 +-
kernel/bpf/precision.c | 142 ++++++++++++++++++++++++
kernel/bpf/precision.h | 39 +++++++
kernel/bpf/verifier.c | 206 +++++++++++++++++++++++++++++++----
6 files changed, 391 insertions(+), 24 deletions(-)
create mode 100644 kernel/bpf/precision.c
create mode 100644 kernel/bpf/precision.h
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 7b1cc56cd73..c26fa0c0cea 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -95,6 +95,8 @@ struct scalar {
u16 step;
/* Not a part of the value: it grew once at a loop header, see join_range() */
bool grew_once;
+ /* The value comes from an unproven insn (its check failed), see skip_unproven_insn() */
+ bool from_unproven;
};
/* The struct scalar that is an anonymous member of *@p */
@@ -617,7 +619,7 @@ static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame)
/*
* Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8).
- * The caller must ensure spi < bpf_stack_nr_slots(frame), see bpf_grow_stack_state().
+ * The caller must ensure spi < bpf_stack_nr_slots(frame), see grow_stack_state().
*/
static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi)
{
@@ -855,6 +857,7 @@ struct bpf_loop {
/* while bpf_loop_per_pass.cost is not above @budget: */
u64 budget;
u32 max_peel_cnt; /* peel up to this trip */
+ bool disjoint_apart; /* disjoint values in data_deps stay apart, also in the leftover */
);
bool partitioned; /* the last partition was for this loop */
struct bpf_loop_per_pass this_pass;
@@ -907,6 +910,7 @@ struct bpf_insn_aux_data {
u64 call_num; /* any of these, or the offset of a refcount */
};
struct btf_struct_meta *kptr_struct_meta;
+ u64 suspect_loops; /* of the states that got here */
u64 map_key_state; /* constant (32 bit) key tracking for maps */
int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
u32 seen; /* this insn was processed by the verifier at env->check_cnt */
@@ -919,7 +923,9 @@ struct bpf_insn_aux_data {
u64 cast_arena:1; /* addr_space_cast of an address in arena */
u64 cast_mov:1; /* addr_space_cast of a pointer that is not in arena, it's a move */
u64 loop_guard:1; /* may_goto is added in front of this instruction */
+ u64 speculative_first:1; /* a speculative state came to it first */
u64 call_recorded:1; /* see record_call_fixup() */
+ u64 pessimized:1; /* see bpf_partition_for_pessimized() */
u64 alu_state:5; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill
* pattern for a bpf_fastcall call.
@@ -945,6 +951,7 @@ struct bpf_insn_aux_data {
/* what one pass gives to the next pass, see bpf_flow_new_pass() */
u32 guard_no_unwind:1; /* it cannot unwind from here, at least sometimes */
u32 guard_unset_reg:1; /* where it goes a register is read that is not set here */
+ u32 non_speculative_first:1; /* speculative states end here, their fixups did not fit */
/* below flags are initialized once */
u32 bb_end:1; /* last instruction of a basic block */
@@ -1097,6 +1104,7 @@ struct bpf_subprog_info {
u32 exit_idx;
u32 callx_edge; /* 1 + the first of its env->callx_edges, 0 if none */
struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */
+ u32 deepest_insn; /* the insn that set stack_depth */
u16 stack_depth; /* max. stack depth used by this function */
u16 stack_extra;
u16 arena_stack_depth; /* size of the frame in the stack in arena */
@@ -1309,6 +1317,8 @@ struct bpf_verifier_env {
bool test_reg_invariants; /* fail verification on register invariants violations */
bool uses_arena_sp; /* the program has insns with BPF_REG_ARENA_SP */
bool path_sensitive; /* no joins, see run_path_sensitive() */
+ bool for_log; /* the verdict is known, see do_check_all() */
+ struct bpf_precision *precision;
u64 cost, mem; /* see bpf_check_limits() */
u64 max_cost, max_mem;
u64 hash_seed; /* see bpf_hash_join_key() */
@@ -1318,10 +1328,12 @@ struct bpf_verifier_env {
/* What a pass writes. Each pass starts from the snapshot. */
struct_group_tagged(bpf_env_per_pass, per_pass,
int err; /* the first error */
+ u32 err_insn;
bool imprecise; /* a state before the first error is imprecise */
bool over_limit; /* see bpf_check_limits() */
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_loop_gets_guards() */
+ bool extra_values; /* a state can have constants that no run has */
bool seen_direct_write;
/* a register may point to the stack and is not PTR_TO_STACK, see bpf_reg_state::buf */
bool seen_stack_slice;
@@ -1629,6 +1641,13 @@ u64 bpf_hash64(const void *p, size_t len, u64 hash);
u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+void bpf_note_reads_from_unproven(struct bpf_verifier_env *env, u32 idx);
+void bpf_note_unproven_insn(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st, u32 idx);
+bool bpf_insn_is_unproven(struct bpf_verifier_env *env);
+void bpf_note_fixups_differ(struct bpf_verifier_env *env);
+void bpf_find_unproven_after_pass(struct bpf_verifier_env *env);
+bool bpf_same_insns_unproven(struct bpf_verifier_env *env);
/*
* The two limits of the verifier, for the whole load. The unit of cost is one step of a scan,
* about a dozen insns of the CPU. Memory is the bytes of states and tables at a time.
@@ -1669,6 +1688,8 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env);
/* verifier.c, for flow.c */
int bpf_flow_loop_enter(struct bpf_verifier_env *env, struct bpf_loop **backedge_loop,
bool *starts_leftover);
+int bpf_flow_edge_unproven(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 idx,
+ int err);
#define BPF_EXIT_NEEDS_CALLER 1 /* verify the call in the frames of the caller */
int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
const struct bpf_verifier_state *exit, int call_idx);
@@ -1686,7 +1707,6 @@ void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
struct bpf_reg_state *reg);
void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm);
void bpf_mark_reg_unknown(struct bpf_reg_state *reg);
-int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size);
static inline int bpf_get_spi(s32 off)
{
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index d1f9ce7cc0c..7c668369231 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -13,6 +13,7 @@ obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_in
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o check_btf.o flow.o join.o
obj-$(CONFIG_BPF_SYSCALL) += equation.o equation_join.o
+obj-$(CONFIG_BPF_SYSCALL) += precision.o failing_path.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
obj-$(CONFIG_BPF_JIT) += trampoline.o
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 8a905f4df9b..dfe40d86677 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -751,7 +751,8 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
break;
case STACK_DYNPTR:
/* skip to main dynptr slot */
- i += BPF_DYNPTR_NR_SLOTS - 1;
+ if (!reg->dynptr.first_slot)
+ i += BPF_DYNPTR_NR_SLOTS - 1;
reg = &bpf_stack_slot(state, i)->spilled_ptr;
verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
new file mode 100644
index 00000000000..6aa54c656a7
--- /dev/null
+++ b/kernel/bpf/precision.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/filter.h>
+#include <linux/slab.h>
+
+#include "precision.h"
+
+/* The insn at @idx is unproven because it reads a value from another unproven insn */
+void bpf_note_reads_from_unproven(struct bpf_verifier_env *env, u32 idx)
+{
+ __set_bit(idx, env->precision->this_pass.bad_read_insns);
+ __set_bit(idx, env->precision->err_insns);
+}
+
+/* The access at @idx is unproven because of its address, whatever it loads or stores */
+static bool bad_address(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u32 idx)
+{
+ const struct bpf_func_state *f = st->frame[st->curframe];
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+ u8 class = BPF_CLASS(insn->code);
+ const struct bpf_reg_state *base;
+
+ if (insn->src_reg >= MAX_BPF_REG || insn->dst_reg >= MAX_BPF_REG ||
+ (class != BPF_LDX && class != BPF_ST &&
+ (class != BPF_STX || f->regs[insn->src_reg].type != SCALAR_VALUE)))
+ return false;
+ base = &f->regs[class == BPF_LDX ? insn->src_reg : insn->dst_reg];
+ if (base->type != PTR_TO_STACK)
+ return !tnum_is_const(base->var_off);
+ return reg_smin(base) < -(s64)env->stack_limit - insn->off ||
+ reg_smax(base) > -insn->off - bpf_size_to_bytes(BPF_SIZE(insn->code));
+}
+
+/* The insn at @idx is unproven in @st */
+void bpf_note_unproven_insn(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st, u32 idx)
+{
+ struct bpf_subprog_info *sub = bpf_find_containing_subprog(env, idx);
+ struct bpf_precision *prec = env->precision;
+ const struct loop_stack_entry *e;
+ u32 i, fr;
+
+ if (!sub)
+ return;
+ for (fr = 0; fr <= st->curframe && st->frame[fr]; fr++) {
+ e = st->frame[fr]->loop_stack;
+ for (i = 0; i < st->frame[fr]->loop_stack_cnt; i++, e++)
+ e->loop->this_pass.has_unproven |= (e->peeled || e->loop->disjoint_apart) &&
+ !(st->suspect_loops &
+ ~e->loop->suspect_bit);
+ }
+ for (i = st->curframe; i > 0 && st->frame[i]; i--)
+ __set_bit(st->frame[i]->callsite, prec->this_pass.call_insns);
+ /* the state does not know who called */
+ __set_bit(st->frame[0] ? st->frame[0]->subprogno : sub - env->subprog_info,
+ prec->this_pass.unproven_funcs);
+ /* peeling comes first: it helps only for the suspect loops of this state */
+ if (!__test_and_set_bit(idx, prec->this_pass.unproven_insns)) {
+ prec->last_unproven_left -= test_bit(idx, prec->last_unproven_insns);
+ prec->this_pass.suspect_loops |= st->suspect_loops;
+ if (idx == env->err_insn)
+ prec->this_pass.first_suspect_loops |= st->suspect_loops;
+ }
+ if (!st->frame[0] || !bad_address(env, st, idx))
+ __set_bit(idx, prec->bad_value_insns);
+ if (!st->holds_too_much)
+ __set_bit(idx, prec->err_insns);
+ for (i = BPF_REG_0; st->frame[0] && i < BPF_REG_FP; i++)
+ if ((env->insn_aux_data[idx].use_regs & BIT(i)) &&
+ !test_bit(idx, prec->bad_read_insns_done) &&
+ st->frame[st->curframe]->regs[i].type == NOT_INIT)
+ __set_bit(idx, prec->this_pass.bad_read_insns);
+}
+
+/* The state that set the fixup is gone. The leftover of a loop possibly made it up. */
+static void suspect_loops_of_insn(struct bpf_verifier_env *env, u32 idx)
+{
+ env->precision->this_pass.suspect_loops |= env->insn_aux_data[idx].suspect_loops;
+ if (!env->err)
+ env->precision->this_pass.first_suspect_loops |=
+ env->insn_aux_data[idx].suspect_loops;
+}
+
+/* The current insn has a fixup that does not fit the current state */
+void bpf_note_fixups_differ(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
+
+ suspect_loops_of_insn(env, env->insn_idx);
+ if (aux->speculative_first && !env->cur_state->speculative)
+ aux->non_speculative_first = env->precision->new_nospec = env->imprecise = true;
+}
+
+bool bpf_insn_is_unproven(struct bpf_verifier_env *env)
+{
+ return !env->path_sensitive &&
+ test_bit(env->insn_idx, env->precision->this_pass.unproven_insns);
+}
+
+static void note_unproven_after_pass(struct bpf_verifier_env *env, u32 idx)
+{
+ static const struct bpf_verifier_state no_state;
+
+ bpf_note_unproven_insn(env, &no_state, idx);
+ /* no check failed, so the failing path cannot end here */
+ __clear_bit(idx, env->precision->err_insns);
+ suspect_loops_of_insn(env, idx);
+}
+
+/* check_after_pass() found an error. Which insns caused it? */
+void bpf_find_unproven_after_pass(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct bpf_insn *insn = env->prog->insnsi;
+ u32 i;
+
+ for (i = 0; i < env->subprog_cnt; i++)
+ note_unproven_after_pass(env, env->subprog_info[i].deepest_insn);
+ for (i = 0; i < env->prog->len; i++, insn++, aux++) {
+ if (aux->pessimized) {
+ note_unproven_after_pass(env, i);
+ } else if (aux->seen && (bpf_calls_function(env, i) || is_stack_arg_st(insn) ||
+ is_stack_arg_stx(insn))) {
+ __set_bit(i, env->precision->this_pass.call_insns);
+ suspect_loops_of_insn(env, i);
+ }
+ }
+}
+
+/*
+ * All unproven insns of the last pass are unproven again, so the pass can stop. It continues if
+ * it added to linear_xregs: later trips can show that the xreg is not linear.
+ */
+bool bpf_same_insns_unproven(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ return env->for_log || (prec->last_unproven_left == 1 && !env->new_linear_xreg &&
+ test_bit(env->insn_idx, prec->last_unproven_insns) &&
+ !test_bit(env->insn_idx, prec->this_pass.unproven_insns));
+}
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
new file mode 100644
index 00000000000..23980622006
--- /dev/null
+++ b/kernel/bpf/precision.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* What precision.c and failing_path.c share */
+#ifndef __BPF_PRECISION_H
+#define __BPF_PRECISION_H
+
+#include <linux/bpf_verifier.h>
+
+/* What a pass finds about its errors */
+struct precision_per_pass {
+ /* bitmaps of insns */
+ unsigned long *unproven_insns;
+ unsigned long *call_insns; /* lead to unproven insns, or write what is held */
+ unsigned long *bad_read_insns; /* unproven because of an unreadable xreg */
+ unsigned long *unproven_funcs; /* of functions */
+ u64 suspect_loops, first_suspect_loops; /* of the unproven states, of the first one */
+};
+
+struct bpf_precision {
+ /* this pass, and the last pass whose partition was kept */
+ struct precision_per_pass this_pass, last_pass;
+ unsigned long *bad_value_insns; /* unproven, its value is possibly the cause */
+ unsigned long *last_unproven_insns; /* in the last pass */
+ unsigned long *bad_read_insns_done;
+ u32 last_unproven_left; /* last_unproven_insns whose check did not fail again */
+ bool new_nospec;
+
+ /* The path-sensitive run */
+ unsigned long *err_insns; /* insns at which the failing path can end */
+};
+
+static inline bool bpf_calls_function(struct bpf_verifier_env *env, int idx)
+{
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+
+ return bpf_pseudo_call(insn) || bpf_is_callx(insn) ||
+ env->insn_aux_data[idx].calls_callback || bpf_is_async_callback_calling_insn(insn);
+}
+
+#endif
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 01c492e0242..df3a9dc2936 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1488,7 +1488,7 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n)
/* Possibly update state->allocated_stack to be at least size bytes. Also
* possibly update the function's high-water mark in its bpf_subprog_info.
*/
-int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
+static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
{
size_t old_n = bpf_stack_nr_slots(state), n;
@@ -1506,8 +1506,10 @@ int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *st
state->allocated_stack = size;
/* update known max for given subprogram */
- if (env->subprog_info[state->subprogno].stack_depth < size)
+ if (env->subprog_info[state->subprogno].stack_depth < size) {
env->subprog_info[state->subprogno].stack_depth = size;
+ env->subprog_info[state->subprogno].deepest_insn = env->insn_idx;
+ }
return 0;
}
@@ -1840,15 +1842,12 @@ int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_st
return 0;
}
-static bool error_recoverable_with_nospec(int err)
+/*
+ * A check failed, so the insn is unproven in this state. The program is not rejected: a later
+ * pass can prove the insn. This pass continues, to find the other unproven insns.
+ */
+static bool error_is_recoverable(int err)
{
- /* Should only return true for non-fatal errors that are allowed to
- * occur during speculative verification. For these we can insert a
- * nospec and the program might still be accepted. Do not include
- * something like ENOMEM because it is likely to re-occur for the next
- * architectural path once it has been recovered-from in all speculative
- * paths.
- */
return err == -EPERM || err == -EACCES || err == -EINVAL;
}
@@ -4412,7 +4411,7 @@ static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_fu
nargs, spi + 1);
bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi,
callee_name, arg_name);
- return -EFAULT;
+ return -EINVAL;
}
}
@@ -7060,7 +7059,7 @@ static int check_stack_access_within_bounds(
/* Note that there is no stack access with offset zero, so the needed stack
* size is -min_off, not -min_off+1.
*/
- return bpf_grow_stack_state(env, state, -min_off /* size */);
+ return grow_stack_state(env, state, -min_off /* size */);
}
static bool get_func_retval_range(struct bpf_prog *prog,
@@ -7594,6 +7593,8 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
return err;
src_reg_type = regs[insn->src_reg].type;
+ /* a fill copies it from the slot, see mark_from_unproven(); dst can be src */
+ regs[insn->dst_reg].from_unproven = regs[insn->src_reg].from_unproven;
/*
* check_stack_read_fixed_off() may refine the modification's origin to
@@ -11354,9 +11355,9 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog)
err = btf_check_func_arg_match(env, subprog, btf);
/* Compiler optimizations can remove arguments from static functions
* or mismatched type can be passed into a global function.
- * In such cases mark the function as unreliable from BTF point of view.
+ * Mark a static function as unreliable from BTF point of view. The other call is unproven.
*/
- if (err)
+ if (err && !bpf_subprog_is_global(env, subprog))
prog->aux->func_info_aux[subprog].unreliable = true;
return err;
}
@@ -20702,6 +20703,33 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
return -EFAULT;
}
+/* Does the insn read what an unproven insn made up? Then what it writes is made up too. */
+static bool reads_from_unproven(struct bpf_verifier_env *env)
+{
+ struct bpf_reg_state *regs = cur_regs(env);
+ int i;
+
+ for (i = BPF_REG_0; env->err && i < BPF_REG_FP; i++)
+ if ((cur_aux(env)->use_regs & BIT(i)) && regs[i].from_unproven)
+ return true;
+ return false;
+}
+
+static void mark_from_unproven(struct bpf_verifier_env *env, int idx, bool from_unproven)
+{
+ struct bpf_reg_state *regs = cur_regs(env);
+ int i;
+
+ /* a fill has what was spilled */
+ if (!from_unproven && BPF_CLASS(env->prog->insnsi[idx].code) == BPF_LDX)
+ return;
+ for (i = BPF_REG_0; env->err && i < BPF_REG_FP; i++)
+ if (env->insn_aux_data[idx].def_regs & BIT(i))
+ regs[i].from_unproven = from_unproven;
+}
+
+static void note_error(struct bpf_verifier_env *env, int err);
+
static u32 held_cnt(const struct bpf_verifier_state *st)
{
return st->acquired_refs + st->active_rcu_locks + st->active_preempt_locks;
@@ -21240,6 +21268,21 @@ static bool is_error_of_state(struct bpf_verifier_env *env, int err)
return err != -ENOMEM && err != -EINTR && !env->over_limit && !env->bug;
}
+/* The pass has an error. The log ends with the first error. */
+static void note_error(struct bpf_verifier_env *env, int err)
+{
+ if (!is_error_of_state(env, err))
+ return;
+ if (!env->err) {
+ env->err = err;
+ env->err_insn = env->insn_idx;
+ env->log.level = 0;
+ env->pass_max_cost = min(env->pass_max_cost, bpf_pass_max_cost(env));
+ }
+ if (!env->path_sensitive)
+ bpf_note_unproven_insn(env, env->cur_state, env->insn_idx);
+}
+
/* An error in a speculative state is no error */
static bool nospec_ends_path(struct bpf_verifier_env *env,
const struct bpf_verifier_state *st, int err)
@@ -21247,6 +21290,104 @@ static bool nospec_ends_path(struct bpf_verifier_env *env,
return st->speculative && err < 0 && is_error_of_state(env, err);
}
+/* @st cannot take the edge to or from @idx. Returns 1 if only its path ends. */
+int bpf_flow_edge_unproven(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 idx,
+ int err)
+{
+ bool last;
+
+ if (nospec_ends_path(env, st, err)) {
+ need_nospec(env, idx);
+ return 1;
+ }
+ swap(env->insn_idx, idx);
+ last = !error_is_recoverable(err) || env->path_sensitive || bpf_same_insns_unproven(env);
+ swap(env->cur_state, st);
+ if (!env->path_sensitive)
+ note_error(env, err);
+ env->cur_state = st;
+ if (last)
+ return err;
+ env->insn_idx = idx;
+ return 1;
+}
+
+/* The call releases, locks, or makes a dynptr, an iterator or an irq flag */
+static bool call_writes_held(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+ const struct bpf_func_proto *fn = NULL;
+ struct bpf_kfunc_desc *desc;
+ enum bpf_arg_type type;
+ int i;
+
+ if (bpf_helper_call(insn)) {
+ bpf_get_helper_proto(env, insn->imm, &fn);
+ } else {
+ desc = find_kfunc_desc(env, insn->imm, insn->off);
+ fn = desc ? &desc->proto : NULL;
+ }
+ for (i = 0; fn && i < MAX_BPF_FUNC_REG_ARGS; i++) {
+ /* nothing is held for a pointer that an unproven insn made up */
+ if (cur_regs(env)[BPF_REG_1 + i].type == SCALAR_VALUE)
+ continue;
+ type = fn->arg_type[i];
+ if (arg_type_is_release(type) || (arg_type_is_dynptr(type) && (type & MEM_UNINIT)))
+ return true;
+ switch (base_type(type)) {
+ case ARG_PTR_TO_SPIN_LOCK:
+ case ARG_PTR_TO_RES_SPIN_LOCK:
+ case ARG_PTR_TO_IRQ_FLAG:
+ case ARG_PTR_TO_ITER:
+ case ARG_PTR_TO_LIST_NODE:
+ case ARG_PTR_TO_RB_NODE:
+ return true;
+ default:
+ break;
+ }
+ }
+ return false;
+}
+
+/*
+ * Continue as if @insn is proven, to find the other unproven insns of the pass.
+ * If it changes what is held, the next errors follow from this one, so the path ends.
+ * Returns 0 if the path continues.
+ */
+static int skip_unproven_insn(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+ struct bpf_func_state *frame = cur_func(env);
+ struct bpf_reg_state *regs = cur_regs(env);
+ u16 def = cur_aux(env)->def_regs;
+ u8 class = BPF_CLASS(insn->code), *type;
+ int i, spi;
+
+ if (bpf_insn_is_cond_jump(insn->code))
+ return PTR_ERR_OR_ZERO(push_stack(env, env->insn_idx + insn->off + 1,
+ env->insn_idx, false));
+ if (class == BPF_LD || ((class == BPF_JMP || class == BPF_JMP32) &&
+ !bpf_helper_call(insn) && !bpf_pseudo_kfunc_call(insn)))
+ return PROCESS_BPF_EXIT;
+ if (class == BPF_JMP && call_writes_held(env, insn))
+ return PROCESS_BPF_EXIT;
+ /* what it stores can be read */
+ for_each_set_bit(spi, cur_aux(env)->may_write_mask, bpf_stack_nr_slots(frame)) {
+ type = bpf_stack_slot(frame, spi)->slot_type;
+ for (i = 0; i < BPF_REG_SIZE; i++)
+ if (type[i] == STACK_INVALID || type[i] == STACK_POISON)
+ type[i] = STACK_MISC;
+ }
+ for (i = BPF_REG_0; i < BPF_REG_FP; i++) {
+ if (!(def & BIT(i)))
+ continue;
+ if (class == BPF_JMP && i)
+ bpf_mark_reg_not_init(env, ®s[i]);
+ else
+ mark_reg_unknown(env, regs, i);
+ }
+ leave_callback(frame);
+ return 0;
+}
+
static void free_pushed_states(struct bpf_verifier_env *env)
{
struct bpf_verifier_stack_elem *elem;
@@ -21264,7 +21405,7 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
struct bpf_verifier_state *state = env->cur_state;
struct bpf_insn *insns = env->prog->insnsi;
int insn_cnt = env->prog->len;
- bool do_print_state = true;
+ bool do_print_state = true, from_unproven;
for (;;) {
struct bpf_insn *insn;
@@ -21341,10 +21482,12 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
}
sanitize_mark_insn_seen(env);
+ insn_aux->suspect_loops |= state->suspect_loops;
prev_insn_idx = env->insn_idx;
/* Sanity check: precomputed constants must match verifier state */
- if (!state->speculative && insn_aux->const_reg_mask) {
+ if (!state->speculative && !env->err && !env->extra_values &&
+ insn_aux->const_reg_mask) {
struct bpf_reg_state *regs = cur_regs(env);
u16 mask = insn_aux->const_reg_mask;
@@ -21367,8 +21510,11 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
/* Reduce verification complexity by stopping speculative path
* verification when a nospec is encountered.
*/
+ if (state->speculative && insn_aux->non_speculative_first)
+ need_nospec(env, env->insn_idx);
if (state->speculative && insn_aux->nospec)
goto process_bpf_exit;
+ insn_aux->speculative_first |= !insn_aux->seen;
if (err)
return err;
@@ -21378,6 +21524,7 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
return err;
bpf_equations_before_insn(env);
+ from_unproven = reads_from_unproven(env);
fixups = insn_aux->fixups;
err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
if (nospec_ends_path(env, state, err)) {
@@ -21389,12 +21536,29 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
need_nospec(env, prev_insn_idx);
free_pushed_states(env);
goto process_bpf_exit;
+ } else if (error_is_recoverable(err) && !env->path_sensitive) {
+ env->insn_idx = prev_insn_idx;
+ if (bpf_same_insns_unproven(env))
+ return err;
+ if (from_unproven)
+ bpf_note_reads_from_unproven(env, prev_insn_idx);
+ else
+ note_error(env, err);
+ err = skip_unproven_insn(env, insn);
+ if (err < 0)
+ return err;
+ if (err)
+ goto process_bpf_exit;
+ mark_from_unproven(env, prev_insn_idx, true);
+ env->insn_idx++;
} else if (err < 0) {
return err;
} else if (err == PROCESS_BPF_EXIT) {
goto process_bpf_exit;
} else if (err == INSN_IDX_UPDATED) {
} else if (err == 0) {
+ if (!bpf_pseudo_call(insn) && !bpf_is_callx(insn))
+ mark_from_unproven(env, prev_insn_idx, from_unproven);
env->insn_idx++;
}
@@ -22667,12 +22831,12 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
ret = do_check(env);
out:
+ if (ret && !env->err)
+ note_error(env, ret);
account_current_path(env);
- if (!ret) {
- if (pop_log)
- bpf_vlog_reset(&env->log, 0);
- bpf_diag_event_log_restore(env, 0);
- }
+ if (!ret && pop_log)
+ bpf_vlog_reset(&env->log, 0);
+ bpf_diag_event_log_restore(env, 0);
free_states(env);
/*
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 23/49] bpf: Find the xregs that an unproven insn depends on
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (21 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 22/49] bpf: Continue a pass after an unproven insn Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 24/49] bpf: Keep two states apart if data and control deps differ Alexei Starovoitov
` (25 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
After an error the next pass has to keep some states apart. To keep all
apart is too costly. Find the xregs that the unproven insn depends on,
the deps:
data_deps their value flows into an operand of the insn
control_deps their value flows into a jump that can skip the insn
bpf_compute_deps() makes a backward slice. It is the analysis of
liveness with other seeds:
data control
1: r6 = ctx->idx r1 -
2: r7 = 0 r6 -
3: if r6 <= 7 goto 5 r6 r7
4: r7 = -22 r6 -
5: if r7 != 0 goto exit join point r6 r7 <- seed
6: r3 = map value r6 -
7: r3 += r6 r6 r3 -
8: r0 = *(u8 *)(r3 + 0) r3 - <- seed
It follows values through calls, returns and the frames of callers.
What a helper stores is not known. But the address and the size decide
which slots it keeps.
A store through a join of two addresses forgets the slots between them.
So may_write_mask has these slots too.
What is held is tracked like an xreg: the insns that acquire or release
write it. find_control_seeds() finds the jumps with one edge that leads
to the insn and one edge that does not.
The masks are a guess. They select where a pass does more work. A wrong
mask cannot make the verifier accept an unsafe program.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 6 +
kernel/bpf/liveness.c | 4 +-
kernel/bpf/precision.c | 539 +++++++++++++++++++++++++++++++++++
kernel/bpf/precision.h | 55 ++++
kernel/bpf/verifier.c | 7 +-
5 files changed, 608 insertions(+), 3 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index c26fa0c0cea..be3d965a3be 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -952,6 +952,8 @@ struct bpf_insn_aux_data {
u32 guard_no_unwind:1; /* it cannot unwind from here, at least sometimes */
u32 guard_unset_reg:1; /* where it goes a register is read that is not set here */
u32 non_speculative_first:1; /* speculative states end here, their fixups did not fit */
+ u32 acquires:1; /* a reference or a lock */
+ u32 releases:1;
/* below flags are initialized once */
u32 bb_end:1; /* last instruction of a basic block */
@@ -1641,6 +1643,8 @@ u64 bpf_hash64(const void *p, size_t len, u64 hash);
u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 fr, u32 xregno);
void bpf_note_reads_from_unproven(struct bpf_verifier_env *env, u32 idx);
void bpf_note_unproven_insn(struct bpf_verifier_env *env,
const struct bpf_verifier_state *st, u32 idx);
@@ -2367,6 +2371,8 @@ int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
+int bpf_insn_callee(struct bpf_verifier_env *env, int idx);
+
/* equation.c */
int bpf_equation_env_init(struct bpf_verifier_env *env);
void bpf_equation_env_free(struct bpf_verifier_env *env);
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index ebfe1603a65..c9fea08eada 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -1664,7 +1664,9 @@ static int record_stack_access_off(struct func_instance *instance, const struct
return err;
}
if (info.may_write) {
- err = mark_stack_may_write(instance, frame, insn_idx, slot_lo, slot_hi);
+ /* A join of the offsets is a range. A write forgets the slots between them. */
+ err = mark_stack_may_write(instance, frame, insn_idx, slot_lo,
+ (-arg->off[0] - 1) / STACK_SLOT_SZ);
if (err)
return err;
}
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index 6aa54c656a7..8af213dccca 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -1,4 +1,18 @@
// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Which states a pass keeps apart. Pass 0 joins all numbers, see bpf_hash_live_types(). After
+ * an error the next pass keeps more apart (trace partitioning), see do_check_all() and
+ * bpf_partition_more(). An insn whose check fails is unproven. A later pass can prove it.
+ *
+ * The deps are the xregs that the unproven insn depends on:
+ * data_deps: their value flows into an operand of the insn
+ * control_deps: their value flows into a jump that can skip the insn
+ *
+ * if (r7 == 0) goto L1; r7 is in control_deps
+ * r2 = r6; r6 is in data_deps
+ * r0 = *(u8 *)(r2 + 0); unproven insn
+ * L1:
+ */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
@@ -6,6 +20,53 @@
#include "precision.h"
+static unsigned long *mask_at(struct bpf_verifier_env *env, unsigned long *masks, u32 func, u32 idx)
+{
+ const struct bpf_precision *prec = env->precision;
+
+ return masks + prec->func[func].first +
+ (size_t)(idx - env->subprog_info[func].start) * prec->func[func].words;
+}
+
+/* @masks for frame @fr of @st. A caller needs what the callee reads and what follows the call. */
+u32 bpf_frame_mask(struct bpf_verifier_env *env, unsigned long *masks,
+ const struct bpf_verifier_state *st, u32 fr, unsigned long *out)
+{
+ u32 func = st->frame[fr]->subprogno, words = env->precision->func[func].words, i;
+ const unsigned long *m = mask_at(env, masks, func, bpf_frame_insn_idx(st, fr));
+
+ for (i = 0; i < words; i++)
+ out[i] = m[i] | (fr < st->curframe ? m[words + i] : 0);
+ return words * BITS_PER_LONG;
+}
+
+bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 fr, u32 xregno)
+{
+ unsigned long data_deps[XREG_WORDS], control_deps[XREG_WORDS];
+
+ if (!env->precision->data_deps ||
+ xregno >= bpf_frame_mask(env, env->precision->data_deps, st, fr, data_deps))
+ return false;
+ bpf_frame_mask(env, env->precision->control_deps, st, fr, control_deps);
+ return test_bit(xregno, data_deps) || test_bit(xregno, control_deps);
+}
+
+/* The function that the insn at @idx calls, or hands to a helper. -1 if that is not known. */
+int bpf_insn_callee(struct bpf_verifier_env *env, int idx)
+{
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+ u16 funcs = aux->const_reg_subprog_mask & aux->use_regs;
+
+ if (bpf_pseudo_call(insn))
+ return bpf_find_subprog(env, idx + insn->imm + 1);
+ if (hweight16(funcs) == 1 && (aux->calls_callback ||
+ bpf_is_async_callback_calling_insn(insn)))
+ return aux->const_reg_vals[__ffs(funcs)];
+ return -1;
+}
+
/* The insn at @idx is unproven because it reads a value from another unproven insn */
void bpf_note_reads_from_unproven(struct bpf_verifier_env *env, u32 idx)
{
@@ -140,3 +201,481 @@ bool bpf_same_insns_unproven(struct bpf_verifier_env *env)
test_bit(env->insn_idx, prec->last_unproven_insns) &&
!test_bit(env->insn_idx, prec->this_pass.unproven_insns));
}
+
+/* What callers need of a function: the registers that it returns, and */
+#define RET_REGS (BIT(BPF_REG_0) | BIT(BPF_REG_2))
+#define RET_HELD BIT(4) /* what is held */
+#define RET_ACQUIRES BIT(5) /* it writes that */
+#define RET_STORES BIT(6) /* what it stores to their frames */
+#define RET_LOADS BIT(7) /* it needs what it loads from there */
+
+/*
+ * What is held is tracked like an xreg, with one bit per insn in held_insns. It includes the way
+ * in which a function was left. It is in the join_key, so only the insns that write it matter.
+ */
+static bool reads_held(struct bpf_verifier_env *env, int idx)
+{
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+ u8 class = BPF_CLASS(insn->code);
+
+ if (bpf_is_unwind_goto(insn) || bpf_pseudo_call(insn) || bpf_is_callx(insn))
+ return true;
+ /* a loop header, ld_abs, a call, an exit */
+ return test_bit(idx, env->precision->last_pass.unproven_insns) &&
+ (aux->loop || class == BPF_LD ||
+ (class == BPF_JMP && !bpf_insn_is_cond_jump(insn->code)));
+}
+
+static bool has_slots(const unsigned long *mask, const unsigned long *slots, u32 bits)
+{
+ u32 i;
+
+ for (i = 0; i < BITS_TO_LONGS(bits); i++)
+ if (mask[i] & bpf_slots_to_xreg_word(slots, i))
+ return true;
+ return false;
+}
+
+/* The call at @idx needs @needs of function @t. Returns the arguments that @t needs. */
+static u16 compute_call_deps(struct bpf_verifier_env *env, unsigned long *masks, u8 *ret, int t,
+ u8 needs, bool *held, bool *changed)
+{
+ u32 start = env->subprog_info[t].start;
+
+ *changed |= needs & ~ret[t];
+ ret[t] |= needs;
+ *held |= test_bit(start, env->precision->held_insns);
+ return *mask_at(env, masks, t, start) &
+ (GENMASK(BPF_REG_5, BPF_REG_1) | BIT(XREG_STACK_ARGS));
+}
+
+/* What the insn at @idx of function @s needs before it. @in has what is needed after it. */
+static bool compute_insn_deps(struct bpf_verifier_env *env, unsigned long *masks, u8 *ret, int s,
+ int idx, bool is_seed, unsigned long *in, bool held)
+{
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+ u32 bits = env->precision->func[s].words * BITS_PER_LONG;
+ u16 use = aux->use_regs, def = aux->def_regs, args = 0;
+ bool changed = false, stores, reads, loads, call, held_after;
+ struct taken_funcs *taken = &env->precision->taken;
+ u8 class = BPF_CLASS(insn->code), needs;
+ int t, callee;
+
+ call = class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL;
+ /* the next call of the callback reads what this one stores */
+ if (aux->calls_callback) {
+ bitmap_or(in, in, mask_at(env, masks, s, idx), bits);
+ held |= test_bit(idx, env->precision->held_insns);
+ }
+ if (insn->code == (BPF_JMP | BPF_EXIT) || bpf_is_throw_kfunc(insn))
+ held = ret[s] & RET_HELD;
+ held_after = held |= is_seed && reads_held(env, idx);
+ stores = has_slots(in, aux->may_write_mask, bits) ||
+ (aux->writes_outer && (ret[s] & RET_STORES)) ||
+ ((is_stack_arg_st(insn) || is_stack_arg_stx(insn)) &&
+ test_bit(XREG_STACK_ARGS, in));
+ needs = (in[0] & def & RET_REGS) | (stores ? RET_STORES : 0) | (held ? RET_HELD : 0);
+ if ((in[0] & def & XREG_REGS) || stores)
+ __set_bit(idx, env->precision->controlled_insns);
+ /* what a helper stores is not known, whatever it reads */
+ reads = (in[0] & def & XREG_REGS) || (stores && !call);
+ loads = reads || test_bit(idx, env->precision->bad_value_insns) ||
+ !(test_bit(idx, env->precision->last_pass.unproven_insns) ||
+ test_bit(idx, env->precision->last_pass.bad_read_insns));
+ if (!loads && class == BPF_STX)
+ use = BIT(insn->dst_reg);
+ reads |= is_seed;
+ in[0] &= ~(unsigned long)(def & XREG_REGS);
+ for (t = 0; t < BITS_TO_LONGS(bits); t++)
+ in[t] &= bpf_slots_to_xreg_word(aux->live_stack_before, t) |
+ (t ? 0 : BIT(MAX_BPF_EXT_REG) - 1);
+ /* the address and the size decide if a helper keeps a slot */
+ if (call && (has_slots(in, aux->may_write_mask, bits) ||
+ (aux->writes_outer && (ret[s] & RET_STORES))))
+ in[0] |= use & (aux->stack_ptrs | aux->stack_ptrs << 1);
+
+ callee = bpf_insn_callee(env, idx);
+ if (callee > 0 && bpf_subprog_is_global(env, callee))
+ callee = -1;
+ if (callee > 0 && bpf_pseudo_call(insn)) {
+ args = compute_call_deps(env, masks, ret, callee, needs, &held, &changed);
+ use = env->insn_aux_data[env->subprog_info[callee].start].live_regs_before;
+ reads = is_seed;
+ needs = ret[callee];
+ } else if (callee > 0) {
+ reads |= compute_call_deps(env, masks, ret, callee, needs & ~RET_REGS, &held,
+ &changed);
+ needs = ret[callee];
+ } else if (bpf_is_callx(insn) || aux->calls_callback) {
+ changed |= needs & ~taken->needs;
+ taken->needs |= needs;
+ held |= taken->held;
+ args = taken->args;
+ needs = taken->ret;
+ reads |= args;
+ } else {
+ needs = 0;
+ }
+ if (insn->code == (BPF_JMP | BPF_EXIT))
+ args = ret[s] & RET_REGS;
+ if (held_after && (aux->acquires || aux->releases || (needs & RET_ACQUIRES) ||
+ bpf_is_unwind_exit(insn) || bpf_is_throw_kfunc(insn))) {
+ __set_bit(idx, env->precision->last_pass.call_insns);
+ reads |= aux->releases;
+ changed |= !(ret[s] & RET_ACQUIRES);
+ ret[s] |= RET_ACQUIRES;
+ }
+ if (held)
+ changed |= !__test_and_set_bit(idx, env->precision->held_insns);
+ in[0] |= args | (reads ? use & XREG_REGS : 0);
+ if (reads && (call || is_stack_arg_ldx(insn)))
+ __set_bit(XREG_STACK_ARGS, in);
+ if ((reads && loads) || (needs & RET_LOADS)) {
+ bpf_add_slots(in, aux->may_read_mask, bits);
+ /* the callee reads what it wrote */
+ if (needs & RET_LOADS)
+ bpf_add_slots(in, aux->may_write_mask, bits);
+ if ((aux->reads_outer || (call && aux->writes_outer)) && !(ret[s] & RET_LOADS)) {
+ ret[s] |= RET_LOADS;
+ changed = true;
+ }
+ }
+ return changed;
+}
+
+/*
+ * A backward slice: the same backward analysis as liveness, from other seeds.
+ *
+ * liveness: start from the reads of ALL insns
+ * data_deps: start from the reads of the unproven insn
+ * control_deps: start from the reads of the jumps that can skip the unproven insn
+ *
+ * data control
+ * 1: r6 = ctx->idx r1 -
+ * 2: r7 = 0 r6 -
+ * 3: if r6 <= 7 goto 5 r6 r7
+ * 4: r7 = -22 r6 -
+ * 5: r8 = 1 join point r6 r7
+ * 6: if r9 == 0 goto 8 r6 r7 r8
+ * 7: r8 = 2 r6 r7
+ * 8: if r8 == 3 goto exit join point r6 r7 r8 <- seed of control_deps: r8
+ * 9: if r7 != 0 goto exit r6 r7 <- seed of control_deps: r7
+ * 10: r2 = r6 r6 -
+ * 11: r2 <<= 3 r2 -
+ * 12: r3 = map value r2 -
+ * 13: r3 += r2 r2 r3 -
+ * 14: r0 = *(u64 *)(r3 + 0) r3 - <- seed of data_deps: r3
+ *
+ * controlled_insns gets the insns that write an xreg of @masks, for add_transitive_deps().
+ */
+int bpf_compute_deps(struct bpf_verifier_env *env, unsigned long *masks,
+ const unsigned long *seed_insns)
+{
+ struct bpf_subprog_info *sub = env->subprog_info;
+ struct bpf_precision *prec = env->precision;
+ unsigned long in[XREG_WORDS], *old;
+ int i, k, s, t, idx, err = 0;
+ struct bpf_iarray *succ;
+ bool changed = true, held;
+ u32 bits;
+ u8 *ret;
+
+ ret = kvcalloc(env->subprog_cnt, sizeof(*ret), BPF_VERIFIER_GFP);
+ if (!ret)
+ return -ENOMEM;
+ memset(masks, 0, prec->words * sizeof(long));
+ bitmap_zero(prec->held_insns, env->prog->len);
+ bitmap_zero(prec->controlled_insns, env->prog->len);
+ prec->taken = (struct taken_funcs){};
+ while (changed) {
+ changed = false;
+ bpf_add_cost(env, env->cfg.gotox_edges, BPF_COST_COPY);
+ err = bpf_add_cost_and_check(env, prec->words + env->prog->len, BPF_COST_XREG);
+ if (err)
+ break;
+ /* callees first */
+ for (k = 0; k < env->subprog_cnt; k++) {
+ s = env->subprog_topo_order[k];
+ bits = prec->func[s].words * BITS_PER_LONG;
+ for (i = sub[s].postorder_start; i <= sub[s + 1].postorder_start; i++) {
+ /* it can be dead code, see push_unwind_exit() */
+ idx = i < sub[s + 1].postorder_start ? env->cfg.insn_postorder[i] :
+ sub[s].bail_insn;
+ if (idx < 0)
+ break;
+ bitmap_zero(in, bits);
+ succ = bpf_insn_successors(env, idx);
+ held = false;
+ iarray_for_each(t, succ) {
+ bitmap_or(in, in, mask_at(env, masks, s, t), bits);
+ held |= test_bit(t, prec->held_insns);
+ }
+ t = bpf_head_guard_target(env, idx);
+ if (t >= 0)
+ bitmap_or(in, in, mask_at(env, masks, s, t), bits);
+ changed |= compute_insn_deps(env, masks, ret, s, idx,
+ test_bit(idx, seed_insns), in, held);
+ if (idx == prec->loop_carried_at) {
+ bitmap_or(in, in, prec->loop_carried, bits);
+ changed |= !(ret[s] & RET_STORES);
+ ret[s] |= RET_STORES;
+ }
+ old = mask_at(env, masks, s, idx);
+ changed |= !bitmap_subset(in, old, bits);
+ bitmap_or(old, old, in, bits);
+ }
+ if (!sub[s].addr_taken)
+ continue;
+ prec->taken.args |= compute_call_deps(env, masks, ret, s, prec->taken.needs,
+ &prec->taken.held, &changed);
+ prec->taken.ret |= ret[s];
+ }
+ }
+ kvfree(ret);
+ return err;
+}
+
+struct block {
+ u32 first, last;
+ /* @last leaves that loop to @out. It is the outermost one, 0 if none. */
+ u32 header, out;
+};
+
+/* The basic blocks of @sub in post order */
+static u32 find_blocks(struct bpf_verifier_env *env, struct bpf_subprog_info *sub, u32 *first,
+ struct block *blocks)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct block *b = blocks;
+ struct bpf_iarray *succ;
+ int i, n, s, t, h;
+
+ for (i = sub->start; i < sub[1].start; i++)
+ first[i] = bpf_flow_is_bb_start(env, i) ? i : first[i - 1];
+ for (i = sub->postorder_start; i < sub[1].postorder_start; i++) {
+ n = env->cfg.insn_postorder[i];
+ succ = bpf_insn_successors(env, n);
+ if (succ->cnt == 1 && !bpf_flow_is_bb_start(env, succ->items[0]))
+ continue;
+ *b = (struct block){ first[n], n };
+ iarray_for_each(s, succ) {
+ t = bpf_loop_at_index(env, s);
+ for (h = bpf_loop_at_index(env, n);
+ h >= 0 && h != t && !bpf_is_nested_loop(env, t, h);
+ h = aux[h].loop_header) {
+ b->header = h;
+ b->out = s;
+ }
+ }
+ b++;
+ }
+ return b - blocks;
+}
+
+static bool may_not_return(struct bpf_verifier_env *env, struct bpf_subprog_info *sub, u32 idx)
+{
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+
+ if (bpf_pseudo_call(insn))
+ return env->subprog_info[bpf_insn_callee(env, idx)].might_throw;
+ return bpf_is_callx(insn) && sub->might_throw;
+}
+
+/*
+ * The jumps that can skip one of controlled_insns: one edge leads to the insn, another does not.
+ * A back edge leads to nothing, so 'continue' skips. The edge that stays in a loop leads to all
+ * that the header leads to, so 'break' skips too. A call that throws skips too.
+ */
+static int find_control_seeds(struct bpf_verifier_env *env)
+{
+ struct bpf_subprog_info *sub = NULL;
+ struct bpf_precision *prec = env->precision;
+ u32 len = env->prog->len, idx = 0, cnt = 0, s, bit, *first;
+ struct block *blocks, *b;
+ struct bpf_iarray *succ;
+ u64 *leads, any, all;
+ int err = 0;
+
+ leads = kvmalloc_array(len, sizeof(*leads) + sizeof(*blocks) + sizeof(*first),
+ BPF_VERIFIER_GFP);
+ if (!leads)
+ return -ENOMEM;
+ blocks = (void *)(leads + len);
+ first = (void *)(blocks + len);
+ bitmap_andnot(prec->controlled_insns, prec->controlled_insns,
+ prec->controlled_insns_done, len);
+ bitmap_or(prec->controlled_insns_done, prec->controlled_insns_done,
+ prec->controlled_insns, len);
+ while (!err && (idx = find_next_bit(prec->controlled_insns, len, idx)) < len) {
+ if (!sub || idx >= sub[1].start) {
+ sub = bpf_find_containing_subprog(env, idx);
+ cnt = find_blocks(env, sub, first, blocks);
+ bpf_add_cost(env, sub[1].start - sub->start, BPF_COST_XREG);
+ }
+ for (b = blocks; b < blocks + cnt; b++)
+ leads[b->first] = 0;
+ for (bit = 0; bit < 64 && idx < sub[1].start; bit++) {
+ leads[first[idx]] = BIT_ULL(bit);
+ /* the other insns of the basic block have the same control seeds */
+ while (++idx < sub[1].start && !bpf_flow_is_bb_start(env, idx))
+ ;
+ idx = find_next_bit(prec->controlled_insns, len, idx);
+ }
+ for (b = blocks; b < blocks + cnt; b++) {
+ if (bpf_is_throw_kfunc(&env->prog->insnsi[b->last]))
+ continue;
+ succ = bpf_insn_successors(env, b->last);
+ bpf_add_cost(env, succ->cnt, BPF_COST_COPY);
+ any = 0;
+ all = U64_MAX;
+ iarray_for_each(s, succ) {
+ any |= leads[s];
+ all &= leads[s];
+ }
+ if (succ->cnt > 1 ? any != all : any && may_not_return(env, sub, b->last))
+ __set_bit(b->last, prec->control_seeds);
+ leads[b->first] |= any;
+ }
+ for (b = blocks; b < blocks + cnt; b++)
+ if (b->out && (leads[b->header] & ~leads[b->out]))
+ __set_bit(b->last, prec->control_seeds);
+ err = bpf_add_cost_and_check(env, cnt, BPF_COST_XREG);
+ }
+ kvfree(leads);
+ return err;
+}
+
+/* The calls that lead to the unproven insns */
+static int find_calls_to_unproven(struct bpf_verifier_env *env)
+{
+ struct bpf_subprog_info *sub = env->subprog_info;
+ struct precision_per_pass *last_pass = &env->precision->last_pass;
+ struct bpf_insn_aux_data *aux;
+ bool taken = false, changed = true;
+ int i, k, n, s, err = 0;
+
+ while (changed && !err) {
+ changed = false;
+ for_each_set_bit(s, last_pass->unproven_funcs, env->subprog_cnt)
+ taken |= sub[s].addr_taken;
+ /* callees first. callx and a cycle of calls need more rounds. */
+ for (k = 0; k < env->subprog_cnt; k++) {
+ n = env->subprog_topo_order[k];
+ for (i = sub[n].start; i < sub[n + 1].start; i++) {
+ aux = &env->insn_aux_data[i];
+ s = bpf_insn_callee(env, i);
+ if (!aux->seen || !bpf_calls_function(env, i) ||
+ !(s > 0 ? test_bit(s, last_pass->unproven_funcs) : taken))
+ continue;
+ __set_bit(i, last_pass->call_insns);
+ changed |= !__test_and_set_bit(n, last_pass->unproven_funcs);
+ }
+ }
+ err = bpf_add_cost_and_check(env, env->prog->len, BPF_COST_XREG);
+ }
+ return err;
+}
+
+static bool add_masks(struct bpf_precision *prec, unsigned long *to, const unsigned long *from)
+{
+ bool more = false;
+ size_t i;
+
+ for (i = 0; i < prec->words; i++) {
+ more |= from[i] & ~to[i];
+ to[i] |= from[i];
+ }
+ return more;
+}
+
+/* The control_deps of controlled_insns. Returns 1 if there is more in the deps. */
+static int add_control_deps(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ int err = find_control_seeds(env) ?:
+ bpf_compute_deps(env, prec->new_control_deps, prec->control_seeds);
+
+ if (err)
+ return err;
+ return add_masks(prec, prec->data_deps, prec->new_data_deps) |
+ add_masks(prec, prec->control_deps, prec->new_control_deps) |
+ add_masks(prec, prec->type_apart, prec->new_data_deps);
+}
+
+static int compute_deps_of_unproven(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ u32 len = env->prog->len, i;
+ int err;
+
+ err = bpf_compute_deps(env, prec->new_data_deps, prec->last_pass.unproven_insns);
+ if (err)
+ return err;
+ /* the insns that write data_deps wait for add_transitive_deps() */
+ bitmap_copy(prec->def_insns, prec->controlled_insns, len);
+ bitmap_zero(prec->controlled_insns_done, len);
+ bitmap_zero(prec->control_seeds, len);
+ bitmap_or(prec->controlled_insns, prec->last_pass.unproven_insns,
+ prec->last_pass.call_insns, len);
+ /* the helper decides how many times the callback runs, callx what runs */
+ for_each_set_bit(i, prec->last_pass.call_insns, len)
+ if (bpf_calls_function(env, i) && !bpf_pseudo_call(&env->prog->insnsi[i]))
+ __set_bit(i, prec->control_seeds);
+ return add_control_deps(env);
+}
+
+static int alloc_masks(struct bpf_verifier_env *env)
+{
+ struct bpf_subprog_info *sub = env->subprog_info;
+ DECLARE_BITMAP(live, MAX_BPF_STACK_SLOTS);
+ struct bpf_precision *prec = env->precision;
+ struct bpf_insn_aux_data *aux;
+ unsigned long *masks;
+ u32 s, i, slots;
+ int err;
+
+ if (prec->func || prec->hops_to_err)
+ return -ENOMEM;
+ prec->func = kvcalloc(env->subprog_cnt, sizeof(*prec->func), BPF_VERIFIER_GFP);
+ prec->hops_to_err = kvcalloc(5 * env->prog->len, sizeof(u16), BPF_VERIFIER_GFP);
+ if (!prec->func || !prec->hops_to_err)
+ return -ENOMEM;
+ prec->hops_to_exit = prec->hops_to_err + env->prog->len;
+ prec->apart_xregno = (void *)(prec->hops_to_exit + env->prog->len);
+ for (s = 0; s < env->subprog_cnt; s++) {
+ bitmap_zero(live, MAX_BPF_STACK_SLOTS);
+ for (i = sub[s].start; i < sub[s + 1].start; i++) {
+ aux = &env->insn_aux_data[i];
+ bitmap_or(live, live, aux->live_stack_before, MAX_BPF_STACK_SLOTS);
+ /* the callee reads what it wrote */
+ if (bpf_pseudo_call(&env->prog->insnsi[i]))
+ bitmap_or(live, live, aux->may_write_mask, MAX_BPF_STACK_SLOTS);
+ }
+ slots = find_last_bit(live, MAX_BPF_STACK_SLOTS) + 1;
+ if (slots > MAX_BPF_STACK_SLOTS)
+ slots = 0;
+ prec->func[s].first = prec->words;
+ prec->func[s].words = BITS_TO_LONGS(bpf_spi_to_xreg(slots));
+ prec->words += (sub[s + 1].start - sub[s].start + 1) * prec->func[s].words;
+ }
+ env->mem += 18 * prec->words * sizeof(long) + 5 * env->prog->len * sizeof(u16);
+ err = bpf_check_limits(env);
+ masks = err ? NULL : kvcalloc(18 * prec->words, sizeof(long), BPF_VERIFIER_GFP);
+ if (!masks) {
+ env->mem -= 18 * prec->words * sizeof(long);
+ return err ?: -ENOMEM;
+ }
+ prec->data_deps = masks;
+ prec->control_deps = masks + prec->words;
+ prec->disjoint_apart = masks + 2 * prec->words;
+ prec->type_apart = masks + 3 * prec->words;
+ prec->in_frames = masks + 4 * prec->words;
+ prec->undo = masks + 5 * prec->words;
+ prec->kept = masks + 10 * prec->words;
+ prec->new_data_deps = masks + 15 * prec->words;
+ prec->new_control_deps = masks + 16 * prec->words;
+ prec->loop_control_deps = masks + 17 * prec->words;
+ return 0;
+}
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index 23980622006..63869360160 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -5,6 +5,14 @@
#include <linux/bpf_verifier.h>
+/*
+ * The xregs of a mask, see MAX_BPF_XREG. r10 is the same in all states.
+ * Its bit stands for the arguments on the stack.
+ */
+#define XREG_STACK_ARGS BPF_REG_FP
+#define XREG_REGS GENMASK(BPF_REG_9, BPF_REG_0)
+#define XREG_WORDS BITS_TO_LONGS(MAX_BPF_XREG)
+
/* What a pass finds about its errors */
struct precision_per_pass {
/* bitmaps of insns */
@@ -15,19 +23,51 @@ struct precision_per_pass {
u64 suspect_loops, first_suspect_loops; /* of the unproven states, of the first one */
};
+/* callx calls one of the functions whose address is taken. They count as one function. */
+struct taken_funcs {
+ u16 args;
+ u8 needs, ret;
+ bool held;
+};
+
struct bpf_precision {
+ /* Masks of xregs per insn. The masks of a function are as wide as its stack is deep. */
+ struct { u32 first, words; } *func;
+ size_t words;
+ unsigned long *data_deps, *control_deps, *disjoint_apart;
+ /* data_deps, and where only type and object matter */
+ unsigned long *type_apart;
+ unsigned long *in_frames; /* see bpf_call_needs_frames() */
+ unsigned long *undo; /* these five before the last partition */
+ unsigned long *kept; /* these five, see keep_precision() */
+ /* of the unproven insns of the last pass */
+ unsigned long *new_data_deps, *new_control_deps;
+ struct taken_funcs taken; /* see bpf_compute_deps() */
/* this pass, and the last pass whose partition was kept */
struct precision_per_pass this_pass, last_pass;
+ /* bitmaps of insns */
+ unsigned long *held_insns; /* see reads_held() */
unsigned long *bad_value_insns; /* unproven, its value is possibly the cause */
unsigned long *last_unproven_insns; /* in the last pass */
unsigned long *bad_read_insns_done;
+ unsigned long *control_seeds, *controlled_insns, *controlled_insns_done, *def_insns;
u32 last_unproven_left; /* last_unproven_insns whose check did not fail again */
bool new_nospec;
+ u16 *apart_xregno; /* per insn, see bpf_range_of_data_dep() */
/* The path-sensitive run */
unsigned long *err_insns; /* insns at which the failing path can end */
+ u32 *hops_to_err, *hops_to_exit; /* per insn, see compute_hops() */
+ unsigned long *loop_control_deps; /* see bpf_jumps_see_same() */
+ u32 loop_carried_at;
+ unsigned long loop_carried[XREG_WORDS];
};
+u32 bpf_frame_mask(struct bpf_verifier_env *env, unsigned long *masks,
+ const struct bpf_verifier_state *st, u32 fr, unsigned long *out);
+int bpf_compute_deps(struct bpf_verifier_env *env, unsigned long *masks,
+ const unsigned long *seed_insns);
+
static inline bool bpf_calls_function(struct bpf_verifier_env *env, int idx)
{
struct bpf_insn *insn = &env->prog->insnsi[idx];
@@ -36,4 +76,19 @@ static inline bool bpf_calls_function(struct bpf_verifier_env *env, int idx)
env->insn_aux_data[idx].calls_callback || bpf_is_async_callback_calling_insn(insn);
}
+/* Word @i of a mask of xregs that has @slots */
+static inline unsigned long bpf_slots_to_xreg_word(const unsigned long *slots, u32 i)
+{
+ return (i < BITS_TO_LONGS(MAX_BPF_STACK_SLOTS) ? slots[i] << MAX_BPF_EXT_REG : 0) |
+ (i ? slots[i - 1] >> (BITS_PER_LONG - MAX_BPF_EXT_REG) : 0);
+}
+
+static inline void bpf_add_slots(unsigned long *mask, const unsigned long *slots, u32 bits)
+{
+ u32 i;
+
+ for (i = 0; i < BITS_TO_LONGS(bits); i++)
+ mask[i] |= bpf_slots_to_xreg_word(slots, i);
+}
+
#endif
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index df3a9dc2936..68f3127ea07 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21406,6 +21406,7 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
struct bpf_insn *insns = env->prog->insnsi;
int insn_cnt = env->prog->len;
bool do_print_state = true, from_unproven;
+ u32 held;
for (;;) {
struct bpf_insn *insn;
@@ -21524,9 +21525,12 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
return err;
bpf_equations_before_insn(env);
+ held = held_cnt(state);
from_unproven = reads_from_unproven(env);
fixups = insn_aux->fixups;
err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
+ insn_aux->releases |= held_cnt(state) < held && err != PROCESS_BPF_EXIT;
+ insn_aux->acquires |= held_cnt(state) > held;
if (nospec_ends_path(env, state, err)) {
/* Prevent this speculative path from ever reaching the
* insn that would have been unsafe to execute.
@@ -25105,13 +25109,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
- bpf_free_subprog_jts(env);
-
ret = mark_fastcall_patterns(env);
if (ret < 0)
goto skip_full_check;
ret = do_check_all(env);
+ bpf_free_subprog_jts(env);
/* while the calls are what they are in the program */
if (env->log.level & BPF_LOG_STATS)
add_insns_of_callees(env);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 24/49] bpf: Keep two states apart if data and control deps differ
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (22 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 23/49] bpf: Find the xregs that an unproven insn depends on Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 25/49] bpf: Join any two states that have the same join_key Alexei Starovoitov
` (24 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Two states meet at a join point. The join does harm if it makes a pair
of values that no path has, and the unproven insn depends on the pair:
r6 (data) r7 (control)
ok_state 0..7 0
err_state 8..max -22
join 0..max -22..0 has r6 = 8 with r7 = 0
One rule decides at all join points, bpf_must_keep_apart():
what differs action
data dep and control dep keep apart
two data deps keep apart
one data dep, with a gap keep apart (0..1 and 100)
one data dep, no gap join (0..7 and 8..9)
control deps only join
other regs only join
In pass 0 the masks are empty, so all states are joined.
At the header of a loop only a type or an object keeps states apart,
else the loop has no end. If the join is imprecise, the loop is a
suspect, also if the other state entered the loop at another time.
At the first insn of a function a difference in data_deps is sufficient.
The hash functions and bpf_range_of_data_dep() let the table of states
skip states that the rule keeps apart. They can skip more: two constants
that differ by 2 or more, or a value and an unreadable xreg.
This rule replaces all flow_*_differs() rules and their tables.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 14 ++
kernel/bpf/precision.c | 390 +++++++++++++++++++++++++++++++++++
kernel/bpf/precision.h | 15 ++
3 files changed, 419 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index be3d965a3be..cd5b99ba066 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -857,9 +857,11 @@ struct bpf_loop {
/* while bpf_loop_per_pass.cost is not above @budget: */
u64 budget;
u32 max_peel_cnt; /* peel up to this trip */
+ bool peeled_apart; /* the join rule holds for states that start a peeled trip */
bool disjoint_apart; /* disjoint values in data_deps stay apart, also in the leftover */
);
bool partitioned; /* the last partition was for this loop */
+ bool peeled_joined; /* at the start of a peeled trip, against the join rule */
struct bpf_loop_per_pass this_pass;
};
@@ -1643,6 +1645,18 @@ u64 bpf_hash64(const void *p, size_t len, u64 hash);
u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+/* precision.c, failing_path.c */
+enum bpf_join_point { BPF_JOIN_POINT, BPF_JOIN_POINT_HEADER, BPF_JOIN_POINT_CALLS };
+bool bpf_must_keep_apart(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ struct bpf_verifier_state *new, enum bpf_join_point kind,
+ const struct loop_stack_entry *entry);
+bool bpf_values_grew_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *new);
+u64 bpf_hash_apart_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u64 hash, bool consts);
+u64 bpf_hash_args(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 hash);
+void bpf_range_of_data_dep(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 *first, u32 *last);
bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 fr, u32 xregno);
void bpf_note_reads_from_unproven(struct bpf_verifier_env *env, u32 idx);
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index 8af213dccca..ad442a5d6b6 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -52,6 +52,396 @@ bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_st
return test_bit(xregno, data_deps) || test_bit(xregno, control_deps);
}
+/* The value in a slot. NULL if the slot has bytes of several types. */
+static const struct bpf_reg_state *slot_value(struct bpf_verifier_env *env,
+ const struct bpf_func_state *f, u32 spi,
+ struct bpf_reg_state *tmp)
+{
+ const struct bpf_stack_state *slot;
+ u32 i, byte_types = BIT(STACK_INVALID);
+
+ if (spi < bpf_stack_nr_slots(f)) {
+ slot = bpf_stack_slot(f, spi);
+ if (bpf_is_spilled_reg(slot) || bpf_slot_is_held(slot))
+ return &slot->spilled_ptr;
+ for (i = 0, byte_types = 0; i < BPF_REG_SIZE; i++)
+ byte_types |= BIT(slot->slot_type[i]);
+ }
+ byte_types &= ~BIT(STACK_POISON);
+ if (env->allow_uninit_stack && (byte_types & BIT(STACK_INVALID)))
+ byte_types = (byte_types & ~BIT(STACK_INVALID)) | BIT(STACK_MISC);
+ if (byte_types == BIT(STACK_ZERO))
+ bpf_mark_reg_known_scalar(tmp, 0);
+ else if (byte_types == BIT(STACK_MISC))
+ bpf_mark_reg_unknown(tmp);
+ else if (!(byte_types & ~BIT(STACK_INVALID)))
+ bpf_mark_reg_not_init(env, tmp);
+ else
+ return NULL;
+ return tmp;
+}
+
+const struct bpf_reg_state *bpf_xreg_value(struct bpf_verifier_env *env,
+ const struct bpf_func_state *f, u32 xregno,
+ struct bpf_reg_state *tmp)
+{
+ return xregno < MAX_BPF_REG ? &f->regs[xregno] :
+ slot_value(env, f, bpf_xreg_to_spi(xregno), tmp);
+}
+
+static u64 hash_type(const struct bpf_reg_state *reg, u64 hash)
+{
+ if (type_is_pkt_pointer(reg->type) || base_type(reg->type) == PTR_TO_ARENA)
+ return bpf_hash64(®->type, sizeof(reg->type), hash);
+ return bpf_hash64(reg, offsetof(typeof(*reg), var_off), hash);
+}
+
+/*
+ * States with another type or object in data_deps stay apart. Hash these like the join_key.
+ * @consts: find a state with the same constants in all deps.
+ */
+u64 bpf_hash_apart_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u64 hash, bool consts)
+{
+ unsigned long data_deps[XREG_WORDS], control_deps[XREG_WORDS];
+ const struct bpf_reg_state *reg;
+ struct bpf_reg_state tmp;
+ u32 fr, xregno, bits;
+
+ for (fr = 0; env->precision->data_deps && fr <= st->curframe; fr++) {
+ bits = bpf_frame_mask(env, env->precision->type_apart, st, fr, data_deps);
+ if (env->allow_uninit_stack)
+ bits = min(bits, bpf_spi_to_xreg(bpf_stack_nr_slots(st->frame[fr])));
+ if (consts) {
+ bpf_frame_mask(env, env->precision->control_deps, st, fr, control_deps);
+ bitmap_or(data_deps, data_deps, control_deps, bits);
+ }
+ bpf_add_cost(env, 2 * bitmap_weight(data_deps, bits), BPF_COST_XREG);
+ for_each_set_bit(xregno, data_deps, bits) {
+ reg = bpf_xreg_value(env, st->frame[fr], xregno, &tmp);
+ if (reg && consts && tnum_is_const(reg->var_off))
+ hash = bpf_hash64(®->var_off.value, sizeof(u64), hash + xregno);
+ if (reg && reg->type != SCALAR_VALUE)
+ hash = hash_type(reg, hash + xregno);
+ }
+ }
+ return hash;
+}
+
+static u64 hash_number(const struct bpf_reg_state *reg, u64 hash)
+{
+ if (reg->type == NOT_INIT || base_type(reg->type) == PTR_TO_ARENA)
+ return hash;
+ return bpf_hash64(®->r64, sizeof(reg->r64), hash);
+}
+
+/* Calls with other numbers in data_deps do not share an instance */
+u64 bpf_hash_args(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 hash)
+{
+ const struct bpf_func_state *f = st->frame[0];
+ unsigned long data_deps;
+ u32 i;
+
+ if (!env->precision->data_deps)
+ return hash;
+ data_deps = *mask_at(env, env->precision->data_deps, f->subprogno, st->insn_idx);
+ for (i = BPF_REG_1; i <= BPF_REG_5; i++)
+ if (data_deps & BIT(i))
+ hash = hash_number(&f->regs[i], hash + i);
+ for (i = 0; (data_deps & BIT(XREG_STACK_ARGS)) && i < f->out_stack_arg_cnt; i++)
+ hash = hash_number(&f->stack_arg_regs[i], hash + i);
+ return hash;
+}
+
+static u32 clamp_number(s64 x)
+{
+ return clamp_t(s64, x, S32_MIN, S32_MAX) - S32_MIN;
+}
+
+/* The range of the xreg that kept two states apart at this insn, and one step more */
+void bpf_range_of_data_dep(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 *first, u32 *last)
+{
+ u32 xregno = env->precision->data_deps ? env->precision->apart_xregno[st->insn_idx] : 0;
+ const struct bpf_reg_state *reg;
+ struct bpf_reg_state tmp;
+
+ reg = xregno ? bpf_xreg_value(env, st->frame[st->curframe], xregno - 1, &tmp) : NULL;
+ if (!reg || reg->type == NOT_INIT || base_type(reg->type) == PTR_TO_ARENA)
+ return;
+ *first = clamp_number(max_t(s64, reg_smin(reg), S32_MIN) - reg->step + 1);
+ *last = clamp_number(min_t(s64, reg_smax(reg), S32_MAX) + reg->step);
+}
+
+enum {
+ CMP_NEW = BIT(0), /* the new state has what the old one does not have */
+ CMP_OLD = BIT(1),
+ CMP_DISJOINT = BIT(2),
+ CMP_GAP = BIT(3), /* the join has what is in neither */
+ CMP_TYPE = BIT(4),
+ CMP_ALL = CMP_NEW | CMP_OLD | CMP_DISJOINT | CMP_GAP | CMP_TYPE,
+};
+
+/* 0 and 3 have no gap: the join is 0..3 step 3. Most accesses ignore the step. */
+static bool has_gap(const struct bpf_reg_state *a, const struct bpf_reg_state *b, bool in64,
+ bool in32)
+{
+ struct scalar join = *scalar_of(a);
+ u64 size;
+
+ scalar_union(&join, scalar_of(b));
+ if (a->type != SCALAR_VALUE || (a->r64.size && a->step != join.step) ||
+ (b->r64.size && b->step != join.step))
+ join.step = 1;
+ size = a->r64.size + b->r64.size + join.step;
+ return (in64 && size > a->r64.size && join.r64.size > size) ||
+ (in32 && join.r32.size > (u64)a->r32.size + b->r32.size + join.step);
+}
+
+/*
+ * If two packet pointers share an id and their distance differs, the join gives them two ids,
+ * see join_pkt_id(). Then a test of one checks no bytes for the other one.
+ * A pointer with no id in both states is a constant: it is disjoint if it differs.
+ */
+static bool pkt_distance_differs(struct bpf_verifier_env *env, const struct bpf_reg_state *a,
+ const struct bpf_reg_state *b)
+{
+ struct bpf_pkt_pair pair = { a->id, b->id, b->r64.base - a->r64.base };
+ const struct bpf_pkt_pair *p = env->pkt_pairs;
+
+ if (!type_is_pkt_pointer(a->type) || !(a->id || b->id))
+ return false;
+ bpf_add_cost(env, env->pkt_pair_cnt, BPF_COST_SCAN);
+ for (; p < env->pkt_pairs + env->pkt_pair_cnt; p++)
+ if (p->old == pair.old && p->cur == pair.cur)
+ return p->delta != pair.delta;
+ bpf_add_pkt_pair(env, &pair);
+ return false;
+}
+
+static u32 compare_values(struct bpf_verifier_env *env, const struct bpf_reg_state *a,
+ const struct bpf_reg_state *b)
+{
+ bool disjoint64, disjoint32;
+ u32 flags, pkt_flags = 0;
+ s64 left_a, left_b;
+
+ bpf_add_cost(env, 1, BPF_COST_XREG);
+ if (!bpf_same_type_and_object(env, a, b))
+ return a->type == NOT_INIT ? CMP_OLD : b->type == NOT_INIT ? CMP_NEW :
+ reg_object_equal(a, b) ? CMP_ALL & ~CMP_TYPE : CMP_ALL;
+ if (a->type == NOT_INIT || base_type(a->type) == PTR_TO_ARENA)
+ return 0;
+ if (type_is_pkt_pointer(a->type)) {
+ if ((a->id && b->id && a->id != b->id) ||
+ (a->range != b->range && min(a->range, b->range) < 0))
+ return CMP_ALL;
+ left_a = a->range ? pkt_bytes_left(a) : S32_MIN;
+ left_b = b->range ? pkt_bytes_left(b) : S32_MIN;
+ pkt_flags = left_b < left_a ? CMP_NEW : left_b > left_a ? CMP_OLD : 0;
+ }
+ if (pkt_distance_differs(env, a, b))
+ pkt_flags = CMP_NEW | CMP_OLD | CMP_GAP;
+ if (scalar_equal(scalar_of(a), scalar_of(b)))
+ return pkt_flags;
+ flags = (bpf_scalar_subsumes(a, b) ? 0 : CMP_NEW) |
+ (bpf_scalar_subsumes(b, a) ? 0 : CMP_OLD);
+ if (pkt_flags && (flags & ~pkt_flags))
+ flags |= CMP_GAP;
+ flags |= pkt_flags;
+ disjoint64 = cnum64_is_empty(cnum64_intersect(a->r64, b->r64));
+ disjoint32 = cnum32_is_empty(cnum32_intersect(a->r32, b->r32));
+ if (disjoint64 || disjoint32 ||
+ ((a->var_off.value ^ b->var_off.value) & ~(a->var_off.mask | b->var_off.mask)))
+ flags |= CMP_DISJOINT;
+ if ((disjoint64 || disjoint32) && has_gap(a, b, disjoint64, disjoint32))
+ flags |= CMP_GAP;
+ return flags;
+}
+
+/* An argument that is not there is unreadable */
+static const struct bpf_reg_state *stack_arg_value(const struct bpf_func_state *f, u32 i)
+{
+ static const struct bpf_reg_state not_init;
+
+ return i < f->out_stack_arg_cnt ? &f->stack_arg_regs[i] : ¬_init;
+}
+
+static void note_cmp(struct state_cmp *cmp, u32 flags, bool data_dep, bool value)
+{
+ if (!(value ? flags : flags & CMP_TYPE))
+ return;
+ cmp->all |= flags;
+ cmp->data_deps |= data_dep || !value ? flags : 0;
+ cmp->cnt++;
+}
+
+static u32 compare_slots(struct bpf_verifier_env *env, struct state_cmp *cmp,
+ const struct bpf_func_state *fa, const struct bpf_func_state *fb, u32 spi)
+{
+ bool same = spi < bpf_stack_nr_slots(fa) && spi < bpf_stack_nr_slots(fb) &&
+ !memcmp(bpf_stack_slot(fa, spi)->slot_type, bpf_stack_slot(fb, spi)->slot_type,
+ BPF_REG_SIZE);
+ const struct bpf_reg_state *a, *b;
+ u32 flags;
+
+ /* bytes and no value */
+ if (same && !bpf_is_spilled_reg(bpf_stack_slot(fa, spi)) &&
+ !bpf_slot_is_held(bpf_stack_slot(fa, spi)))
+ return 0;
+ a = slot_value(env, fa, spi, &cmp->tmp[0]);
+ b = slot_value(env, fb, spi, &cmp->tmp[1]);
+ if (!a || !b)
+ return same ? 0 : CMP_ALL;
+ flags = compare_values(env, a, b);
+ /* the join has the bytes and no value */
+ if (!same && ((flags & CMP_NEW) ?: a != &cmp->tmp[0]) &&
+ ((flags & CMP_OLD) ?: b != &cmp->tmp[1]))
+ return CMP_ALL;
+ return flags;
+}
+
+/* What differs in the deps, or in the masks @only. The ivs of @iv_loop have no gaps. */
+void bpf_compare_states(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *new, unsigned long *only,
+ const struct bpf_loop *iv_loop, struct state_cmp *cmp)
+{
+ unsigned long data_deps[XREG_WORDS], deps[XREG_WORDS], all[XREG_WORDS];
+ unsigned long disjoint_apart[XREG_WORDS];
+ const struct bpf_reg_state *a, *b;
+ const struct bpf_func_state *fa, *fb;
+ struct bpf_precision *prec = env->precision;
+ u32 fr, xregno, i, flags, arg_cnt, bits;
+
+ cmp->all = cmp->data_deps = cmp->disjoint_apart = cmp->cnt = cmp->disjoint_xregno = 0;
+ env->pkt_pair_cnt = 0;
+ for (fr = 0; prec->data_deps && fr <= old->curframe; fr++) {
+ fa = old->frame[fr];
+ fb = new->frame[fr];
+ bpf_frame_mask(env, prec->disjoint_apart, old, fr, disjoint_apart);
+ bpf_frame_mask(env, only ?: prec->control_deps, old, fr, deps);
+ bpf_frame_mask(env, only ?: prec->type_apart, old, fr, all);
+ bits = min(bpf_frame_mask(env, only ?: prec->data_deps, old, fr, data_deps),
+ bpf_spi_to_xreg(max(bpf_stack_nr_slots(fa), bpf_stack_nr_slots(fb))));
+ bitmap_or(all, all, deps, bits);
+ bitmap_or(deps, deps, data_deps, bits);
+ bpf_add_cost(env, 4 + bitmap_weight(all, bits), BPF_COST_COPY);
+ for_each_set_bit(xregno, all, bits) {
+ if (xregno == XREG_STACK_ARGS) {
+ arg_cnt = max(fa->out_stack_arg_cnt, fb->out_stack_arg_cnt);
+ for (i = 0; i < arg_cnt; i++) {
+ flags = compare_values(env, stack_arg_value(fa, i),
+ stack_arg_value(fb, i));
+ note_cmp(cmp, flags, test_bit(xregno, data_deps),
+ test_bit(xregno, deps));
+ if (test_bit(xregno, disjoint_apart))
+ cmp->disjoint_apart |= flags;
+ }
+ flags = 0;
+ } else if (xregno < MAX_BPF_REG) {
+ a = &fa->regs[xregno];
+ b = &fb->regs[xregno];
+ flags = compare_values(env, a, b);
+ } else {
+ flags = compare_slots(env, cmp, fa, fb, bpf_xreg_to_spi(xregno));
+ }
+ note_cmp(cmp, flags, test_bit(xregno, data_deps), test_bit(xregno, deps));
+ if (fr == old->curframe && (flags & CMP_DISJOINT) &&
+ test_bit(xregno, data_deps))
+ cmp->disjoint_xregno = cmp->disjoint_xregno ?: xregno + 1;
+ cmp->disjoint_apart |= test_bit(xregno, disjoint_apart) ? flags : 0;
+ }
+ }
+}
+
+/* See PARTITION_LOOP_DISJOINT_APART */
+static bool in_disjoint_apart_loop(const struct bpf_verifier_state *st)
+{
+ const struct bpf_loop *loop;
+ u32 fr, i;
+
+ for (fr = 0; fr <= st->curframe; fr++) {
+ for (i = 0; i < st->frame[fr]->loop_stack_cnt; i++) {
+ loop = st->frame[fr]->loop_stack[i].loop;
+ if (loop->disjoint_apart && bpf_loop_partition_is_active(loop))
+ return true;
+ }
+ }
+ return false;
+}
+
+/*
+ * The join rule. Two states at one insn have the same join_key.
+ *
+ * insn 5: r6 r7 see the example at bpf_compute_deps()
+ * ok_state 0..7 0
+ * err_state 8..max -22
+ * (data) (control) -> keep apart
+ * insn 8: r6 r7 r8
+ * ok_state 0..7 0 1
+ * ok_state 0..7 0 2
+ * (control only) -> join
+ *
+ * what differs action
+ * data dep and control dep keep apart
+ * two data deps keep apart
+ * one data dep, with a gap keep apart (0..1 and 100)
+ * one data dep, no gap join (0..7 and 8..9, 0 and 3),
+ * apart after keep_disjoint_apart()
+ * control deps only join
+ * other regs only join
+ *
+ * If one state subsumes the other in the deps, the join adds no pair.
+ *
+ * BPF_JOIN_POINT_HEADER: @new came back to the header of the loop of @entry. Only a type or
+ * an object keeps the states apart, unless the loop is partitioned. An imprecise join makes
+ * the loop a suspect.
+ * BPF_JOIN_POINT_CALLS: two calls get to the first insn of a function. The call counts as one
+ * more xreg that differs.
+ */
+bool bpf_must_keep_apart(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ struct bpf_verifier_state *new, enum bpf_join_point kind,
+ const struct loop_stack_entry *entry)
+{
+ bool keep, disjoint_apart = in_disjoint_apart_loop(new);
+ struct bpf_loop *loop;
+ struct state_cmp cmp;
+
+ /* in two entries an iv can have two values for one trip index */
+ bpf_compare_states(env, old, new, NULL, disjoint_apart && entry &&
+ kind == BPF_JOIN_POINT_HEADER ? entry->loop : NULL, &cmp);
+ if (kind == BPF_JOIN_POINT_CALLS)
+ return cmp.data_deps;
+ if ((cmp.data_deps & CMP_DISJOINT) && disjoint_apart)
+ return true;
+ keep = (cmp.data_deps & CMP_GAP) || (cmp.disjoint_apart & CMP_DISJOINT) ||
+ (cmp.data_deps && cmp.cnt > 1 && (cmp.all & CMP_NEW) && (cmp.all & CMP_OLD));
+ env->precision->this_pass.disjoint_joined |= !keep && (cmp.data_deps & CMP_DISJOINT);
+ if (keep && !env->precision->apart_xregno[new->insn_idx])
+ env->precision->apart_xregno[new->insn_idx] = cmp.disjoint_xregno;
+ if (keep && (kind == BPF_JOIN_POINT || (cmp.all & CMP_TYPE)))
+ return true;
+ if (!entry)
+ return false;
+ loop = entry->loop;
+ if (keep && entry->peeled && loop->peeled_apart && bpf_loop_partition_is_active(loop))
+ return true;
+ if (entry->peeled ? keep : cmp.data_deps & CMP_NEW)
+ new->suspect_loops |= loop->suspect_bit;
+ loop->peeled_joined |= entry->peeled && keep;
+ return false;
+}
+
+/* Does @new have more in the deps? */
+bool bpf_values_grew_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *new)
+{
+ struct state_cmp cmp;
+
+ bpf_compare_states(env, old, new, NULL, NULL, &cmp);
+ return cmp.all & CMP_NEW;
+}
+
/* The function that the insn at @idx calls, or hands to a helper. -1 if that is not known. */
int bpf_insn_callee(struct bpf_verifier_env *env, int idx)
{
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index 63869360160..e057e452ec1 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -21,6 +21,7 @@ struct precision_per_pass {
unsigned long *bad_read_insns; /* unproven because of an unreadable xreg */
unsigned long *unproven_funcs; /* of functions */
u64 suspect_loops, first_suspect_loops; /* of the unproven states, of the first one */
+ bool disjoint_joined; /* disjoint values in data_deps were joined */
};
/* callx calls one of the functions whose address is taken. They count as one function. */
@@ -55,6 +56,7 @@ struct bpf_precision {
bool new_nospec;
u16 *apart_xregno; /* per insn, see bpf_range_of_data_dep() */
+
/* The path-sensitive run */
unsigned long *err_insns; /* insns at which the failing path can end */
u32 *hops_to_err, *hops_to_exit; /* per insn, see compute_hops() */
@@ -63,8 +65,21 @@ struct bpf_precision {
unsigned long loop_carried[XREG_WORDS];
};
+struct state_cmp {
+ struct bpf_reg_state tmp[2];
+ u32 all, data_deps, cnt;
+ u32 disjoint_xregno; /* an xreg of data_deps in the top frame, plus one */
+ u32 disjoint_apart; /* what differs in these xregs */
+};
+
u32 bpf_frame_mask(struct bpf_verifier_env *env, unsigned long *masks,
const struct bpf_verifier_state *st, u32 fr, unsigned long *out);
+const struct bpf_reg_state *bpf_xreg_value(struct bpf_verifier_env *env,
+ const struct bpf_func_state *f, u32 xregno,
+ struct bpf_reg_state *tmp);
+void bpf_compare_states(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *new, unsigned long *only,
+ const struct bpf_loop *iv_loop, struct state_cmp *cmp);
int bpf_compute_deps(struct bpf_verifier_env *env, unsigned long *masks,
const unsigned long *seed_insns);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 25/49] bpf: Join any two states that have the same join_key
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (23 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 24/49] bpf: Keep two states apart if data and control deps differ Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 8:25 ` sashiko-bot
2026-11-20 7:41 ` [PATCH bpf-next 26/49] bpf: Check the equations when a state is pruned or joined Alexei Starovoitov
` (23 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The old join could fail: a pointer and a number had no join, and the
states stayed apart. So the number of states had no bound but the caps.
The new join is total:
map ptr | scalar -> unreadable -+
two objects -> unreadable -+-> a use is unproven
p+1, p+2, ... at header -> p+unknown -+ -> next pass keeps apart
If nothing uses the value, the join costs nothing. An unreadable xreg
can hold a pointer, so __is_pointer_value() is true for it.
A number can grow in each trip of a leftover. The first growth is a
join. After that the bound that moves goes to a threshold:
do { a[i] = 0; i += 3; } while (i < 30); 29
while (i < 30) { a[i] = 0; i += 3; } 32
The thresholds are the constants of the loop and of the functions that
it calls, one less, and one step more. They apply to the deps.
Other xregs stop at 0, S32_MAX, U32_MAX or S32_MIN. A threshold is a
guess: the state that comes back is compared with it.
The old join compared its result with both inputs on each call. The
new one changes dst in place. It compares the result only with src, and
only with BPF_F_TEST_REG_INVARIANTS.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 16 ++-
kernel/bpf/fixups.c | 1 +
kernel/bpf/join.c | 243 +++++++++++++++++++++++++++++++++++
kernel/bpf/loops.c | 107 +++++++++++++++
kernel/bpf/verifier.c | 59 ++++-----
5 files changed, 386 insertions(+), 40 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index cd5b99ba066..1c04bc2d5e4 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -863,6 +863,10 @@ struct bpf_loop {
bool partitioned; /* the last partition was for this loop */
bool peeled_joined; /* at the start of a peeled trip, against the join rule */
struct bpf_loop_per_pass this_pass;
+ /* sorted constants of the loop, the largest and the smallest constant that an insn adds */
+ s64 *thresholds;
+ u32 threshold_cnt, threshold_cap;
+ s32 max_step, min_step;
};
static inline bool bpf_loop_has_budget(const struct bpf_loop *loop)
@@ -1645,6 +1649,11 @@ u64 bpf_hash64(const void *p, size_t len, u64 hash);
u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
int bpf_renumber_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+/* join.c */
+int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen);
+
/* precision.c, failing_path.c */
enum bpf_join_point { BPF_JOIN_POINT, BPF_JOIN_POINT_HEADER, BPF_JOIN_POINT_CALLS };
bool bpf_must_keep_apart(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
@@ -2242,8 +2251,8 @@ void scalar_add(struct scalar *dst, const struct scalar *a, const struct scalar
void scalar_alu(struct scalar *dst, const struct scalar *src, const struct bpf_insn *insn);
bool scalar_intersect(struct scalar *dst, const struct scalar *src, u32 bits);
void scalar_union(struct scalar *acc, const struct scalar *src);
-void scalar_widen(struct scalar *s, const struct scalar *old,
- const struct bpf_loop_rungs *rungs, int linear);
+void scalar_widen(struct scalar *s, const struct scalar *old, s64 min_threshold, s64 max_threshold);
+
int bpf_stack_liveness_init(struct bpf_verifier_env *env);
void bpf_stack_liveness_free(struct bpf_verifier_env *env);
int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
@@ -2384,7 +2393,8 @@ int bpf_compute_idoms(struct bpf_verifier_env *env);
int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
-
+u32 bpf_loop_thresholds_below(const struct bpf_loop *loop, s64 k);
+int bpf_loop_collect_thresholds(struct bpf_verifier_env *env, struct bpf_loop *loop);
int bpf_insn_callee(struct bpf_verifier_env *env, int idx);
/* equation.c */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 751b25c92c9..17da785f8fa 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -758,6 +758,7 @@ static void free_loop(struct bpf_loop *loop)
{
if (!loop)
return;
+ kvfree(loop->thresholds);
kvfree(loop);
}
diff --git a/kernel/bpf/join.c b/kernel/bpf/join.c
index 95de080617b..b1d42ad23ca 100644
--- a/kernel/bpf/join.c
+++ b/kernel/bpf/join.c
@@ -1,6 +1,249 @@
// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Join of verifier states.
+ *
+ * Two states at one insn with the same join_key always have a join, see bpf_same_join_key().
+ *
+ * map ptr | scalar -> unreadable -+
+ * two objects -> unreadable -+-> a use is unproven -> partition
+ * p+1, p+2, ... at header -> p+unknown -+
+ *
+ * Values of one type and object differ only in a number, see bpf_same_type_and_object().
+ * Numbers are joined. At a loop header they are widened, so the sequence of states ends.
+ *
+ * bpf_clean_verifier_state() cleans the inputs: dead xregs are the same in both.
+ */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/cnum.h>
#include <linux/tnum.h>
#include <linux/filter.h>
+
+struct join {
+ struct bpf_verifier_env *env;
+ struct bpf_verifier_state *dst;
+ struct bpf_loop *leftover_loop; /* the states are at its header, in its leftover */
+ bool widen;
+ u32 fr;
+ u32 xregno; /* see MAX_BPF_XREG, BPF_REG_FP for stack args */
+ int err;
+};
+
+/*
+ * A number can grow in each trip. The bound that moved goes to a threshold. The constants of
+ * the loop give the thresholds:
+ * do { a[i] = 0; i += 3; } while (i < 30); 29 at most comes back to the header
+ * while (i < 30) { a[i] = 0; i += 3; } 32
+ * loop { if (...) { if (n >= 64) break; a[n++] = x; } } 64
+ * A threshold is a guess. The state that comes back to the header is compared with it.
+ *
+ * Only a number on a cycle grows again and again. So the first growth is a join.
+ * x = 0; loop { a[x]; x = rnd & 7; }
+ */
+static void join_range(struct join *j, struct bpf_reg_state *dst, const struct bpf_reg_state *src)
+{
+ s64 min_threshold = S64_MIN, max_threshold = S64_MAX, off;
+ struct bpf_loop *loop = j->leftover_loop;
+ struct bpf_reg_state old = *dst;
+ bool in_deps;
+ int i, n;
+
+ if (scalar_equal(scalar_of(dst), scalar_of(src)))
+ return;
+ scalar_union(scalar_of(dst), scalar_of(src));
+ if (loop && !scalar_equal(scalar_of(dst), scalar_of(&old))) {
+ if (!j->widen || !old.grew_once)
+ goto out;
+ /* two pointers are compared by their offsets up to there */
+ if (dst->type != SCALAR_VALUE) {
+ if (reg_smax(dst) < BPF_MAX_VAR_OFF)
+ max_threshold = BPF_MAX_VAR_OFF - 1;
+ if (reg_smin(dst) > -BPF_MAX_VAR_OFF)
+ min_threshold = -BPF_MAX_VAR_OFF + 1;
+ }
+ in_deps = bpf_xreg_in_deps(j->env, j->dst, j->fr, j->xregno);
+ if (in_deps)
+ j->err = j->err ?: bpf_loop_collect_thresholds(j->env, loop);
+ /* at the constant or one before it, and the largest step or this step further */
+ for (i = 0; i < 6 && in_deps && !j->err; i++) {
+ off = i & 4 ? reg_smax(dst) - reg_smax(&old) : i & 2 ? loop->max_step : 0;
+ off -= i & 1;
+ n = bpf_loop_thresholds_below(loop, reg_smax(dst) - off);
+ if (n < loop->threshold_cnt && loop->thresholds[n] + off >= reg_smax(dst))
+ max_threshold = min(max_threshold, loop->thresholds[n] + off);
+ off = i & 4 ? reg_smin(dst) - reg_smin(&old) : i & 2 ? loop->min_step : 0;
+ off += i & 1;
+ n = bpf_loop_thresholds_below(loop, reg_smin(dst) - off + 1);
+ if (n && loop->thresholds[n - 1] + off <= reg_smin(dst))
+ min_threshold = max(min_threshold, loop->thresholds[n - 1] + off);
+ }
+ scalar_widen(scalar_of(dst), scalar_of(&old), min_threshold, max_threshold);
+out:
+ dst->grew_once = true;
+ }
+ if (!bpf_scalar_subsumes(dst, &old) || !bpf_scalar_subsumes(dst, src))
+ scalar_set_unknown(scalar_of(dst));
+}
+
+/*
+ * Pointers to the packet that share an id keep their displacement. No id is an id too:
+ * the start of the packet plus a constant, see find_good_pkt_pointers().
+ */
+static u32 join_pkt_id(struct bpf_verifier_env *env, const struct bpf_reg_state *ra,
+ const struct bpf_reg_state *rb)
+{
+ struct bpf_pkt_pair new = { ra->id, rb->id, rb->r64.base - ra->r64.base, ra->type, ra->id };
+ struct bpf_pkt_pair *p = env->pkt_pairs;
+ bool taken = !ra->id;
+
+ bpf_add_cost(env, env->pkt_pair_cnt, BPF_COST_SCAN);
+ for (; p < env->pkt_pairs + env->pkt_pair_cnt; taken |= p++->id == ra->id)
+ if (!memcmp(p, &new, offsetof(struct bpf_pkt_pair, id)))
+ return p->id;
+ if (taken && (new.old || new.cur || new.delta))
+ new.id = bpf_new_id(env);
+ /* with no memory the load fails, see bpf_check_limits() */
+ bpf_add_pkt_pair(env, &new);
+ return new.id;
+}
+
+/* The bytes that are left after the pointer in both states are left in the join */
+static void join_pkt(struct join *j, struct bpf_reg_state *dst, const struct bpf_reg_state *src)
+{
+ struct bpf_reg_state old = *dst;
+ struct cnum64 both = cnum64_union(old.r64, src->r64);
+
+ dst->id = join_pkt_id(j->env, &old, src);
+ join_range(j, dst, src);
+ dst->pkt_min_len = min(pkt_min_len(&old), pkt_min_len(src));
+ if (old.r64.size == dst->r64.size && src->r64.size == dst->r64.size) {
+ if (old.range != src->range)
+ dst->range = max(min(old.range, src->range), 0);
+ return;
+ }
+ if (pkt_has_bytes_left(&old) && pkt_has_bytes_left(src) && pkt_has_bytes_left(dst))
+ dst->range = reg_smax(dst) + min(pkt_bytes_left(&old), pkt_bytes_left(src));
+ else
+ dst->range = 0;
+ dst->range = max(dst->range, dst->pkt_min_len);
+ /* pointers that share an id have ranges of the same size */
+ if (cnum64_umax(dst->r64) > MAX_PACKET_OFF || dst->r64.size != both.size)
+ dst->id = bpf_new_id(j->env);
+}
+
+static void join_reg(struct join *j, struct bpf_reg_state *dst, const struct bpf_reg_state *src)
+{
+ bool from_unproven = dst->from_unproven || (src && src->from_unproven);
+
+ if (!src || dst->type == NOT_INIT || !bpf_same_type_and_object(j->env, dst, src))
+ bpf_mark_reg_not_init(j->env, dst);
+ else if (type_is_pkt_pointer(dst->type))
+ join_pkt(j, dst, src);
+ else if (base_type(dst->type) != PTR_TO_ARENA) /* its offset is not tracked */
+ join_range(j, dst, src);
+ dst->from_unproven = from_unproven;
+}
+
+/* What a byte of a slot is if the number in the slot is not known */
+static u8 join_byte(struct bpf_stack_state *s, int i)
+{
+ if (s->slot_type[i] != STACK_SPILL)
+ return s->slot_type[i];
+ return bpf_register_is_null(&s->spilled_ptr) ? STACK_ZERO : STACK_MISC;
+}
+
+static bool byte_is_set(u8 t)
+{
+ return t != STACK_INVALID && t != STACK_POISON;
+}
+
+static void join_slot(struct join *j, struct bpf_stack_state *dst, struct bpf_stack_state *src)
+{
+ u8 x, y, ptr_type = STACK_SPILL;
+ bool ptr;
+ int i;
+
+ /* it is in the join_key */
+ if (bpf_slot_is_held(dst))
+ return;
+ if (!memcmp(dst->slot_type, src->slot_type, sizeof(dst->slot_type))) {
+ if (bpf_is_spilled_reg(dst))
+ join_reg(j, &dst->spilled_ptr, &src->spilled_ptr);
+ return;
+ }
+ ptr = (bpf_is_spilled_reg(dst) && !bpf_is_spilled_scalar_reg(dst)) ||
+ (bpf_is_spilled_reg(src) && !bpf_is_spilled_scalar_reg(src));
+ /* a byte that is possibly not written is not read, not even into an unreadable register */
+ for (i = 0; i < BPF_REG_SIZE && !j->env->allow_uninit_stack; i++)
+ if (!byte_is_set(dst->slot_type[i]) || !byte_is_set(src->slot_type[i]))
+ ptr_type = STACK_INVALID;
+ for (i = 0; i < BPF_REG_SIZE; i++) {
+ x = join_byte(dst, i);
+ y = join_byte(src, i);
+ dst->slot_type[i] = ptr ? ptr_type : !byte_is_set(x) || x == y ? x :
+ !byte_is_set(y) ? STACK_INVALID : STACK_MISC;
+ }
+ bpf_mark_reg_not_init(j->env, &dst->spilled_ptr);
+}
+
+static void join_func(struct join *j, struct bpf_func_state *dst, struct bpf_func_state *src,
+ u16 live_regs)
+{
+ static struct bpf_stack_state invalid_slot;
+ const struct loop_stack_entry *s;
+ struct loop_stack_entry *d;
+ u32 i;
+
+ for (i = 0; i < dst->loop_stack_cnt; i++) {
+ d = &dst->loop_stack[i];
+ s = &src->loop_stack[i];
+ d->trip = max(d->trip, s->trip);
+ d->held_cnt = min(d->held_cnt, s->held_cnt);
+ }
+ for (i = 0; i < BPF_REG_FP; i++) {
+ j->xregno = i;
+ if (live_regs & BIT(i))
+ join_reg(j, &dst->regs[i], &src->regs[i]);
+ else
+ bpf_mark_reg_not_init(j->env, &dst->regs[i]);
+ }
+ /* slots and arguments that only @src has are unreadable */
+ for (i = 0; i < bpf_stack_nr_slots(dst); i++) {
+ j->xregno = bpf_spi_to_xreg(i);
+ join_slot(j, bpf_stack_slot(dst, i),
+ i < bpf_stack_nr_slots(src) ? bpf_stack_slot(src, i) : &invalid_slot);
+ }
+ j->xregno = BPF_REG_FP;
+ for (i = 0; i < dst->out_stack_arg_cnt; i++)
+ join_reg(j, &dst->stack_arg_regs[i],
+ bpf_stack_arg(dst, src, i) ? bpf_stack_arg(src, dst, i) : NULL);
+ dst->no_stack_arg_load |= src->no_stack_arg_load;
+ dst->callback_depth = min(dst->callback_depth, src->callback_depth);
+}
+
+/*
+ * Make @dst describe all that @dst or @src describe. They have the same join_key.
+ * Both are speculative, or neither is.
+ * @leftover_loop: they are at its header, in its leftover.
+ */
+int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen)
+{
+ struct join j = { .env = env, .dst = dst, .leftover_loop = leftover_loop, .widen = widen };
+
+ bpf_add_cost(env, bpf_xreg_cnt(dst), BPF_COST_XREG);
+ env->pkt_pair_cnt = 0;
+ env->imprecise |= !env->err;
+ dst->suspect_loops |= src->suspect_loops;
+ for (j.fr = 0; j.fr <= dst->curframe; j.fr++)
+ join_func(&j, dst->frame[j.fr], src->frame[j.fr],
+ env->insn_aux_data[bpf_frame_insn_idx(dst, j.fr)].live_regs_before);
+ dst->max_held_cnt = max(dst->max_held_cnt, src->max_held_cnt);
+ /* no equation is found over a limit */
+ if (env->test_reg_invariants && !bpf_state_subsumes(env, dst, src) &&
+ verifier_bug_if(!bpf_check_limits(env), env, "join at %d", dst->insn_idx))
+ return -EFAULT;
+ /* the first xreg that had an id can be gone */
+ return j.err ?: bpf_renumber_ids(env, dst);
+}
diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
index f8b2f8befa6..23c443ab3a2 100644
--- a/kernel/bpf/loops.c
+++ b/kernel/bpf/loops.c
@@ -2,6 +2,7 @@
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/slab.h>
+#include <linux/sort.h>
#include <linux/bpf_verifier.h>
static bool is_cfg_jump(struct bpf_insn *insn)
@@ -555,3 +556,109 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
kvfree(dfs.state);
return err;
}
+
+static int cmp_thresholds(const void *a, const void *b)
+{
+ return *(s64 *)a < *(s64 *)b ? -1 : *(s64 *)a > *(s64 *)b;
+}
+
+static int add_threshold(struct bpf_verifier_env *env, struct bpf_loop *loop, s64 k)
+{
+ if (!bpf_grow_array(env, &loop->thresholds, &loop->threshold_cap, loop->threshold_cnt + 1,
+ sizeof(k)))
+ return -ENOMEM;
+ loop->thresholds[loop->threshold_cnt++] = k;
+ return 0;
+}
+
+static int add_insn_thresholds(struct bpf_verifier_env *env, struct bpf_loop *loop, int idx)
+{
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+ u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
+ bool alu = class == BPF_ALU || class == BPF_ALU64;
+ bool cmp = bpf_insn_is_cond_jump(insn->code);
+ u8 reg;
+
+ if (bpf_is_ldimm64(insn) && !insn->src_reg)
+ return add_threshold(env, loop, (u32)insn->imm | (u64)insn[1].imm << 32);
+ if (class != BPF_ST && !cmp && !alu)
+ return 0;
+ if (class != BPF_ST && BPF_SRC(insn->code) == BPF_X) {
+ reg = aux->const_reg_mask & BIT(insn->src_reg) ? insn->src_reg : insn->dst_reg;
+ return cmp && (aux->const_reg_mask & BIT(reg)) ?
+ add_threshold(env, loop, aux->const_reg_vals[reg]) : 0;
+ }
+ if (!alu || (op != BPF_ADD && op != BPF_SUB))
+ return add_threshold(env, loop, insn->imm);
+ if (insn->imm != S32_MIN && !(aux->stack_ptrs & BIT(insn->dst_reg))) {
+ loop->max_step = max(loop->max_step, op == BPF_ADD ? insn->imm : -insn->imm);
+ loop->min_step = min(loop->min_step, op == BPF_ADD ? insn->imm : -insn->imm);
+ }
+ return 0;
+}
+
+/* The number of constants of @loop that are below @k */
+u32 bpf_loop_thresholds_below(const struct bpf_loop *loop, s64 k)
+{
+ u32 lo = 0, hi = loop->threshold_cnt, mid;
+
+ while (lo < hi) {
+ mid = lo + (hi - lo) / 2;
+ if (loop->thresholds[mid] < k)
+ lo = mid + 1;
+ else
+ hi = mid;
+ }
+ return lo;
+}
+
+/* The constants in the body of @loop and in the functions that it calls */
+int bpf_loop_collect_thresholds(struct bpf_verifier_env *env, struct bpf_loop *loop)
+{
+ struct bpf_subprog_info *sub = bpf_find_containing_subprog(env, loop->header);
+ u32 words = BITS_TO_LONGS(env->subprog_cnt);
+ unsigned long *todo, *done;
+ int i, s, n, err = 0;
+
+ if (loop->thresholds || !loop->nr_insns)
+ return 0;
+ todo = kvcalloc(2 * words, sizeof(long), BPF_VERIFIER_GFP);
+ if (!todo)
+ return -ENOMEM;
+ done = todo + words;
+ for (;;) {
+ err = err ?: bpf_add_cost_and_check(env, n, BPF_COST_XREG);
+ for (; n-- && i >= 0 && i < sub[1].start && !err; i = bpf_flow_insn_after(env, i)) {
+ err = add_insn_thresholds(env, loop, i);
+ s = bpf_insn_callee(env, i);
+ if (s > 0 && !bpf_subprog_is_global(env, s) && !__test_and_set_bit(s, done))
+ __set_bit(s, todo);
+ }
+ bpf_add_cost(env, words, BPF_COST_SCAN);
+ s = find_first_bit(todo, env->subprog_cnt);
+ if (s == env->subprog_cnt)
+ break;
+ __clear_bit(s, todo);
+ sub = &env->subprog_info[s];
+ i = sub->start;
+ n = sub[1].start - i;
+ }
+ kvfree(todo);
+ err = err ?: add_threshold(env, loop, 0) ?:
+ bpf_add_cost_and_check(env, loop->threshold_cnt * ilog2(loop->threshold_cnt),
+ BPF_COST_COPY);
+ if (err) {
+ /* only the pass ends, the next pass starts again */
+ kvfree(loop->thresholds);
+ loop->thresholds = NULL;
+ loop->threshold_cnt = loop->threshold_cap = 0;
+ return err;
+ }
+ sort(loop->thresholds, loop->threshold_cnt, sizeof(s64), cmp_thresholds, NULL);
+ for (i = 1, n = 1; i < loop->threshold_cnt; i++)
+ if (loop->thresholds[i] != loop->thresholds[n - 1])
+ loop->thresholds[n++] = loop->thresholds[i];
+ loop->threshold_cnt = n;
+ return 0;
+}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 68f3127ea07..3d4c52c59c7 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3569,7 +3569,8 @@ static bool __is_pointer_value(bool allow_ptr_leaks,
if (allow_ptr_leaks)
return false;
- return is_pointer_regtype(reg->type);
+ /* an unreadable xreg can have a pointer */
+ return reg->type != SCALAR_VALUE;
}
static void save_register_state(struct bpf_verifier_env *env,
@@ -11552,8 +11553,9 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
{
u16 live = env->insn_aux_data[env->subprog_info[subprog].start].live_regs_before;
struct bpf_verifier_state *state = env->cur_state;
- struct bpf_func_state *caller = state->frame[state->curframe], *callee;
+ struct bpf_func_state *caller = state->frame[state->curframe], *callee, *frame;
struct bpf_verifier_state entry = {};
+ struct bpf_reg_state *reg;
int i, err;
callee = kzalloc_obj(*callee, BPF_VERIFIER_GFP);
@@ -11583,6 +11585,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
bpf_free_verifier_state(env, &entry, false);
return err;
}
+ bpf_for_each_reg_in_vstate(&entry, frame, reg, reg->grew_once = false);
entry.max_held_cnt = state->max_held_cnt;
callee->entry_refs = entry.acquired_refs;
callee->entry_rcu_locks = entry.active_rcu_locks;
@@ -17967,61 +17970,43 @@ void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax)
/*
* @s is what @old became after a trip around a cycle, @old included. The end
- * of the range that moved is going to move again. Put it where it may come
- * to rest. There are few such places, so this is done a few times at most.
- * @rungs: where the jumps of the loop stop it, if they do.
- * It's put at the nearest of all these places: every value between the range and the place is
- * made up, and the program is checked with it. Wherever it's put, the range only gets larger.
- * @linear: every iteration adds a constant to it, in that many bits. Then nothing but a jump
- * stops it. If there is none ahead it goes on until it wraps, and is any number of that many bits.
+ * of the range that moved will move again. Move it to a threshold where it
+ * can stop. @min_threshold, @max_threshold: the caller guesses that it stops there.
+ * Take the nearest threshold. Each value between the range and the threshold is made up,
+ * and the program is checked with it. With each threshold the range only gets larger.
*/
-void scalar_widen(struct scalar *s, const struct scalar *old,
- const struct bpf_loop_rungs *rungs, int linear)
+void scalar_widen(struct scalar *s, const struct scalar *old, s64 min_threshold, s64 max_threshold)
{
bool moved = s->r64.base != old->r64.base || s->r64.size != old->r64.size ||
s->r32.base != old->r32.base || s->r32.size != old->r32.size;
- s64 min = cnum64_smin(s->r64), max = cnum64_smax(s->r64), stop;
+ s64 min = cnum64_smin(s->r64), max = cnum64_smax(s->r64), threshold;
/* Only the step got weaker. It comes to rest without help, see below. */
if (!moved && s->var_off.value == old->var_off.value &&
s->var_off.mask == old->var_off.mask)
return;
- if (linear &&
- ((max > cnum64_smax(old->r64) && !(rungs && rungs->has_max && max <= rungs->max[1])) ||
- (min < cnum64_smin(old->r64) && !(rungs && rungs->has_min && min >= rungs->min[1])))) {
- if (linear == 64 || min < 0 || max > U32_MAX) {
- scalar_set_unknown(s);
- return;
- }
- min = 0;
- max = U32_MAX;
- }
if (max > cnum64_smax(old->r64)) {
- stop = max <= 0 ? 0 :
- max <= S32_MAX ? S32_MAX :
- max <= U32_MAX ? U32_MAX : S64_MAX;
- if (rungs && rungs->has_max && max <= rungs->max[1])
- stop = min(stop, max <= rungs->max[0] ? rungs->max[0] : rungs->max[1]);
- max = stop;
+ threshold = max <= 0 ? 0 :
+ max <= S32_MAX ? S32_MAX :
+ max <= U32_MAX ? U32_MAX : S64_MAX;
+ max = min(threshold, max_threshold);
}
if (min < cnum64_smin(old->r64)) {
- stop = min >= 0 ? 0 :
- min >= S32_MIN ? S32_MIN : S64_MIN;
- if (rungs && rungs->has_min && min >= rungs->min[1])
- stop = max(stop, min >= rungs->min[0] ? rungs->min[0] : rungs->min[1]);
- min = stop;
+ threshold = min >= 0 ? 0 :
+ min >= S32_MIN ? S32_MIN : S64_MIN;
+ min = max(threshold, min_threshold);
}
if (min == S64_MIN && max == S64_MAX) {
scalar_set_unknown(s);
return;
}
/*
- * A fixed stop is a guess. A number of 32 bits that gets past S32_MAX comes to rest at
- * U32_MAX. What gets past S32_MAX, U32_MAX or S32_MIN and out of 32 bits is stopped by
- * nothing:
+ * A fixed threshold is a guess. A number of 32 bits that gets past S32_MAX stops at
+ * U32_MAX. No threshold stops what gets past S32_MAX, U32_MAX or S32_MIN and out of
+ * 32 bits:
* sum += a[i]; if (sum == 77) ...
- * It wraps in the next pass, and the other bound is gone as well.
+ * It wraps in the next trip, and the other bound is gone as well.
*/
if ((max == S64_MAX &&
(cnum64_smax(old->r64) == S32_MAX || cnum64_smax(old->r64) == U32_MAX)) ||
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 25/49] bpf: Join any two states that have the same join_key
2026-11-20 7:41 ` [PATCH bpf-next 25/49] bpf: Join any two states that have the same join_key Alexei Starovoitov
@ 2026-11-20 8:25 ` sashiko-bot
0 siblings, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:25 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] bpf: missing env->mem adjustment on error in
bpf_loop_collect_thresholds()
--
commit 666a19a48516a34b60842263fb6f1aaa87b240b3
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Join any two states that have the same join_key
This commit implements total state joining in place for any two verifier
states sharing a join_key, along with loop-constant threshold collection for
scalar widening at loop headers.
> diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
> index f8b2f8befa601..23c443ab3a23e 100644
> --- a/kernel/bpf/loops.c
> +++ b/kernel/bpf/loops.c
[ ... ]
> @@ -555,3 +556,109 @@ int bpf_compute_loops(struct bpf_verifier_env *env)
[ ... ]
> +static int add_threshold(struct bpf_verifier_env *env, struct bpf_loop *loop, s64 k)
> +{
> + if (!bpf_grow_array(env, &loop->thresholds, &loop->threshold_cap, loop->threshold_cnt + 1,
> + sizeof(k)))
> + return -ENOMEM;
> + loop->thresholds[loop->threshold_cnt++] = k;
> + return 0;
> +}
[ ... ]
> +/* The constants in the body of @loop and in the functions that it calls */
> +int bpf_loop_collect_thresholds(struct bpf_verifier_env *env, struct bpf_loop *loop)
> +{
[ ... ]
> + err = err ?: add_threshold(env, loop, 0) ?:
> + bpf_add_cost_and_check(env, loop->threshold_cnt * ilog2(loop->threshold_cnt),
> + BPF_COST_COPY);
> + if (err) {
> + /* only the pass ends, the next pass starts again */
> + kvfree(loop->thresholds);
> + loop->thresholds = NULL;
> + loop->threshold_cnt = loop->threshold_cap = 0;
> + return err;
> + }
[Severity: Medium]
Does this error path leak loop->threshold_cap * sizeof(s64) in env->mem?
add_threshold() calls bpf_grow_array(), which charges the allocated capacity
to env->mem. If bpf_loop_collect_thresholds() later fails (for example,
when bpf_add_cost_and_check() hits the per-pass cost limit
env->pass_max_cost), freeing loop->thresholds and zeroing
loop->threshold_cap without subtracting loop->threshold_cap * sizeof(s64)
from env->mem leaves env->mem inflated across subsequent passes, which can
cause a later pass to hit env->max_mem (-E2BIG) prematurely.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=25
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 26/49] bpf: Check the equations when a state is pruned or joined
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (24 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 25/49] bpf: Join any two states that have the same join_key Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:41 ` [PATCH bpf-next 27/49] bpf: Guess the equations of a join Alexei Starovoitov
` (22 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Ranges do not show that a state has a relation:
old: r1 0..7, r2 0..7, r1 == r2
cur: r1 0..7, r2 0..7 old does not subsume cur
check_equations() looks for a value in cur for each sym of old:
old: r1 = sym0 r2 = 4 * sym0 + 8 sym0 0..7
cur: r1 = sym5 + 1 r2 = 4 * sym5 + 12 sym5 0..3
r1: sym0 is sym5 + 1, and 1..4 is in 0..7
r2: (4 * sym5 + 12 - 8) / 4 is sym5 + 1 too
A join needs equations that hold in both states. The next patch guesses
them. This patch checks the guess with the same function:
guess -> check in dst and in src -> delete what fails -> check again
The worst guess gives no equation.
A widened state is saved as it is. Its equations are applied to the
copy that runs, see needs_refine. Else the state at the header can get
smaller again, and the loop has no fixpoint.
An infeasible state ends after the next jump. It does not subsume a
state that is feasible.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 7 ++
kernel/bpf/equation.c | 183 +++++++++++++++++++++++++++++++++++
kernel/bpf/join.c | 2 +
kernel/bpf/states.c | 2 +-
4 files changed, 193 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 1c04bc2d5e4..f93ca5cf0df 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -2416,7 +2416,14 @@ void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *fra
void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
bool taken);
+bool bpf_equations_subsume(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *cur);
void bpf_equations_print(struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
+void bpf_equations_pre_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen);
+void bpf_equations_post_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src);
u32 bpf_equations_bytes(const struct bpf_equations *eqs);
void scalar_bounds_sync(struct scalar *s);
bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index 479cb1466e1..3d9df6e6845 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -2000,6 +2000,189 @@ void bpf_equations_clean(struct bpf_verifier_env *env, struct bpf_verifier_state
eqs->clean_sym_cnt = eqs->sym_cnt;
}
+/* Subsume */
+
+/* Translate @expr to the other state */
+static bool match_expr(const struct sym_scratch *scratch, struct expr *expr)
+{
+ if (!expr->coef)
+ return true;
+ scratch += var_symno(expr->var);
+ *expr = expr_scale(scratch->cur_expr, expr->coef, expr->off);
+ return scratch->determined;
+}
+
+/* @sym of @old is @expr in @cur */
+static bool match_sym(struct bpf_verifier_env *env, const struct bpf_equations *old,
+ const struct bpf_verifier_state *cur, u32 sym, const struct expr *expr)
+{
+ struct sym_scratch *scratch = &env->eq_env->scratch[sym];
+ struct scalar bounds;
+ struct expr diff;
+ struct equation_pos pos = {};
+
+ while (scratch->determined &&
+ bpf_find_next_diff(env, &cur->equations, &scratch->cur_expr, expr, &pos, &diff))
+ if (!diff.coef && !diff.off)
+ return true;
+ if (scratch->determined)
+ return false;
+ bpf_eval_expr(env, cur, expr, &bounds);
+ scratch->determined = scalar_subsumes(&old->syms[sym], &bounds);
+ scratch->cur_expr = *expr;
+ return scratch->determined;
+}
+
+/* @hint: after the equation of @cur that was found last. Most are in the same order. */
+static bool check_sym_equation(struct bpf_verifier_env *env, const struct bpf_equations *old,
+ const struct bpf_verifier_state *cur,
+ const struct bpf_sym_equation *sym_eq, u32 *hint)
+{
+ const struct sym_scratch *scratch = env->eq_env->scratch;
+ struct equation_pos pos = { .next = *hint };
+ struct expr expr[2], diff;
+ struct scalar bounds;
+ int i, unmatched;
+
+ if (is_op(sym_eq)) {
+ expr[0] = rhs_expr(sym_eq, 0);
+ expr[1] = rhs_expr(sym_eq, 1);
+ return match_expr(scratch, &expr[0]) && match_expr(scratch, &expr[1]) &&
+ bpf_find_op(env, &cur->equations, sym_eq->insn_idx, expr, &diff) &&
+ match_sym(env, old, cur, sym_eq->sym[0], &diff);
+ }
+ /* -coef[unmatched] * sym[unmatched] = expr[0] - expr[1] */
+ for (unmatched = 0; unmatched < 2 && scratch[sym_eq->sym[unmatched]].determined;
+ unmatched++)
+ ;
+ for (i = 0; i < 2; i++) {
+ expr[i] = sym_expr(sym_eq->coef[i + (i >= unmatched)] * (i ? -1 : 1),
+ sym_eq->sym[i + (i >= unmatched)], i ? 0 : sym_eq->off);
+ if (!match_expr(scratch, &expr[i]))
+ return false;
+ }
+ while (bpf_find_next_diff(env, &cur->equations, &expr[0], &expr[1], &pos, &diff)) {
+ if (!sym_eq->coef[unmatched] ? !diff.coef && !diff.off :
+ bpf_expr_div_exact(&diff, 0, -sym_eq->coef[unmatched]) &&
+ match_sym(env, old, cur, sym_eq->sym[unmatched], &diff)) {
+ *hint = pos.next;
+ return true;
+ }
+ }
+ /* no other equation uses it */
+ if (!sym_has_one_use(scratch, sym_eq->sym[unmatched]) ||
+ !is_plus_minus_one(sym_eq->coef[unmatched]))
+ return false;
+ bpf_eval_diff(env, cur, &expr[sym_eq->coef[unmatched] > 0],
+ &expr[sym_eq->coef[unmatched] < 0], &bounds);
+ return scalar_subsumes(&old->syms[sym_eq->sym[unmatched]], &bounds);
+}
+
+/*
+ * Are there values for the syms of @old for which its equations hold in @cur?
+ * @forget: it is @old. Mark the equations that do not hold, normalize_equations() deletes them.
+ */
+static bool check_equations(struct bpf_verifier_env *env, const struct bpf_equations *old,
+ const struct bpf_verifier_state *cur, struct bpf_equations *forget)
+{
+ struct bpf_xreg_equation *xreg_eq;
+ struct bpf_sym_equation *sym_eq;
+ bool is_ptr, ok = true;
+ struct expr expr;
+ u32 hint = 0;
+
+ count_sym_uses(env, old);
+ for_each_xreg_equation(env, old, xreg_eq) {
+ if (bpf_xreg_expr(env, cur, xreg_eq->xreg, &expr, &is_ptr) &&
+ is_ptr == xreg_eq->is_ptr &&
+ bpf_expr_div_exact(&expr, xreg_eq->off, xreg_eq->coef) &&
+ match_sym(env, old, cur, xreg_eq->sym, &expr))
+ continue;
+ if (!forget)
+ return false;
+ xreg_eq->coef = 0;
+ forget->normalized = ok = false;
+ }
+ for_each_sym_equation(env, old, sym_eq) {
+ /* it depends on an equation that does not hold */
+ if (!ok && find_undetermined_sym(env->eq_env->scratch, sym_eq) < 0)
+ continue;
+ if (check_sym_equation(env, old, cur, sym_eq, &hint))
+ continue;
+ if (!forget)
+ return false;
+ sym_eq->sym[0] = NO_SYM;
+ forget->normalized = ok = false;
+ }
+ return ok;
+}
+
+bool bpf_equations_subsume(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *cur)
+{
+ const struct bpf_xreg_equation *xreg_eq;
+ const struct scalar *s;
+
+ /* @old ends after the next jump. The equations that showed it can be gone. */
+ if (old->equations.infeasible && !cur->equations.infeasible && env->bypass_spec_v1)
+ return false;
+ /* see loop_stack_safe() */
+ for_each_xreg_equation(env, &old->equations, xreg_eq) {
+ if ((u32)xreg_eq->xreg < BPF_XREG_TRIP)
+ continue;
+ s = equation_xreg_bounds(env, cur, xreg_eq);
+ if (!s || !scalar_subsumes(equation_xreg_bounds(env, old, xreg_eq), s))
+ return false;
+ }
+ return !old->equations.xreg_equation_cnt ||
+ check_equations(env, &old->equations, cur, NULL);
+}
+
+/* Join */
+
+/*
+ * Make the equations of @dst hold for @src too. Call it before the xregs are joined.
+ * @leftover_loop: the states are at its header. @widen: @src came around it.
+ */
+void bpf_equations_pre_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen)
+{
+ const struct loop_stack_entry *top = bpf_top_loop(src->frame[src->curframe]);
+ bool enters = leftover_loop && top &&
+ top->trip < bpf_top_loop(dst->frame[dst->curframe])->trip;
+ struct bpf_equations *eqs;
+
+ if (env->eq_env) {
+ eqs = &env->eq_env->join_state.equations;
+ /* what enters the loop can break the equations that keep the xregs small */
+ if (enters && dst->equations.needs_refine)
+ bpf_equations_refine_all(env, dst);
+ bpf_guess_equations(env, dst, src, leftover_loop, widen);
+ normalize_equations(env, &env->eq_env->join_state);
+ while (!check_equations(env, eqs, dst, eqs) || !check_equations(env, eqs, src, eqs))
+ normalize_equations(env, &env->eq_env->join_state);
+ if (widen && !dst->speculative)
+ bpf_note_linear_xregs(env, dst, eqs);
+ eqs->needs_refine = widen || dst->equations.needs_refine ||
+ src->equations.needs_refine;
+ swap(dst->equations, *eqs);
+ }
+}
+
+/* The xregs of @dst are joined. Some of them are gone. */
+void bpf_equations_post_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src)
+{
+ u32 cnt = dst->equations.xreg_equation_cnt;
+
+ bpf_equations_clean(env, dst);
+ while (cnt != dst->equations.xreg_equation_cnt &&
+ !check_equations(env, &dst->equations, src, &dst->equations))
+ normalize_equations(env, dst);
+ dst->equations.infeasible = false;
+}
+
static void print_operand(struct bpf_verifier_log *log, const char *prefix, struct expr expr)
{
if (expr.coef)
diff --git a/kernel/bpf/join.c b/kernel/bpf/join.c
index b1d42ad23ca..b1f34513b9f 100644
--- a/kernel/bpf/join.c
+++ b/kernel/bpf/join.c
@@ -236,9 +236,11 @@ int bpf_states_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst
env->pkt_pair_cnt = 0;
env->imprecise |= !env->err;
dst->suspect_loops |= src->suspect_loops;
+ bpf_equations_pre_join(env, dst, src, leftover_loop, leftover_loop && widen);
for (j.fr = 0; j.fr <= dst->curframe; j.fr++)
join_func(&j, dst->frame[j.fr], src->frame[j.fr],
env->insn_aux_data[bpf_frame_insn_idx(dst, j.fr)].live_regs_before);
+ bpf_equations_post_join(env, dst, src);
dst->max_held_cnt = max(dst->max_held_cnt, src->max_held_cnt);
/* no equation is found over a limit */
if (env->test_reg_invariants && !bpf_state_subsumes(env, dst, src) &&
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 174b1a613fb..3f4edb9bafb 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -649,5 +649,5 @@ bool bpf_state_subsumes(struct bpf_verifier_env *env, const struct bpf_verifier_
bpf_frame_insn_idx(old, i)))
return false;
- return true;
+ return bpf_equations_subsume(env, old, cur);
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 27/49] bpf: Guess the equations of a join
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (25 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 26/49] bpf: Check the equations when a state is pruned or joined Alexei Starovoitov
@ 2026-11-20 7:41 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 28/49] bpf: Prune or join each state that arrives Alexei Starovoitov
` (21 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:41 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A join of ranges forgets what moves together. A phi is a sym of the
join. It has one value in each state:
trip r1 r2 r3
old state 0 0 p 0..9
new state 1 4 p + 8 1..9
join sym0 4 * sym0 p + 8 * sym0 sym0 + (0..9)
join_expr() looks for a phi that gives both values. If there is none,
it makes one. The sums and ops of the two states are translated to the
phis. A sym can become two phis. Each of them is tried.
The phi that is 0 in the old state and 1 in the new state is the
selector. It can be a stored condition:
c = 0, len = 0..8 | c = 1, len = 64..72
join: len - 64 * c = 0..8
A number gets a phi if its two ranges are disjoint, in 64 bits or in
the low 32 bits. A helper returns an int, and the high bits are not
known:
w9 = 0; if w0 s> -1 goto L; w9 = -155; L: ...; if w0 s< 0 goto err
A phi that grows is widened to a threshold, and keeps its step. The
low 32 bits go around their own circle, so an int can go through 0:
int i = -5; do { ... } while (++i != 3); 0xfffffffb .. 0 .. 2
It terminates. From the third join around a cycle only the equations
of the old state are kept, and an equation can only get weaker.
The cpu can speculate past the test of c. So a constant makes no
phi without bypass_spec_v1.
A number can grow in a loop and in the loop around it:
for (j = 0; j < 13; j++) {
for (k = 0; k < 20; k++)
off += 0..2;
off += 0..1;
}
buf[off] off is 0..533
The second outer trip enters the inner loop again. Each state shows one
trip index, the other one is 0:
old state j = 0 off = 2 * k + d
new state k = 0 off = 41 * j + d
No equation holds in both, so off has no bound, and all trips have to
be peeled. Make a sym for the two parts:
join m = 2 * k + 41 * j, off = m + d
All of this file is a guess, see the previous patch.
insns processed
program bpf-next without m with m
strobemeta_nounroll1 10658 48984 1127
strobemeta_nounroll2 22116 52061 1127
strobemeta_subprogs 10829 45634 1007
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 3 +
kernel/bpf/equation.h | 5 +
kernel/bpf/equation_join.c | 541 +++++++++++++++++++++++++++++++++++
kernel/bpf/fixups.c | 1 +
4 files changed, 550 insertions(+)
create mode 100644 kernel/bpf/equation_join.c
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index f93ca5cf0df..5cbffbe9d0a 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -450,6 +450,7 @@ struct loop_stack_entry {
bool peeled; /* in a peeled trip, not in the leftover */
/* an xreg: how many times the program came back to the header */
struct scalar trip_index;
+ bool trip_widened; /* it was moved to a threshold, or it does not grow */
};
#define LOOP_STACK_SIZE 16
@@ -862,6 +863,8 @@ struct bpf_loop {
);
bool partitioned; /* the last partition was for this loop */
bool peeled_joined; /* at the start of a peeled trip, against the join rule */
+ struct bpf_iv_threshold *iv_thresholds; /* where an iv can stop, see equation_join.c */
+ u32 iv_threshold_cnt, iv_threshold_cap;
struct bpf_loop_per_pass this_pass;
/* sorted constants of the loop, the largest and the smallest constant that an insn adds */
s64 *thresholds;
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index 35efc6d86a6..c88740ffb42 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -79,6 +79,7 @@ struct bpf_equation_env {
struct bpf_verifier_state join_state; /* it has equations and no frames */
struct join_phi *phis;
u32 *phi_index;
+ u32 phi_cap, phi_index_cap;
};
#define for_each_sym_equation(env, eqs, pos) \
@@ -185,4 +186,8 @@ bool bpf_find_next_diff(struct bpf_verifier_env *env, const struct bpf_equations
void bpf_eval_diff(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
const struct expr *a, const struct expr *b, struct scalar *bounds);
+/* equation_join.c */
+void bpf_guess_equations(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen);
#endif
diff --git a/kernel/bpf/equation_join.c b/kernel/bpf/equation_join.c
new file mode 100644
index 00000000000..d47c4f04453
--- /dev/null
+++ b/kernel/bpf/equation_join.c
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * The equations of a join and the thresholds of a trip index.
+ *
+ * A phi is a sym of the join. It has one value in each state:
+ *
+ * trip r1 r2 r3
+ * old state 0 0 p 0..9
+ * new state 1 4 p + 8 1..9
+ * join sym0 4 * sym0 p + 8 * sym0 sym1 = sym0 + sym2, sym2 is 0..9
+ */
+#include <linux/bpf.h>
+#include <linux/cnum.h>
+#include <linux/filter.h>
+#include <linux/slab.h>
+#include <linux/tnum.h>
+#include "equation.h"
+
+/* The values of a sym of the join in the two states */
+struct join_phi {
+ struct expr arg[2];
+ u32 next_phi; /* see phi_list() */
+ bool in_sym_equation;
+};
+
+/* Only the range is known */
+static const struct expr no_expr = { 1, 0, VAR_NONE };
+
+struct equation_join {
+ struct bpf_verifier_env *env;
+ const struct bpf_verifier_state *st[2];
+ struct bpf_equations *eqs;
+ const struct bpf_loop *leftover_loop;
+ bool widen;
+ /* the sym that is 0 in the old state and 1 in the new state */
+ u64 selector;
+ /* by the sym of a state. 0: none, else the sym of the join + 1 */
+ u32 *sym2phi[2], *phi_lists;
+ /* after the equation that was found last in the two states and in the join */
+ u32 hint[3];
+ u32 sums;
+};
+
+enum iv_threshold_kind {
+ THRESHOLD_EXIT_JUMP, /* of the trip index, by a jump that leaves the loop */
+ THRESHOLD_BODY_JUMP, /* by another jump: the next trip can test its result */
+ THRESHOLD_SYM, /* of a number that moves with it */
+ THRESHOLD_DIFF, /* of the difference of two numbers */
+};
+
+struct bpf_iv_threshold {
+ s64 value;
+ enum iv_threshold_kind kind;
+ /* the jump that gave it, and its index among the thresholds of that jump */
+ u32 insn_idx, idx;
+};
+
+#define for_each_iv_threshold(env, loop, threshold, k) \
+ for (bpf_add_cost(env, (loop)->iv_threshold_cnt, BPF_COST_SCAN), \
+ threshold = (loop)->iv_thresholds; \
+ threshold < (loop)->iv_thresholds + (loop)->iv_threshold_cnt; threshold++) \
+ if (threshold->kind == (k))
+
+static const struct expr *phi_args(const struct equation_join *j, u64 var)
+{
+ return j->env->eq_env->phis[var_symno(var)].arg;
+}
+
+static bool phi_has_exprs(const struct expr *arg)
+{
+ return !expr_equal(&arg[0], &no_expr) && !expr_equal(&arg[1], &no_expr);
+}
+
+/* The list of phis that are made of the vars in @arg. NULL if there can be only one. */
+static u32 *phi_list(const struct equation_join *j, const struct expr *arg)
+{
+ int i = !arg[0].coef;
+
+ if (!arg[i].coef)
+ return j->phi_lists;
+ return !var_is_sym(arg[i].var) ? NULL :
+ j->phi_lists + 1 + i * j->st[0]->equations.sym_cnt + var_symno(arg[i].var);
+}
+
+/*
+ * The low 32 bits go around their own circle to the next threshold:
+ * int i = -5; do { ... } while (++i != 3); i is 0xfffffffb..0xffffffff, 0..2
+ */
+static void widen_low32(struct equation_join *j, struct scalar *widened,
+ const struct scalar *joined, const struct scalar *old,
+ enum iv_threshold_kind kind)
+{
+ u32 start = joined->r32.base, end = start + joined->r32.size;
+ u64 up = end == old->r32.base + old->r32.size ? 0 : U32_MAX;
+ u64 down = start == old->r32.base ? 0 : U32_MAX;
+ const struct bpf_iv_threshold *threshold;
+
+ if (!up && !down)
+ widened->var_off = tnum_with_subreg(widened->var_off, joined->var_off);
+ for_each_iv_threshold(j->env, j->leftover_loop, threshold, kind) {
+ up = min_t(u64, up, (u32)(threshold->value - end));
+ down = min_t(u64, down, (u32)(start - threshold->value));
+ }
+ if (joined->r32.size + up + down < U32_MAX)
+ cnum32_intersect_with(&widened->r32, (struct cnum32){ start - down,
+ joined->r32.size +
+ up + down });
+}
+
+static u64 new_phi(struct equation_join *j, const struct expr *arg, const struct scalar *bounds,
+ enum iv_threshold_kind kind)
+{
+ struct bpf_equation_env *eq_env = j->env->eq_env;
+ struct scalar joined = bounds[0], widened;
+ s64 min = S64_MIN, max = S64_MAX;
+ const struct bpf_iv_threshold *threshold;
+ bool is_ptr, grew_before, grows;
+ u32 *list;
+ u64 var;
+ int i;
+
+ bpf_add_cost(j->env, 1, BPF_COST_INSN);
+ scalar_union(&joined, &bounds[1]);
+ grows = j->leftover_loop && !scalar_equal(&joined, &bounds[0]);
+ grew_before = arg[0].var != VAR_NONE &&
+ bpf_find_var_bounds(j->env, j->st[0], arg[0].var, &is_ptr)->grew_once;
+ if (grows && j->widen && grew_before) {
+ widened = joined;
+ for_each_iv_threshold(j->env, j->leftover_loop, threshold, kind) {
+ if (threshold->value >= cnum64_smax(joined.r64))
+ max = min(max, threshold->value);
+ if (threshold->value <= cnum64_smin(joined.r64))
+ min = max(min, threshold->value);
+ }
+ if (kind == THRESHOLD_SYM) {
+ scalar_widen(&widened, &bounds[0], min, max);
+ } else {
+ if (cnum64_smin(joined.r64) >= cnum64_smin(bounds[0].r64))
+ min = cnum64_smin(joined.r64);
+ if (cnum64_smax(joined.r64) <= cnum64_smax(bounds[0].r64))
+ max = cnum64_smax(joined.r64);
+ scalar_set_unknown(&widened);
+ widened.r64 = cnum64_union(cnum64_from_srange(min, max), joined.r64);
+ }
+ widen_low32(j, &widened, &joined, &bounds[0], kind);
+ /* the step changes only to a divisor. A number that can wrap has no step. */
+ if (cnum64_smin(widened.r64) != S64_MIN && cnum64_smax(widened.r64) != S64_MAX) {
+ widened.step = joined.step;
+ widened.base = joined.base;
+ }
+ joined = widened;
+ }
+ if (!bpf_grow_array(j->env, &eq_env->phis, &eq_env->phi_cap,
+ j->eqs->sym_cnt + 1, sizeof(*eq_env->phis)))
+ return VAR_NONE;
+ var = bpf_new_sym(j->env, j->eqs, &joined);
+ if (!var_is_sym(var))
+ return var;
+ j->eqs->syms[var_symno(var)].grew_once = grew_before || grows;
+ for (i = 0; i < 2; i++) {
+ eq_env->phis[var_symno(var)].arg[i] = arg[i];
+ if (arg[i].coef == 1 && var_is_sym(arg[i].var) &&
+ !j->sym2phi[i][var_symno(arg[i].var)])
+ j->sym2phi[i][var_symno(arg[i].var)] = var_symno(var) + 1;
+ }
+ eq_env->phis[var_symno(var)].next_phi = 0;
+ eq_env->phis[var_symno(var)].in_sym_equation = false;
+ list = phi_has_exprs(arg) ? phi_list(j, arg) : NULL;
+ for (; list && *list; list = &eq_env->phis[*list - 1].next_phi)
+ bpf_add_cost(j->env, 1, BPF_COST_SCAN);
+ if (list)
+ *list = var_symno(var) + 1;
+ if (!arg[0].coef && !arg[1].coef)
+ j->selector = var;
+ return var;
+}
+
+/* The expr that is @expr[0] in the old state and @expr[1] in the new state */
+static struct expr join_expr(struct equation_join *j, const struct expr *expr,
+ enum iv_threshold_kind kind)
+{
+ struct expr arg[2], res;
+ struct scalar bounds[2];
+ const struct expr *phi_arg;
+ u32 symno, i, *list;
+
+ if (!expr[0].coef && expr_equal(&expr[0], &expr[1]))
+ return expr[0];
+ /* the cpu can speculate past the test of one number that only goes with another one */
+ if ((!expr[0].coef || !expr[1].coef) && !j->env->bypass_spec_v1)
+ return no_expr;
+ for (list = phi_list(j, expr); list && *list;
+ list = &j->env->eq_env->phis[symno].next_phi) {
+ symno = *list - 1;
+ phi_arg = phi_args(j, symno);
+ bpf_add_cost(j->env, 1, BPF_COST_COPY);
+ if (phi_arg[0].var != expr[0].var || phi_arg[1].var != expr[1].var)
+ continue;
+ bpf_add_cost(j->env, 1, BPF_COST_XREG);
+ i = !phi_arg[0].coef;
+ res = make_expr(0, 0, phi_arg[i].coef ? expr[i].coef : expr[1].off - expr[0].off);
+ if (phi_arg[i].coef && !bpf_expr_div_exact(&res, 0, phi_arg[i].coef))
+ continue;
+ res = make_expr(res.off, VAR_SYM | symno, expr[0].off - res.off * phi_arg[0].off);
+ arg[0] = expr_scale(phi_arg[0], res.coef, res.off);
+ arg[1] = expr_scale(phi_arg[1], res.coef, res.off);
+ if (expr_equal(&arg[0], &expr[0]) && expr_equal(&arg[1], &expr[1]))
+ return res;
+ }
+ res.coef = expr[0].coef ?: expr[1].coef ?: expr[1].off - expr[0].off;
+ res.off = expr[0].coef || !expr[1].coef ? expr[0].off : expr[1].off;
+ arg[0] = expr[0];
+ arg[1] = expr[1];
+ if (!bpf_expr_div_exact(&arg[0], res.off, res.coef) ||
+ !bpf_expr_div_exact(&arg[1], res.off, res.coef)) {
+ res = make_expr(1, VAR_NONE, 0);
+ arg[0] = expr[0];
+ }
+ bpf_eval_expr(j->env, j->st[0], &arg[0], &bounds[0]);
+ bpf_eval_expr(j->env, j->st[1], &arg[1], &bounds[1]);
+ res.var = new_phi(j, arg, bounds, kind);
+ return res;
+}
+
+static void join_xreg(struct equation_join *j, u64 xreg)
+{
+ struct expr expr[2];
+ bool is_ptr[2];
+
+ if (over_limit(j->env) || bpf_find_xreg_equation(j->eqs, xreg) ||
+ !bpf_xreg_expr(j->env, j->st[0], xreg, &expr[0], &is_ptr[0]) ||
+ !bpf_xreg_expr(j->env, j->st[1], xreg, &expr[1], &is_ptr[1]) || is_ptr[0] != is_ptr[1])
+ return;
+ expr[0] = join_expr(j, expr,
+ (u32)xreg < BPF_XREG_TRIP ? THRESHOLD_SYM : THRESHOLD_EXIT_JUMP);
+ bpf_set_xreg_equation(j->env, j->eqs, xreg, &expr[0], is_ptr[0]);
+}
+
+static bool scalars_are_disjoint(const struct scalar *a, const struct scalar *b)
+{
+ return cnum64_is_empty(cnum64_intersect(a->r64, b->r64)) ||
+ cnum32_is_empty(cnum32_intersect(a->r32, b->r32));
+}
+
+/* The xregs that have no equation. @all: what a trip changes. */
+static void join_other_xregs(struct equation_join *j, bool all)
+{
+ const struct scalar *a, *b;
+ u32 fr, xregno, cnt;
+ bool is_ptr;
+
+ for (fr = 0; fr <= j->st[0]->curframe; fr++) {
+ cnt = bpf_spi_to_xreg(bpf_stack_nr_slots(j->st[0]->frame[fr]));
+ bpf_add_cost(j->env, cnt, BPF_COST_XREG);
+ for (xregno = 0; xregno < cnt; xregno++) {
+ a = bpf_find_var_bounds(j->env, j->st[0], bpf_xreg(fr, xregno), &is_ptr);
+ b = a ? bpf_find_var_bounds(j->env, j->st[1], bpf_xreg(fr, xregno),
+ &is_ptr) : NULL;
+ if (b && !scalar_equal(a, b) && (all || scalars_are_disjoint(a, b)))
+ join_xreg(j, bpf_xreg(fr, xregno));
+ }
+ }
+}
+
+/* Most equations are in the same order in all tables */
+static bool find_diff(struct equation_join *j, int i, const struct expr *a, const struct expr *b,
+ struct expr *diff)
+{
+ struct equation_pos pos = { .next = j->hint[i] };
+
+ if (!bpf_find_next_diff(j->env, i < 2 ? &j->st[i]->equations : j->eqs, a, b, &pos, diff))
+ return false;
+ j->hint[i] = pos.next;
+ return true;
+}
+
+static void join_add_sym_equation(struct equation_join *j, struct bpf_sym_equation sym_eq)
+{
+ int i;
+
+ for (i = 0; i < 3; i++)
+ if (sym_eq.coef[i] && sym_eq.sym[i] != NO_SYM)
+ j->env->eq_env->phis[sym_eq.sym[i]].in_sym_equation = true;
+ j->sums += bpf_add_sym_equation(j->env, j->eqs, sym_eq);
+}
+
+/* An equation for @a - @b. True if it needs no new sym. @needs_expr: a range is not enough. */
+static bool join_diff(struct equation_join *j, struct expr a, struct expr b, bool needs_expr)
+{
+ struct join_phi *phis = j->env->eq_env->phis;
+ u32 sym_cnt = j->eqs->sym_cnt;
+ struct expr expr[2], x, y, res;
+ struct scalar bounds[2];
+ int i;
+
+ if (!a.coef || !b.coef || a.var == b.var || over_limit(j->env))
+ return true;
+ if (phis[var_symno(a.var)].in_sym_equation && phis[var_symno(b.var)].in_sym_equation &&
+ find_diff(j, 2, &a, &b, &res))
+ return true;
+ if (!phi_has_exprs(phi_args(j, a.var)) || !phi_has_exprs(phi_args(j, b.var)))
+ return false;
+ for (i = 0; i < 2; i++) {
+ x = expr_scale(phi_args(j, a.var)[i], a.coef, 0);
+ y = expr_scale(phi_args(j, b.var)[i], b.coef, 0);
+ if (find_diff(j, i, &x, &y, &expr[i]))
+ continue;
+ if (needs_expr || !j->env->bypass_spec_v1 || (i && expr_equal(&expr[0], &no_expr)))
+ return false;
+ bpf_eval_diff(j->env, j->st[i], &x, &y, &bounds[i]);
+ expr[i] = no_expr;
+ }
+ i = expr_equal(&expr[0], &no_expr);
+ if (!phi_has_exprs(expr))
+ bpf_eval_expr(j->env, j->st[i], &expr[i], &bounds[i]);
+ res = phi_has_exprs(expr) ? join_expr(j, expr, THRESHOLD_DIFF) :
+ make_expr(1, new_phi(j, expr, bounds, THRESHOLD_DIFF), 0);
+ if (res.coef && !var_is_sym(res.var))
+ return false;
+ join_add_sym_equation(j, (struct bpf_sym_equation){
+ { a.coef, -b.coef, -res.coef },
+ { a.var, b.var, res.var },
+ -1, -res.off });
+ return sym_cnt == j->eqs->sym_cnt;
+}
+
+/* Translate @expr from state @input to the join */
+static bool translate_expr(struct equation_join *j, int input, struct expr *expr)
+{
+ u32 symno;
+
+ if (!expr->coef)
+ return true;
+ symno = j->sym2phi[input][var_symno(expr->var)];
+ if (!symno--)
+ return false;
+ *expr = make_expr(expr->coef, VAR_SYM | symno,
+ expr->off - expr->coef * phi_args(j, symno)[input].off);
+ return true;
+}
+
+static void join_op(struct equation_join *j, int input, const struct bpf_sym_equation *sym_eq)
+{
+ struct expr op[2], arg[2], res[2];
+ int i, k;
+
+ for (k = 0; k < 2; k++) {
+ op[k] = rhs_expr(sym_eq, k);
+ if (!translate_expr(j, input, &op[k]) ||
+ (op[k].coef && !phi_has_exprs(phi_args(j, op[k].var))))
+ return;
+ }
+ if (bpf_find_op(j->env, j->eqs, sym_eq->insn_idx, op, res))
+ return;
+ for (i = 0; i < 2; i++) {
+ for (k = 0; k < 2; k++)
+ arg[k] = !op[k].coef ? op[k] :
+ expr_scale(phi_args(j, op[k].var)[i], op[k].coef, op[k].off);
+ if (!bpf_find_op(j->env, &j->st[i]->equations, sym_eq->insn_idx, arg, &res[i]))
+ return;
+ }
+ res[0] = join_expr(j, res, THRESHOLD_SYM);
+ bpf_add_op(j->env, j->eqs, sym_eq->insn_idx, op, res);
+}
+
+/* The equations between the syms of state @input */
+static void join_sym_equations(struct equation_join *j, int input)
+{
+ const struct bpf_sym_equation *sym_eq;
+ struct expr a, b;
+ int last;
+
+ for_each_sym_equation(j->env, &j->st[input]->equations, sym_eq) {
+ if (is_op(sym_eq))
+ join_op(j, input, sym_eq);
+ if (is_op(sym_eq) || j->sums >= SUMS_PER_XREG * j->eqs->xreg_equation_cnt)
+ continue;
+ for (last = 2; last >= (sym_eq->coef[2] ? 0 : 2); last--) {
+ a = sym_expr(sym_eq->coef[!last], sym_eq->sym[!last], 0);
+ b = sym_expr(-sym_eq->coef[2 - (last == 2)],
+ sym_eq->sym[2 - (last == 2)], 0);
+ if (translate_expr(j, input, &a) && translate_expr(j, input, &b) &&
+ join_diff(j, a, b, false))
+ break;
+ }
+ }
+}
+
+/* Two phis are one sym in state @input: i, or i + 1. The first @xreg_phi_cnt phis have xregs. */
+static void join_keep_diffs(struct equation_join *j, int input, u32 xreg_phi_cnt)
+{
+ u32 symno, first_symno, cnt = j->eqs->sym_cnt, *sym2phi;
+ struct expr arg;
+
+ for (symno = 1; symno < cnt && !over_limit(j->env); symno++) {
+ arg = phi_args(j, symno)[input];
+ sym2phi = var_is_sym(arg.var) ? &j->sym2phi[input][var_symno(arg.var)] : NULL;
+ first_symno = sym2phi ? *sym2phi - 1 : symno;
+ if (first_symno >= symno)
+ continue;
+ join_diff(j, sym_expr(1, symno, 0), sym_expr(arg.coef, first_symno, 0), true);
+ if (arg.coef != 1 || symno >= xreg_phi_cnt)
+ continue;
+ /* the equations of the sym can hold for this phi only */
+ *sym2phi = symno + 1;
+ join_sym_equations(j, input);
+ *sym2phi = first_symno + 1;
+ }
+}
+
+/* Find "@x = *@part + rest" in state @input. *@part is a constant in the other state. */
+static bool find_sum_part(struct equation_join *j, int input, u64 x, struct expr *part)
+{
+ const struct bpf_sym_equation *sym_eq;
+ int i, pos;
+
+ for_each_sum(j->env, &j->st[input]->equations, sym_eq) {
+ pos = find_sym_pos(sym_eq, var_symno(x));
+ for (i = 0; i < 3 && pos < 3 && is_plus_minus_one(sym_eq->coef[pos]); i++) {
+ if (i == pos || !sym_eq->coef[i])
+ continue;
+ *part = sym_expr(-sym_eq->coef[pos] * sym_eq->coef[i],
+ sym_eq->sym[i], 0);
+ if (!translate_expr(j, input, part) || phi_args(j, part->var)[!input].coef)
+ continue;
+ part->off = 0;
+ return true;
+ }
+ }
+ return false;
+}
+
+/*
+ * A number grows with two trip indexes. Each state shows one of them:
+ * old state j = 0 x = 2 * k + d join m = 2 * k + 41 * j
+ * new state k = 0 x = 41 * j + d x = m + d
+ */
+static void join_sum_parts(struct equation_join *j, u32 phi_cnt)
+{
+ struct expr part[2], arg[2], sum;
+ u32 symno;
+
+ for (symno = 0; symno < phi_cnt && !over_limit(j->env); symno++) {
+ memcpy(arg, phi_args(j, symno), sizeof(arg));
+ if (arg[0].coef != 1 || arg[1].coef != 1 ||
+ !var_is_sym(arg[0].var) || !var_is_sym(arg[1].var) ||
+ !find_sum_part(j, 0, arg[0].var, &part[0]) ||
+ !find_sum_part(j, 1, arg[1].var, &part[1]))
+ continue;
+ part[1].coef = -part[1].coef;
+ join_diff(j, part[0], part[1], true);
+ if (find_diff(j, 2, &part[0], &part[1], &sum))
+ join_diff(j, sym_expr(1, symno, 0), sum, true);
+ }
+}
+
+/*
+ * The ends of the range of @var in the two states are on two lines. So a stored condition
+ * gives the range:
+ * c = x == 0; ... if (c)
+ */
+static void join_bounds_by_selector(struct equation_join *j, u64 var, bool always)
+{
+ const struct expr *arg = phi_args(j, var);
+ struct scalar a, b;
+ s64 min_slope, max_slope;
+
+ if (var == j->selector || !phi_has_exprs(arg))
+ return;
+ bpf_eval_expr(j->env, j->st[0], &arg[0], &a);
+ bpf_eval_expr(j->env, j->st[1], &arg[1], &b);
+ if (!always && !scalars_are_disjoint(&a, &b))
+ return;
+ min_slope = b.r64.base - a.r64.base;
+ max_slope = min_slope + b.r64.size - a.r64.size;
+ if (max(a.r64.size, b.r64.size) >= U32_MAX) {
+ if (!always && max(a.r32.size, b.r32.size) == U32_MAX)
+ return;
+ min_slope = (s32)(b.r32.base - a.r32.base);
+ max_slope = (s32)(min_slope + b.r32.size - a.r32.size);
+ }
+ if (min_slope)
+ join_diff(j, make_expr(1, var, 0), make_expr(min_slope, j->selector, 0), false);
+ if (max_slope && max_slope != min_slope)
+ join_diff(j, make_expr(1, var, 0), make_expr(max_slope, j->selector, 0), false);
+}
+
+/*
+ * Guess the equations that hold for @dst and @src, see bpf_equations_pre_join().
+ * The first join shows what a trip moves. The next walk finds the thresholds.
+ *
+ * It terminates. From the third join around a cycle, only the equations of @dst are kept.
+ * An equation can only get weaker:
+ * two xregs with one sym -> a sum with a new sym -> a wider sym -> no equation
+ * New equations come from constants, and a constant in @dst gets a range only once.
+ */
+void bpf_guess_equations(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct equation_join j = { env, { dst, src }, &eq_env->join_state.equations,
+ leftover_loop, widen, .selector = VAR_NONE };
+ const struct loop_stack_entry *top = bpf_top_loop(dst->frame[dst->curframe]);
+ const struct bpf_xreg_equation *xreg_eq;
+ u32 i, sym_cnt;
+ bool selector_is_trip, both_inputs;
+ int fr;
+
+ j.eqs->xreg_equation_cnt = j.eqs->sym_equation_cnt = j.eqs->sym_cnt = 0;
+ sym_cnt = dst->equations.sym_cnt + src->equations.sym_cnt;
+ if (!bpf_grow_array(env, &eq_env->phi_index, &eq_env->phi_index_cap,
+ 2 * sym_cnt + 1, sizeof(u32)))
+ return;
+ memset(eq_env->phi_index, 0, (2 * sym_cnt + 1) * sizeof(u32));
+ j.sym2phi[0] = eq_env->phi_index;
+ j.sym2phi[1] = eq_env->phi_index + dst->equations.sym_cnt;
+ j.phi_lists = eq_env->phi_index + sym_cnt;
+ eq_env->equation_added = true;
+ for (fr = dst->curframe; fr >= 0; fr--)
+ for (i = dst->frame[fr]->loop_stack_cnt; i--;)
+ join_xreg(&j, bpf_xreg(fr, BPF_XREG_TRIP + i));
+ selector_is_trip = j.selector != VAR_NONE;
+ for (i = 0; i < 2; i++)
+ for_each_xreg_equation(env, &j.st[i]->equations, xreg_eq)
+ join_xreg(&j, xreg_eq->xreg);
+ join_other_xregs(&j, selector_is_trip);
+ both_inputs = !widen || selector_is_trip ||
+ (top && top->trip_index.grew_once && !top->trip_widened);
+ for (i = 0, sym_cnt = j.eqs->sym_cnt; i <= both_inputs; i++) {
+ join_keep_diffs(&j, i, sym_cnt);
+ join_sym_equations(&j, i);
+ }
+ /* the equations made phis too: i, and i before it was truncated to 32 bits */
+ if (leftover_loop)
+ join_keep_diffs(&j, 0, 0);
+ if (leftover_loop && !widen)
+ join_sum_parts(&j, sym_cnt);
+ for (i = 0, sym_cnt = j.eqs->sym_cnt; j.selector != VAR_NONE && i < sym_cnt; i++)
+ join_bounds_by_selector(&j, VAR_SYM | i, selector_is_trip);
+}
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 17da785f8fa..ccde914c57a 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -759,6 +759,7 @@ static void free_loop(struct bpf_loop *loop)
if (!loop)
return;
kvfree(loop->thresholds);
+ kvfree(loop->iv_thresholds);
kvfree(loop);
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 28/49] bpf: Prune or join each state that arrives
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (26 preceding siblings ...)
2026-11-20 7:41 ` [PATCH bpf-next 27/49] bpf: Guess the equations of a join Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 29/49] bpf: Order the pending states and free the states that are done Alexei Starovoitov
` (20 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Add the table of states of the main pass. All states go one way:
+---------+ next +-------------------+
+---->| pending |---------->| check basic block |
| +---------+ +---------+---------+
| new v
| +-------------------+ subsumed
+---------------------------| join point |-----> prune
joined, or kept apart +-------------------+
bpf_flow_deliver() cleans the state and updates its loop stack.
flow_prune_or_join() compares it with the states that have the same
join_key:
if bpf_must_keep_apart() look at the next state
if old subsumes new prune new
if new subsumes old free old, but widen in a leftover
else join, and compare the join again
There is no cap on the states at an insn. A list makes n states cost
n * n. So one interval tree sorts all states by the hash of the
join_key, then by the range of one xreg of data_deps, see flow_find().
The hash has kernel addresses, such as the address of a map. If it has
32 bits, two of 2048 states at one insn collide in 1 of 2000 loads, and
that can change the result. So it has 64 bits. The tree has room for
32 bits. A state that collides only there is skipped.
@again is for an insn that is checked once more with a part of the
state: one function of a set, one part of a range. It meets no other
state at that insn, see FLOW_AGAIN. The parts meet at the next insn.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 3 +
kernel/bpf/diagnostics.c | 5 +
kernel/bpf/diagnostics.h | 1 +
kernel/bpf/flow.c | 607 ++++++++++++++++++++++++++++++++++-
kernel/bpf/verifier.c | 24 +-
5 files changed, 629 insertions(+), 11 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 5cbffbe9d0a..f153e5e7c28 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1709,6 +1709,9 @@ static inline int bpf_add_cost_and_check(struct bpf_verifier_env *env, u64 cnt,
int bpf_flow_prepare(struct bpf_verifier_env *env);
void bpf_flow_reset(struct bpf_verifier_env *env);
void bpf_flow_free(struct bpf_verifier_env *env);
+bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx);
+int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ int insn_idx, int prev_insn_idx, bool again);
int bpf_flow_call(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
struct bpf_verifier_state *entry, int subprog, int call_idx);
int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int exit_idx);
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c
index 25d7982327d..2cb9eceb408 100644
--- a/kernel/bpf/diagnostics.c
+++ b/kernel/bpf/diagnostics.c
@@ -392,6 +392,11 @@ static u32 log_pos(const struct bpf_diag_log *log, u32 idx)
return pos < log->cap ? pos : pos - log->cap;
}
+u64 bpf_diag_event_log_save(struct bpf_verifier_env *env)
+{
+ return env->diag ? log_end(&env->diag->log) : 0;
+}
+
void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos)
{
struct bpf_diag *diag = env->diag;
diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h
index f6380137e9a..cb255218c41 100644
--- a/kernel/bpf/diagnostics.h
+++ b/kernel/bpf/diagnostics.h
@@ -53,6 +53,7 @@ const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __p
const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id);
const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type);
const char *bpf_diag_arg_type_plain(enum bpf_arg_type type);
+u64 bpf_diag_event_log_save(struct bpf_verifier_env *env);
void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos);
u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state);
void bpf_diag_free(struct bpf_verifier_env *env);
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 0cb56390ffd..9c96f10f3ac 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -1,18 +1,125 @@
// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * The main pass of the verifier. It is a fixpoint over the control flow graph.
+ *
+ * +---------+ next +-------------------+
+ * +---->| pending |---------->| check basic block |
+ * | +---------+ +---------+---------+
+ * | new v
+ * | +-------------------+ subsumed
+ * +---------------------------| join point |------------> prune
+ * joined, or kept apart +-------------------+
+ *
+ * A state that arrives is compared with the states that have the same join_key, see
+ * flow_prune_or_join(). The order of pending states completes a loop before the code that
+ * follows it, see flow_compute_order(). The check of a function ends when nothing is pending.
+ *
+ * Loops. The peeled trips are kept apart. The leftover stands for all later trips. A state
+ * that comes back to the header of the leftover is joined and widened.
+ *
+ * Functions. A static function can get only numbers and pointers that are not to the stack.
+ * Then it is verified on its own: once for each instance, not for each chain of calls.
+ * The call waits, see bpf_flow_call().
+ */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
-#include <linux/jhash.h>
+#include <linux/interval_tree_generic.h>
#include <linux/rbtree.h>
#include <linux/slab.h>
+#include "diagnostics.h"
+
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
+enum flow_kind {
+ FLOW_AT, /* before the insn */
+ FLOW_AGAIN, /* the same, it meets no state */
+ FLOW_LEFTOVER, /* about to start the leftover of the loop */
+ FLOW_PEELED, /* started an earlier peeled trip, see flow_prune_peeled() */
+ FLOW_FUNC_ENTRY, /* starts an instance */
+ FLOW_JOINED_CALLS, /* the same, for all numbers, see flow_may_add_instance() */
+ FLOW_FUNC_EXIT, /* came out of an instance */
+};
+
+struct flow_state {
+ struct bpf_verifier_state st;
+ struct rb_node tree_node; /* see flow_find() */
+ u64 hash, first, last, subtree_last;
+ /* in a list of its instance, or in waiting_calls of @callee */
+ struct list_head node;
+ struct flow_instance *started_instance; /* FLOW_FUNC_ENTRY */
+ struct flow_instance *callee; /* of a call */
+ struct rb_node rb; /* pending states */
+ int prev_insn_idx;
+ u64 log_pos, diag_pos; /* the logs have the path up to there */
+ u32 seq;
+ u32 join_key_id; /* states with one id have the same join_key */
+ enum flow_kind kind;
+ bool pending;
+ bool late_exit_reaches; /* a late exit state can get to it */
+ /* it is kept until the states leave this loop, see flow_loop_is_left() */
+ u8 kept_frameno, kept_depth;
+ /* the order of pending states, see flow_set_pending() */
+ u64 priority;
+ u8 order_len;
+ u32 order[MAX_CALL_FRAMES];
+};
+
+#define FLOW_FIRST(s) ((s)->first)
+#define FLOW_LAST(s) ((s)->last)
+INTERVAL_TREE_DEFINE(struct flow_state, tree_node, u64, subtree_last, FLOW_FIRST, FLOW_LAST, static,
+ flow_tree)
+
+/* One check of a function: its entry state, its exit states and the calls that get them */
+struct flow_instance {
+ u32 id;
+ u64 priority; /* 0 until a call waits for it, see flow_set_pending() */
+ struct flow_state *entry_state; /* NULL after the start: it does not change any more */
+ u32 pending_cnt; /* states */
+ /* calls from this instance that can get more exit states, and those that call itself */
+ u32 open_call_cnt, self_call_cnt;
+ u64 *cost; /* of its function, see flow_may_add_instance() */
+ bool in_peeled_trip; /* a call of it, or of a caller, is in a peeled trip */
+ bool ended; /* see flow_end_instance() */
+ struct flow_instance *next;
+ struct list_head waiting_calls; /* calls of this instance */
+ struct list_head exit_states; /* these go to @waiting_calls */
+ struct list_head done; /* see flow_keep() */
+};
+
+#define INSTANCE_BYTES (sizeof(struct flow_instance) + 2 * sizeof(void *))
+
struct flow_kept {
struct list_head node;
struct bpf_verifier_state st;
};
+struct bpf_flow {
+ /* computed once */
+ u32 *pos; /* position of the insn in the order */
+ u32 *rank; /* callers go first */
+ unsigned long *bb_starts; /* insn starts a basic block */
+ unsigned long *join_points; /* more than one edge leads to the insn */
+ struct bpf_loop call_loop; /* from the exit of a function to its next call */
+
+ /* of the pass */
+ u64 *cost; /* per function */
+ u64 budget, charged_cost;
+ u32 budget_func_cnt;
+
+ /* of the check */
+ struct rb_root_cached state_tree;
+ u32 *nr_states; /* per insn */
+ struct rb_root_cached pending;
+ struct flow_instance **instances; /* bpf_verifier_state.instance_id is the index */
+ u32 instance_cnt, instance_cap;
+ struct flow_instance *last_instance; /* of the last pending state */
+ u64 priority_gen;
+ u32 seq;
+ u32 live_states;
+};
+
/*
* The order. Reverse post order is a topological order of the graph without
* back edges, but it may put what follows a loop between the insns of
@@ -298,6 +405,19 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
return err ?: ret;
}
+static bool flow_may_load_pkt_end(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+ struct bpf_insn_access_aux info = { .reg_type = SCALAR_VALUE };
+ bool tstamp = env->prog->tstamp_type_access, valid;
+
+ valid = env->subprog_cnt > 1 && insn->src_reg != BPF_REG_FP && env->ops->is_valid_access &&
+ env->ops->is_valid_access(insn->off, bpf_size_to_bytes(BPF_SIZE(insn->code)),
+ BPF_READ, env->prog, &info);
+ /* it notes what the prog reads, and this is a question */
+ env->prog->tstamp_type_access = tstamp;
+ return valid && info.reg_type == PTR_TO_PACKET_END;
+}
+
int bpf_check_limits(struct bpf_verifier_env *env)
{
env->aborted |= bpf_must_abort();
@@ -330,3 +450,488 @@ static struct bpf_loop *flow_new_call_loop(struct bpf_verifier_env *env, enum bp
env->subprog_info[func + 1].start - env->subprog_info[func].start;
return loop;
}
+
+int bpf_flow_prepare(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct bpf_subprog_info *sub = env->subprog_info;
+ struct bpf_insn *insns = env->prog->insnsi;
+ int len = env->prog->len, i, s, err = 0;
+ struct bpf_iarray *succ;
+ struct bpf_flow *flow;
+ unsigned long *pred;
+
+ flow = kzalloc_obj(*flow, BPF_VERIFIER_GFP);
+ if (!flow)
+ return -ENOMEM;
+ env->flow = flow;
+ flow->state_tree = flow->pending = RB_ROOT_CACHED;
+ flow->call_loop.kind = BPF_LOOP_RECURSION;
+
+ flow->pos = kvcalloc(2 * len, sizeof(u32), BPF_VERIFIER_GFP);
+ flow->rank = kvcalloc(env->subprog_cnt + 1, sizeof(u32), BPF_VERIFIER_GFP);
+ flow->cost = kvcalloc(env->subprog_cnt, sizeof(u64), BPF_VERIFIER_GFP);
+ flow->bb_starts = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
+ flow->join_points = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
+ flow->nr_states = kvcalloc(len, sizeof(u32), BPF_VERIFIER_GFP);
+ pred = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
+ if (!flow->pos || !flow->rank || !flow->cost || !flow->bb_starts || !flow->join_points ||
+ !flow->nr_states || !flow->groups || !pred) {
+ kvfree(pred);
+ return -ENOMEM;
+ }
+
+ /* subprog_topo_order[] has callees first */
+ for (i = 0; i < env->subprog_cnt; i++)
+ flow->rank[env->subprog_topo_order[i]] = env->subprog_cnt - i;
+
+ for (i = 0; i < len; i++) {
+ succ = bpf_insn_successors(env, i);
+ iarray_for_each(s, succ) {
+ if (__test_and_set_bit(s, pred))
+ __set_bit(s, flow->join_points);
+ if (aux[i].bb_end)
+ __set_bit(s, flow->bb_starts);
+ }
+ if (bpf_is_throw_kfunc(&insns[i]) && i + 1 < len)
+ __set_bit(i + 1, flow->bb_starts);
+ /* where the callees come back to */
+ if ((bpf_pseudo_call(&insns[i]) || bpf_is_callx(&insns[i])) && i + 1 < len)
+ __set_bit(i + 1, flow->join_points);
+ if (aux[i].calls_callback) {
+ aux[i].callback_loop = flow_new_call_loop(env, BPF_LOOP_CALLBACK, i,
+ bpf_insn_callee(env, i));
+ err = aux[i].callback_loop ? err : -ENOMEM;
+ }
+ if (aux[i].calls_callback || aux[i].loop)
+ __set_bit(i, flow->join_points);
+ if (bpf_is_ldimm64(&insns[i]))
+ i++;
+ /* a guess: the base register can point to something else than ctx */
+ if (BPF_CLASS(insns[i].code) == BPF_LDX && flow_may_load_pkt_end(env, &insns[i]))
+ bpf_find_containing_subprog(env, i)->loads_pkt_end = true;
+ }
+ for (s = 0; s < env->subprog_cnt; s++, sub++) {
+ i = sub->start;
+ __set_bit(i, flow->join_points);
+ if (!sub->cycle || err)
+ continue;
+ if (sub->cycle != s) {
+ aux[env->subprog_info[sub->cycle].start].recursion_loop->nr_insns +=
+ sub[1].start - i;
+ continue;
+ }
+ aux[i].recursion_loop = flow_new_call_loop(env, BPF_LOOP_RECURSION, i, s);
+ err = aux[i].recursion_loop ? 0 : -ENOMEM;
+ }
+ bitmap_or(flow->bb_starts, flow->bb_starts, flow->join_points, len);
+ kvfree(pred);
+ return err ?: flow_compute_order(env, flow);
+}
+
+static struct flow_state *flow_new_state(struct bpf_verifier_env *env, int prev_insn_idx)
+{
+ struct flow_state *s = kzalloc_obj(*s, BPF_VERIFIER_GFP);
+ struct bpf_flow *flow = env->flow;
+
+ if (!s)
+ return NULL;
+ RB_CLEAR_NODE(&s->tree_node);
+ INIT_LIST_HEAD(&s->node);
+ s->prev_insn_idx = prev_insn_idx;
+ s->seq = ++flow->seq;
+ s->log_pos = env->log.end_pos;
+ s->diag_pos = bpf_diag_event_log_save(env);
+ bpf_add_cost(env, 1, BPF_COST_STATE);
+ env->total_states++;
+ flow->live_states++;
+ env->peak_states = max(env->peak_states, flow->live_states);
+ return s;
+}
+
+static struct flow_instance *flow_state_instance(struct bpf_flow *flow, const struct flow_state *s)
+{
+ return s->started_instance ?: flow->instances[s->st.instance_id];
+}
+
+static void flow_clear_pending(struct bpf_flow *flow, struct flow_state *s)
+{
+ if (!s->pending)
+ return;
+ rb_erase_cached(&s->rb, &flow->pending);
+ flow_state_instance(flow, s)->pending_cnt--;
+ s->pending = false;
+}
+
+static void flow_unlink(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ struct flow_instance *instance;
+
+ flow_clear_pending(env->flow, s);
+ if (s->callee || !RB_EMPTY_NODE(&s->tree_node))
+ env->flow->nr_states[s->st.insn_idx]--;
+ if (s->callee) {
+ instance = flow_state_instance(env->flow, s);
+ instance->open_call_cnt--;
+ instance->self_call_cnt -= instance == s->callee;
+ s->callee = NULL;
+ }
+ if (!RB_EMPTY_NODE(&s->tree_node))
+ flow_tree_remove(s, &env->flow->state_tree);
+ RB_CLEAR_NODE(&s->tree_node);
+ list_del_init(&s->node);
+}
+
+static void flow_free_state(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ flow_unlink(env, s);
+ bpf_free_verifier_state(env, &s->st, false);
+ kfree(s);
+ env->flow->live_states--;
+}
+
+static void flow_free_list(struct bpf_verifier_env *env, struct list_head *list)
+{
+ struct flow_state *s, *sn;
+
+ list_for_each_entry_safe(s, sn, list, node)
+ flow_free_state(env, s);
+}
+
+/* What a check leaves behind. */
+void bpf_flow_reset(struct bpf_verifier_env *env)
+{
+ struct bpf_flow *flow = env->flow;
+ struct rb_node *n;
+ int i;
+
+ if (!flow)
+ return;
+ while ((n = rb_first_cached(&flow->state_tree)))
+ flow_free_state(env, rb_entry(n, struct flow_state, tree_node));
+ for (i = 0; i < flow->instance_cnt; i++)
+ flow_free_list(env, &flow->instances[i]->waiting_calls);
+ for (i = 0; i < flow->instance_cnt; i++) {
+ flow->instances[i]->group->instances = NULL;
+ kfree(flow->instances[i]);
+ env->mem -= flow->instance_cnt * INSTANCE_BYTES;
+ flow->instance_cnt = 0;
+ flow->last_instance = NULL;
+}
+
+void bpf_flow_free(struct bpf_verifier_env *env)
+{
+ struct bpf_flow *flow = env->flow;
+
+ if (!flow)
+ return;
+ bpf_flow_reset(env);
+ kvfree(flow->pos);
+ kvfree(flow->rank);
+ kvfree(flow->cost);
+ kvfree(flow->bb_starts);
+ kvfree(flow->join_points);
+ kvfree(flow->nr_states);
+ kvfree(flow->instances);
+ kfree(flow);
+ env->flow = NULL;
+}
+
+bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx)
+{
+ return test_bit(insn_idx, env->flow->bb_starts);
+}
+
+/*
+ * States meet only if they have the same hash. The tree sorts them by a half of the hash,
+ * then by the range of one xreg of data_deps. Two ranges with a gap stay apart, but not where
+ * states come around a loop. Two constants, such as 0 and 3, can miss their join.
+ * Returns the first state near @s.
+ */
+static struct flow_state *flow_find(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ const struct bpf_insn_aux_data *aux = &env->insn_aux_data[s->st.insn_idx];
+ struct loop_stack_entry *e = bpf_top_loop(s->st.frame[0]);
+ u32 first = 0, last = U32_MAX;
+ u64 hash;
+
+ hash = bpf_hash_apart_xregs(env, &s->st, bpf_hash_join_key(env, &s->st) + s->kind,
+ s->kind == FLOW_PEELED);
+ if (s->kind == FLOW_FUNC_ENTRY && (!e || e->peeled))
+ hash = bpf_hash_args(env, &s->st, hash);
+ if (s->kind == FLOW_AT ? !aux->loop && !aux->callback_loop :
+ s->kind == FLOW_FUNC_EXIT && !e)
+ bpf_range_of_data_dep(env, &s->st, &first, &last);
+ bpf_add_cost(env, bpf_xreg_cnt(&s->st), BPF_COST_COPY);
+ s->join_key_id = ++env->flow->seq;
+ s->hash = hash;
+ s->first = hash << 32 | first;
+ s->last = hash << 32 | last;
+ return flow_tree_iter_first(&env->flow->state_tree, s->first, s->last);
+}
+
+#define flow_for_each_near(env, s, t, tn) \
+ for (t = flow_find(env, s); \
+ t && (tn = flow_tree_iter_next(t, (s)->first, (s)->last), true); t = tn)
+
+static struct flow_instance *flow_new_instance(struct bpf_verifier_env *env,
+ struct flow_state *entry)
+{
+ struct bpf_flow *flow = env->flow;
+ struct flow_instance **instances, *instance;
+ u32 cap;
+
+ if (flow->instance_cnt == flow->instance_cap) {
+ cap = 2 * flow->instance_cap + 2;
+ instances = kvrealloc(flow->instances, cap * sizeof(*instances), BPF_VERIFIER_GFP);
+ if (!instances)
+ return NULL;
+ flow->instances = instances;
+ flow->instance_cap = cap;
+ }
+ instance = kzalloc_obj(*instance, BPF_VERIFIER_GFP);
+ if (!instance)
+ return NULL;
+ INIT_LIST_HEAD(&instance->waiting_calls);
+ INIT_LIST_HEAD(&instance->exit_states);
+ INIT_LIST_HEAD(&instance->done);
+ instance->entry_state = entry;
+ env->mem += INSTANCE_BYTES;
+ instance->id = flow->instance_cnt++;
+ flow->instances[instance->id] = instance;
+ return instance;
+}
+
+static bool flow_same_join_key(struct bpf_verifier_env *env, const struct flow_state *t,
+ struct flow_state *s)
+{
+ /* The tree has a half of the hash. A collision there is random and costs nothing. */
+ if (t->hash != s->hash)
+ return false;
+ if (t->join_key_id == s->join_key_id)
+ return true;
+ bpf_add_cost(env, bpf_xreg_cnt(&t->st) + t->st.acquired_refs, BPF_COST_COPY);
+ if (t->kind != s->kind || t->st.insn_idx != s->st.insn_idx ||
+ !bpf_same_join_key(&t->st, &s->st))
+ return false;
+ s->join_key_id = t->join_key_id;
+ return true;
+}
+
+/* After flow_find(), unless @s is a call */
+static void flow_insert(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ u32 *nr = &env->flow->nr_states[s->st.insn_idx];
+
+ bpf_charge_state(env, &s->st);
+ if (s->callee)
+ list_add_tail(&s->node, &s->callee->waiting_calls);
+ else
+ flow_tree_insert(s, &env->flow->state_tree);
+ if (s->kind == FLOW_FUNC_EXIT)
+ list_add_tail(&s->node, &env->flow->instances[s->st.instance_id]->exit_states);
+ env->max_states_per_insn = max(env->max_states_per_insn, ++*nr);
+}
+
+static void flow_link(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ flow_insert(env, s);
+ flow_set_pending(env, s);
+}
+
+/* The entry of @loop in the top frame of @st */
+static struct loop_stack_entry *flow_loop_entry(const struct bpf_verifier_state *st,
+ const struct bpf_loop *loop)
+{
+ struct bpf_func_state *frame = st->frame[st->curframe];
+ u32 i = frame->loop_stack_cnt;
+
+ while (frame->loop_stack[--i].loop != loop)
+ ;
+ return &frame->loop_stack[i];
+}
+
+/*
+ * @s came around @loop and arrives at @t. Does @s come from @t?
+ * ret = 0; if (...) { denied++; ret = -EPERM; }
+ * bpf_for(i, 0, n) { ... } return ret;
+ * Two states enter the loop. What comes around with 0 is for the state with 0. Next to -EPERM
+ * it looks like a number that grew.
+ */
+static bool flow_same_entry(struct bpf_loop *loop, const struct flow_state *t,
+ const struct flow_state *s)
+{
+ return s->kind == FLOW_FUNC_EXIT ||
+ flow_loop_entry(&t->st, loop)->entry_id == flow_loop_entry(&s->st, loop)->entry_id;
+}
+
+/*
+ * All states arrive here. @s is
+ * - pruned, if a state subsumes it,
+ * - joined with a state, unless the join rule keeps them apart,
+ * - pending otherwise.
+ * @loop: @s came around it, or @enters its leftover.
+ */
+static int flow_prune_or_join(struct bpf_verifier_env *env, struct flow_state *s,
+ struct bpf_loop *loop, bool enters)
+{
+ bool log = (env->log.level & BPF_LOG_LEVEL2) && s->kind == FLOW_AT, widen, subsumed;
+ int insn_idx = s->st.insn_idx, prev = s->prev_insn_idx, err = 0;
+ struct loop_stack_entry *e;
+ struct flow_state *t, *tn;
+ bool joined = false, widened = false;
+ struct bpf_loop *leftover_loop;
+
+again:
+ err = err ?: bpf_check_limits(env);
+ if (err)
+ goto free;
+
+ flow_for_each_near(env, s, t, tn) {
+ if (!flow_same_join_key(env, t, s))
+ continue;
+ /* an exit state can get into the function again */
+ leftover_loop = s->kind == FLOW_FUNC_EXIT && t->pending ? NULL : loop;
+ e = leftover_loop ? flow_loop_entry(&s->st, leftover_loop) : NULL;
+ if (bpf_must_keep_apart(env, &t->st, &s->st,
+ e && flow_same_entry(leftover_loop, t, s) ?
+ BPF_JOIN_POINT_HEADER : BPF_JOIN_POINT, e))
+ continue;
+ if (e && e->peeled)
+ leftover_loop = NULL;
+ subsumed = bpf_state_subsumes(env, &t->st, &s->st);
+ if (!subsumed && t->st.speculative != s->st.speculative)
+ continue;
+ if (subsumed) {
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "from %d to %d: pruned\n", prev, insn_idx);
+ err = flow_check_trip_counts(env, t, s);
+ goto free;
+ }
+ /* what grows in a loop is widened, it is not replaced */
+ if (!leftover_loop && bpf_state_subsumes(env, &s->st, &t->st)) {
+ err = flow_check_trip_counts(env, s, t);
+ if (err)
+ goto free;
+ /* it goes on with the leftover that @t started */
+ if (s->kind == FLOW_LEFTOVER)
+ bpf_top_loop(s->st.frame[s->st.curframe])->entry_id =
+ bpf_top_loop(t->st.frame[t->st.curframe])->entry_id;
+ flow_free_state(env, t);
+ joined = true;
+ continue;
+ }
+ joined = true;
+ widened = leftover_loop;
+ widen = !t->pending && !enters;
+ flow_unlink(env, t);
+ err = bpf_states_join(env, &t->st, &s->st, leftover_loop, widen);
+ swap(s->st, t->st);
+ s->st.last_insn_idx = prev;
+ flow_free_state(env, t);
+ /* another state can subsume the join, or have a join with it */
+ goto again;
+ }
+
+ /*
+ * The state is printed when its turn comes. That says enough about a state that
+ * is new, and about a state that goes on to the next insn.
+ */
+ if (prev >= 0 && insn_idx == prev + 1 + bpf_is_ldimm64(&env->prog->insnsi[prev]) &&
+ !widened)
+ joined = false;
+ if (log && joined)
+ verbose(env, "from %d to %d: %s\n", prev, insn_idx, widened ? "widened" : "joined");
+ flow_link(env, s);
+ return 0;
+free:
+ flow_free_state(env, s);
+ return err;
+}
+
+/* @s is taken. @starts_leftover: the states of FLOW_LEFTOVER are joined, start the leftover */
+static int flow_add(struct bpf_verifier_env *env, struct flow_state *s, bool starts_leftover)
+{
+ struct bpf_verifier_state *cur = env->cur_state;
+ int cur_idx = env->insn_idx, err;
+ struct bpf_loop *loop = NULL;
+ u32 id_gen = env->id_gen;
+ bool leftover = false;
+
+ env->cur_state = &s->st;
+ env->insn_idx = s->st.insn_idx;
+ s->kind = FLOW_AT;
+ err = starts_leftover ? bpf_flow_start_leftover(env) :
+ bpf_flow_loop_enter(env, &loop, &leftover);
+ env->cur_state = cur;
+ env->insn_idx = cur_idx;
+ if (err < 0)
+ err = bpf_flow_edge_unproven(env, &s->st, s->st.insn_idx, err);
+ /* a pointer to the packet that moves in the loop got an id */
+ if (!err && env->id_gen != id_gen)
+ err = bpf_renumber_ids(env, &s->st);
+ if (!err && env->path_sensitive) {
+ if (test_bit(s->st.insn_idx, env->flow->join_points))
+ err = bpf_failing_path_was_here(env, &s->st);
+ } else if (!err && loop && flow_loop_entry(&s->st, loop)->peeled) {
+ err = flow_prune_peeled(env, s, loop);
+ }
+ if (err) {
+ flow_free_state(env, s);
+ return min(err, 0);
+ }
+ if (env->path_sensitive) {
+ flow_link(env, s);
+ return 0;
+ }
+ if (leftover)
+ s->kind = FLOW_LEFTOVER;
+ if (starts_leftover)
+ loop = env->insn_aux_data[s->st.insn_idx].loop;
+ return flow_prune_or_join(env, s, loop, starts_leftover);
+}
+
+/*
+ * @st gets to @insn_idx. The flow takes what it has and leaves @st empty.
+ * @again: the insn is checked again, for a part of the values. There is no join.
+ */
+int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ int insn_idx, int prev_insn_idx, bool again)
+{
+ struct flow_state *s;
+ int err;
+
+ if (insn_idx < 0 || insn_idx >= env->prog->len) {
+ verbose(env, "invalid insn idx %d insn_cnt %d\n", insn_idx, env->prog->len);
+ return -EFAULT;
+ }
+
+ /* the check starts */
+ if (!env->flow->instance_cnt && !flow_new_instance(env, NULL))
+ return -ENOMEM;
+ s = flow_new_state(env, prev_insn_idx);
+ if (!s)
+ return -ENOMEM;
+ s->st = *st;
+ memset(st, 0, sizeof(*st));
+ s->st.insn_idx = insn_idx;
+ s->st.last_insn_idx = prev_insn_idx;
+
+ if (env->path_sensitive && (s->st.infeasible || s->st.speculative ||
+ bpf_failing_path_hops(env, &s->st) == U32_MAX)) {
+ flow_free_state(env, s);
+ return 0;
+ }
+ err = bpf_clean_verifier_state(env, &s->st);
+ if (err) {
+ flow_free_state(env, s);
+ return err;
+ }
+ if (again) {
+ s->kind = FLOW_AGAIN;
+ flow_find(env, s);
+ flow_link(env, s);
+ return 0;
+ }
+ return flow_add(env, s, false);
+}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3d4c52c59c7..710c3113e61 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -196,7 +196,6 @@ struct bpf_verifier_stack_elem {
struct bpf_verifier_state st;
int insn_idx;
int prev_insn_idx;
- /* it goes there only if may_goto in front of prev_insn_idx does, see check_loop_guard() */
enum bpf_guard_edge guard_edge;
bool again; /* see bpf_flow_deliver() */
struct bpf_verifier_stack_elem *next;
@@ -11949,6 +11948,7 @@ static int split_func_set(struct bpf_verifier_env *env, int regno)
branch = push_stack(env, env->insn_idx, env->insn_idx, false);
if (IS_ERR(branch))
return PTR_ERR(branch);
+ env->head->again = true;
}
return 0;
}
@@ -11974,6 +11974,7 @@ static int check_func_callx_set(struct bpf_verifier_env *env, int regno, int *in
branch = push_stack(env, *insn_idx, *insn_idx, false);
if (IS_ERR(branch))
return PTR_ERR(branch);
+ env->head->again = true;
continue;
}
copy = kzalloc_obj(*copy, BPF_VERIFIER_GFP);
@@ -21221,12 +21222,9 @@ static int flow_drain(struct bpf_verifier_env *env)
while ((elem = env->head)) {
env->head = elem->next;
- if (!err && elem->guard_edge)
- err = bpf_flow_guard_edge(env, &elem->st, elem->insn_idx,
- elem->prev_insn_idx, elem->guard_edge);
- else if (!err)
+ if (!err)
err = bpf_flow_deliver(env, &elem->st, elem->insn_idx,
- elem->prev_insn_idx);
+ elem->prev_insn_idx, elem->again);
bpf_free_verifier_state(env, &elem->st, false);
kfree(elem);
}
@@ -21388,11 +21386,14 @@ static void free_pushed_states(struct bpf_verifier_env *env)
static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
{
struct bpf_verifier_state *state = env->cur_state;
+ struct bpf_verifier_stack_elem *elem;
struct bpf_insn *insns = env->prog->insnsi;
int insn_cnt = env->prog->len;
bool do_print_state = true, from_unproven;
+ bool again = prev_insn_idx == env->insn_idx;
u32 held;
+ mark_verifier_state_scratched(env);
for (;;) {
struct bpf_insn *insn;
struct bpf_insn_aux_data *insn_aux;
@@ -21556,20 +21557,23 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
bpf_is_mem_insn(insn) && BPF_MODE(insn->code) == BPF_MEM) {
process_bpf_exit:
account_current_path(env);
- mark_verifier_state_scratched(env);
bpf_free_verifier_state(env, state, false);
memset(state, 0, sizeof(*state));
return flow_drain(env);
}
+ /* if the insn pushed a part, the current state is a part too */
+ for (elem = env->head; elem; elem = elem->next)
+ again |= elem->again;
err = flow_drain(env);
if (err)
return err;
- if (env->insn_idx < insn_cnt && bpf_flow_is_point(env, env->insn_idx)) {
+ /* the parts of a state meet at the next insn */
+ if (env->insn_idx < insn_cnt &&
+ (again || bpf_flow_is_bb_start(env, env->insn_idx))) {
account_current_path(env);
- mark_verifier_state_scratched(env);
- return bpf_flow_deliver(env, state, env->insn_idx, prev_insn_idx);
+ return bpf_flow_deliver(env, state, env->insn_idx, prev_insn_idx, false);
}
}
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 29/49] bpf: Order the pending states and free the states that are done
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (27 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 28/49] bpf: Prune or join each state that arrives Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 30/49] bpf: Get the trip count from the trip index Alexei Starovoitov
` (19 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The order decides how often an insn is checked. A join point has to get
all its states before it runs:
if (c) x = 1; else x = 2; both branches first,
use(x); then one check with x = 1..2
flow_set_pending() sets the order:
1. a function that a call waits for,
2. callers, so that the calls of a function meet before it is checked,
3. the trip and the position in each frame: a trip is complete before
the next trip, a loop before the code after it,
4. exit states last.
With this order most states can meet no later state. They are freed
when their turn comes. flow_keep() keeps a state at a join point
- in the leftover of a loop, until the states leave the loop,
- at the start of a peeled trip (0, 1, 2, 4, 8, ...), to find a trip
that repeats a state,
- at the entry of a function, or while a call can get more exit states,
- in an async callback, because it can start again.
Without that the memory grows with the number of peeled trips.
do_check() takes the next state and checks its basic block.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 10 ++
kernel/bpf/flow.c | 293 +++++++++++++++++++++++++++++++++++
kernel/bpf/verifier.c | 41 +++++
3 files changed, 344 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index f153e5e7c28..a310ad50941 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -453,6 +453,12 @@ struct loop_stack_entry {
bool trip_widened; /* it was moved to a threshold, or it does not grow */
};
+/* The states of trip 0, 1, 2, 4, 8, ... are saved. A compare with them finds any period. */
+static inline bool bpf_trip_is_saved(const struct loop_stack_entry *entry)
+{
+ return !(entry->trip & (entry->trip - 1));
+}
+
#define LOOP_STACK_SIZE 16
/* state of the program:
@@ -831,6 +837,7 @@ struct bpf_loop_per_pass {
bool has_leftover; /* the peeled trips were not enough */
/* a check failed in a state that the loop kept apart, with no other suspect */
bool has_unproven;
+ u32 last_peeled_trip; /* see flow_keep_peeled() */
u32 peeled_cnt; /* trips that the pass peeled */
u64 cost; /* for the states that the loop keeps apart */
};
@@ -1329,10 +1336,12 @@ struct bpf_verifier_env {
bool uses_arena_sp; /* the program has insns with BPF_REG_ARENA_SP */
bool path_sensitive; /* no joins, see run_path_sensitive() */
bool for_log; /* the verdict is known, see do_check_all() */
+ bool fallthrough_first; /* see find_failing_path() */
struct bpf_precision *precision;
u64 cost, mem; /* see bpf_check_limits() */
u64 max_cost, max_mem;
u64 hash_seed; /* see bpf_hash_join_key() */
+ u64 failing_path_budget, failing_path_max_cost; /* see find_failing_path() */
u64 pass_max_cost; /* see bpf_undo_precision() */
bool aborted; /* a thaw does not take it back */
bool no_mem; /* a table cannot grow, see bpf_check_limits() */
@@ -1712,6 +1721,7 @@ void bpf_flow_free(struct bpf_verifier_env *env);
bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx);
int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
int insn_idx, int prev_insn_idx, bool again);
+int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos, u64 *diag_pos);
int bpf_flow_call(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
struct bpf_verifier_state *entry, int subprog, int call_idx);
int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int exit_idx);
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 9c96f10f3ac..5fbbb09a665 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -642,6 +642,82 @@ bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx)
return test_bit(insn_idx, env->flow->bb_starts);
}
+/* A peeled trip is complete before the next one starts. The leftover is last. */
+static int flow_cmp_trips(const struct bpf_func_state *fa, const struct bpf_func_state *fb)
+{
+ const struct loop_stack_entry *ea = fa->loop_stack, *eb = fb->loop_stack;
+ u32 i, ta, tb;
+
+ for (i = 0; i < fa->loop_stack_cnt && i < fb->loop_stack_cnt; i++, ea++, eb++) {
+ if (ea->loop != eb->loop)
+ break;
+ ta = ea->peeled ? ea->trip : U32_MAX;
+ tb = eb->peeled ? eb->trip : U32_MAX;
+ if (ta != tb)
+ return ta < tb ? -1 : 1;
+ }
+ return 0;
+}
+
+static bool flow_less(struct rb_node *na, const struct rb_node *nb)
+{
+ const struct flow_state *a = rb_entry(na, struct flow_state, rb);
+ const struct flow_state *b = rb_entry(nb, struct flow_state, rb);
+ int cmp;
+ u32 i;
+
+ if (a->priority != b->priority)
+ return a->priority < b->priority;
+ for (i = 0; i < a->order_len && i < b->order_len; i++) {
+ cmp = flow_cmp_trips(a->st.frame[i], b->st.frame[i]);
+ if (cmp)
+ return cmp < 0;
+ if (a->order[i] != b->order[i])
+ return a->order[i] < b->order[i];
+ }
+ if (a->order_len != b->order_len)
+ return a->order_len < b->order_len;
+ /* a speculative state gets a barrier if the fixups of the others do not fit */
+ if (a->st.speculative != b->st.speculative)
+ return b->st.speculative;
+ return a->seq < b->seq;
+}
+
+/*
+ * The order of pending states:
+ * 1. An instance that a call waits for. The call has no exit state to continue. It waits at
+ * its position in the order. The code after the call waits too.
+ * 2. Callers. So the calls of a function get to it before it is checked.
+ * 3. The trip, then the position, in each frame. A callee in the frames of its callers is
+ * complete before the code after the call.
+ * 4. Exit states. They go to the calls when the instance is complete.
+ *
+ * A path-sensitive run goes depth first, to the state that is nearer to an unproven insn.
+ */
+static void flow_set_pending(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ const struct bpf_verifier_state *st = &s->st;
+ struct bpf_flow *flow = env->flow;
+ struct flow_instance *instance = flow_state_instance(flow, s);
+ u32 i;
+
+ s->priority = instance->priority ?:
+ BIT_ULL(63) | (u64)flow->rank[st->frame[0]->subprogno] << 32 | ~instance->id;
+ for (i = 0; i < st->curframe; i++)
+ s->order[i] = flow->pos[st->frame[i + 1]->callsite];
+ s->order[i] = s->kind == FLOW_FUNC_EXIT ? U32_MAX : flow->pos[st->insn_idx];
+ s->order_len = i + 1;
+ if (env->path_sensitive) {
+ s->priority = (u64)~env->insn_processed << 32 | (env->fallthrough_first ?
+ st->insn_idx != s->prev_insn_idx + 1 :
+ bpf_failing_path_hops(env, st));
+ s->order_len = 0;
+ }
+ s->pending = true;
+ instance->pending_cnt++;
+ rb_add_cached(&s->rb, &flow->pending, flow_less);
+}
+
/*
* States meet only if they have the same hash. The tree sorts them by a half of the hash,
* then by the range of one xreg of data_deps. Two ranges with a gap stay apart, but not where
@@ -674,6 +750,125 @@ static struct flow_state *flow_find(struct bpf_verifier_env *env, struct flow_st
for (t = flow_find(env, s); \
t && (tn = flow_tree_iter_next(t, (s)->first, (s)->last), true); t = tn)
+/*
+ * The loop that @st came around or entered, if @st is at a header. One insn can be the header
+ * of a jump loop and of the calls of a callback.
+ */
+static struct loop_stack_entry *flow_header_entry(const struct bpf_verifier_state *st)
+{
+ const struct bpf_func_state *frame = st->frame[st->curframe];
+ struct loop_stack_entry *e = NULL;
+ u32 i = frame->loop_stack_cnt;
+
+ while (i-- && frame->loop_stack[i].loop->header == st->insn_idx && !(e && e->trip) &&
+ frame->loop_stack[i].loop->kind != BPF_LOOP_RECURSION)
+ e = &frame->loop_stack[i];
+ return e;
+}
+
+/* If more than one state starts a peeled trip, paths of other lengths can come to one state */
+static bool flow_keep_peeled(struct bpf_verifier_env *env, struct loop_stack_entry *e)
+{
+ bool more = e->loop->this_pass.last_peeled_trip == e->trip;
+
+ e->loop->this_pass.last_peeled_trip = e->trip;
+ if (!more && !bpf_trip_is_saved(e))
+ return false;
+ e->entry_id = ++env->loop_entry_cnt;
+ return true;
+}
+
+/*
+ * @s is the next pending state. Can a state with the same join_key arrive later?
+ * - In the leftover of a loop: until the states leave the loop.
+ * - At the start of a peeled trip: later trips are compared with it.
+ * - A call can get more exit states: until all calls have them and the late states went by.
+ * In a peeled trip the call waits.
+ * - An async callback can start again: until the check ends.
+ */
+static bool flow_keep(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ const struct bpf_verifier_state *st = &s->st;
+ struct loop_stack_entry *e, *header_entry = flow_header_entry(st);
+ struct flow_instance *instance = env->flow->instances[st->instance_id];
+ bool in_loop = false, keep;
+ u32 fr, i;
+
+ if (!test_bit(st->insn_idx, env->flow->join_points) || env->path_sensitive ||
+ s->kind == FLOW_AGAIN)
+ return false;
+ if (s->started_instance)
+ return true;
+ s->late_exit_reaches = true;
+ s->kept_frameno = U8_MAX;
+ for (fr = 0; fr <= st->curframe; fr++) {
+ e = st->frame[fr]->loop_stack;
+ for (i = 0; i < st->frame[fr]->loop_stack_cnt; i++, e++) {
+ if (s->kind == FLOW_LEFTOVER && e == bpf_top_loop(st->frame[st->curframe]))
+ break;
+ if (e == header_entry && e->peeled && s->kind == FLOW_AT &&
+ flow_keep_peeled(env, e))
+ s->kind = FLOW_PEELED;
+ keep = !e->peeled || (e == header_entry && s->kind == FLOW_PEELED);
+ if (e->loop->kind != BPF_LOOP_RECURSION && (e->peeled || !in_loop)) {
+ s->late_exit_reaches &= !e->peeled;
+ s->kept_frameno = fr;
+ s->kept_depth = i;
+ in_loop = keep;
+ }
+ if (e == header_entry && keep)
+ break;
+ }
+ }
+ if (s->late_exit_reaches && st->frame[0]->in_async_callback_fn)
+ return true;
+ keep = in_loop || (s->late_exit_reaches && instance->open_call_cnt);
+ if (keep)
+ list_add_tail(&s->node, &instance->done);
+ return keep;
+}
+
+/* Later trips find @s without the number of its trip */
+static void flow_insert_peeled(struct bpf_verifier_env *env, struct flow_state *s)
+{
+ flow_header_entry(&s->st)->trip = 0;
+ flow_tree_remove(s, &env->flow->state_tree);
+ flow_find(env, s);
+ flow_tree_insert(s, &env->flow->state_tree);
+}
+
+/* @next is the next pending state. Did the states leave the loop that @s is kept for? */
+static bool flow_loop_is_left(struct flow_state *s, const struct flow_state *next)
+{
+ const struct loop_stack_entry *es, *en;
+ const struct bpf_func_state *fs, *fn;
+ u32 fr, i;
+
+ /* a call returned late */
+ if (!flow_less(&s->rb, &next->rb))
+ return false;
+ if (next->st.curframe < s->kept_frameno)
+ return true;
+ for (fr = 0; fr <= s->kept_frameno; fr++) {
+ fs = s->st.frame[fr];
+ fn = next->st.frame[fr];
+ if (fs->subprogno != fn->subprogno || fs->callsite != fn->callsite)
+ return true;
+ for (i = 0;
+ i < (fr < s->kept_frameno ? fs->loop_stack_cnt : s->kept_depth + 1); i++) {
+ es = &fs->loop_stack[i];
+ en = &fn->loop_stack[i];
+ if (i >= fn->loop_stack_cnt || es->loop != en->loop)
+ return true;
+ if (es->peeled && (!en->peeled || es->trip != en->trip) &&
+ !(s->kind != FLOW_AT && en->peeled && fr == s->kept_frameno &&
+ i == s->kept_depth))
+ return true;
+ }
+ }
+ return false;
+}
+
static struct flow_instance *flow_new_instance(struct bpf_verifier_env *env,
struct flow_state *entry)
{
@@ -935,3 +1130,101 @@ int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st
}
return flow_add(env, s, false);
}
+
+/*
+ * Make the next pending state the current one. env->cur_state has to be empty.
+ * Returns -ENOENT when nothing is pending.
+ */
+int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos, u64 *diag_pos)
+{
+ struct bpf_flow *flow = env->flow;
+ struct flow_state *s, *t, *tn;
+ struct flow_instance *instance;
+ struct rb_node *n;
+ bool leftover, starts;
+ int err = 0;
+
+again:
+ if (flow->last_instance && flow->last_instance->cost)
+ *flow->last_instance->cost += env->cost - flow->charged_cost;
+ flow->charged_cost = env->cost;
+ if (flow->last_instance)
+ flow_end_instance(env, flow->last_instance);
+ n = rb_first_cached(&flow->pending);
+ if (!n)
+ return -ENOENT;
+ s = rb_entry(n, struct flow_state, rb);
+ flow_clear_pending(flow, s);
+ flow->last_instance = flow_state_instance(flow, s);
+ instance = s->callee;
+ if (instance) {
+ if (instance->entry_state && !instance->priority) {
+ instance->priority = BIT_ULL(63) - ++flow->priority_gen;
+ flow_clear_pending(flow, instance->entry_state);
+ flow_set_pending(env, instance->entry_state);
+ }
+ goto again;
+ }
+ instance = flow->instances[s->st.instance_id];
+ while (!s->started_instance && !list_empty(&instance->done)) {
+ t = list_last_entry(&instance->done, struct flow_state, node);
+ if ((t->late_exit_reaches && instance->open_call_cnt) || !flow_loop_is_left(t, s))
+ break;
+ flow_free_state(env, t);
+ }
+
+
+ env->insn_idx = s->st.insn_idx;
+ if (s->kind == FLOW_FUNC_EXIT) {
+ list_for_each_entry_safe(t, tn, &instance->waiting_calls, node) {
+ err = flow_apply_exit(env, t, s);
+ if (err)
+ return err;
+ }
+ goto again;
+ }
+ *prev_insn_idx = s->prev_insn_idx;
+ if (env->path_sensitive) {
+ *log_pos = s->log_pos;
+ *diag_pos = s->diag_pos;
+ }
+ leftover = s->kind == FLOW_LEFTOVER;
+ starts = leftover || s->started_instance;
+ if (!flow_keep(env, s)) {
+ flow_unlink(env, s);
+ } else if (!starts) {
+ err = bpf_copy_verifier_state(env, env->cur_state, &s->st);
+ if (s->kind != FLOW_AT)
+ flow_insert_peeled(env, s);
+ goto out;
+ } else {
+ t = s;
+ s = flow_new_state(env, t->prev_insn_idx);
+ if (!s)
+ return -ENOMEM;
+ err = bpf_copy_verifier_state(env, &s->st, &t->st);
+ if (err) {
+ flow_free_state(env, s);
+ return err;
+ }
+ if (t->started_instance) {
+ s->st.instance_id = t->started_instance->id;
+ t->started_instance->entry_state = NULL;
+ }
+ }
+ if (starts) {
+ err = flow_add(env, s, leftover);
+ if (err)
+ return err;
+ goto again;
+ }
+ swap(*env->cur_state, s->st);
+ flow_free_state(env, s);
+out:
+ /* all ids have names, see bpf_renumber_ids() */
+ env->id_gen = 0;
+ /* the state at the header stays as it was widened, else the sequence possibly never ends */
+ if (!err && env->cur_state->equations.needs_refine)
+ bpf_equations_refine_all(env, env->cur_state);
+ return err;
+}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 710c3113e61..cb79c91ae1c 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21584,6 +21584,47 @@ static bool round_is_dropped(struct bpf_verifier_env *env)
return (env->restart && bpf_flow_loops_lost_bound(env)) || env->guards_late;
}
+static int do_check(struct bpf_verifier_env *env)
+{
+ bool level2 = env->log.level & BPF_LOG_LEVEL2;
+ u64 log_pos = env->log.end_pos, diag_pos = 0, cost = env->cost;
+ int prev_insn_idx, err;
+
+ err = bpf_flow_deliver(env, env->cur_state, env->insn_idx, -1, false);
+ while (!err && !(env->path_sensitive && env->cost >= env->failing_path_max_cost)) {
+ err = bpf_flow_next(env, &prev_insn_idx, &log_pos, &diag_pos);
+ /* at the limit the work on equations is not complete, such a pass proves nothing */
+ if (err == -ENOENT)
+ return bpf_check_limits(env);
+ if (err)
+ break;
+ charge_loops(env, env->cost - cost);
+ cost = env->cost;
+ /* A pass logs one basic block at a time. A path-sensitive run logs a full path. */
+ bpf_diag_event_log_restore(env, diag_pos);
+ if (!level2) {
+ bpf_vlog_reset(&env->log, log_pos);
+ env->prev_linfo = NULL;
+ }
+ err = do_check_block(env, prev_insn_idx);
+ if (env->bug)
+ return -EFAULT;
+ /* a pass had no error at this insn, possibly with a state that has more values */
+ if (err && env->path_sensitive && is_error_of_state(env, err) &&
+ (env->cur_state->infeasible || env->cur_state->equations.infeasible ||
+ !bpf_failing_path_may_end_at(env, env->insn_idx))) {
+ bpf_free_verifier_state(env, env->cur_state, false);
+ memset(env->cur_state, 0, sizeof(*env->cur_state));
+ free_pushed_states(env);
+ err = 0;
+ }
+ }
+ /* bpf_flow_edge_unproven() noted it with the state that failed */
+ if (err && !bpf_insn_is_unproven(env))
+ note_error(env, err);
+ return err;
+}
+
static int find_btf_percpu_datasec(struct btf *btf)
{
const struct btf_type *t;
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 30/49] bpf: Get the trip count from the trip index
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (28 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 29/49] bpf: Order the pending states and free the states that are done Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 8:22 ` sashiko-bot
2026-11-20 7:42 ` [PATCH bpf-next 31/49] bpf: Give a loop a checked trip count or may_goto Alexei Starovoitov
` (18 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A loop needs a trip count or may_goto. The trip index k is an xreg. It
is 0 at the entry, and each back edge adds 1. The join at the header
finds what moves with k. A jump that tests such an xreg refines k.
for (i = 0; i < 100; i += 4)
join i = 4 * k
threshold 'i < 100' holds up to k = 24: widen k to 0..25
jump i < 100 -> k 0..24
back edge k + 1 is 1..25, the header state subsumes it
This is the proof: k grows in each trip and stays in 0..25. The next
patch checks that. No jump has to test p here, its equation limits it:
for (i = 0, p = buf; i < 8; i++, p += 8) p = buf + 8 * k
*p = 0;
A threshold is a guess, see bpf_collect_iv_thresholds(). If it is too
small, k grows to the next one, and at last to S64_MAX.
Two states can enter a loop with two bounds. So each entry has the trip
index at which its jumps leave the loop, see exit_threshold:
(n, size) = (1 << 18, 4) or (1 << 17, 8)
for (i = 0; i != n; i++)
buf[i * size]; 1 << 18 is too much for size 8
- The trip count is known only when the pass ends. So pass 0 takes the
edges of may_goto in all loops. After it only the loops that need
may_goto keep them.
- 'i != n' with two ranges: add_distance_equation() makes a sym for
i - n in the first trip. It does that for two pointers into one
object too: for (p = t + 0..3; p != t + 4..7; p++)
- A test against the rest of a buffer gets a sym for the difference
too, in a loop or not:
if (len > 64 - off) return; read(buf + off, len);
An equation has a cost, so len has to be in data_deps. If it is not,
the pass is imprecise, and a later pass makes the sym.
- With -mcpu=v1 and v2 clang can count down and test the carry:
x = (u32)x + 0xffffffff; if (x != (u32)x) continue;
The two operands have one step. If the carry is set in the first
trips, the jump gives the threshold of 'x > U32_MAX'.
- A distance of two truncated numbers can have a range of 2^31 after a
widening. clamp_diff_start() takes the ranges of the two numbers.
- bpf_note_linear_xregs() notes the xregs that are linear in k, and
those that are linear only at first: i = (i + 1) & 63
- A leftover is walked more than once. The new walk makes all values
again. So flow_is_stale() frees a state of an earlier walk.
- With may_goto k has no bound. trip_index_safe() lets it start again
from a smaller number, so the states can be pruned.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 20 +++
kernel/bpf/equation.c | 128 ++++++++++++++++
kernel/bpf/equation.h | 5 +
kernel/bpf/equation_join.c | 274 +++++++++++++++++++++++++++++++++++
kernel/bpf/flow.c | 24 +++
kernel/bpf/precision.c | 10 ++
kernel/bpf/states.c | 18 +++
kernel/bpf/verifier.c | 24 +++
8 files changed, 503 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index a310ad50941..be499471e12 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -427,6 +427,9 @@ static inline u32 bpf_xreg_to_spi(u32 xregno)
/* The trip index of a loop is an xreg too. The loops of a frame come after its slots. */
#define BPF_XREG_TRIP MAX_BPF_XREG
+/* A loop that can make more trips gets may_goto */
+#define BPF_MAX_TRIPS BIT_ULL(32)
+
static inline u64 bpf_xreg(u32 frameno, u32 xregno)
{
return (u64)frameno << 32 | xregno;
@@ -451,6 +454,7 @@ struct loop_stack_entry {
/* an xreg: how many times the program came back to the header */
struct scalar trip_index;
bool trip_widened; /* it was moved to a threshold, or it does not grow */
+ u64 exit_threshold; /* by the jumps that the state passed and that leave the loop */
};
/* The states of trip 0, 1, 2, 4, 8, ... are saved. A compare with them finds any period. */
@@ -834,6 +838,7 @@ enum bpf_loop_kind {
/* What a pass finds about a loop */
struct bpf_loop_per_pass {
+ bool needs_guard; /* see bpf_flow_needs_guard() */
bool has_leftover; /* the peeled trips were not enough */
/* a check failed in a state that the loop kept apart, with no other suspect */
bool has_unproven;
@@ -868,10 +873,17 @@ struct bpf_loop {
bool peeled_apart; /* the join rule holds for states that start a peeled trip */
bool disjoint_apart; /* disjoint values in data_deps stay apart, also in the leftover */
);
+ struct bpf_loop_per_pass last_pass; /* the last pass whose partition was kept */
bool partitioned; /* the last partition was for this loop */
bool peeled_joined; /* at the start of a peeled trip, against the join rule */
+ bool guarded; /* states take the edges of may_goto */
struct bpf_iv_threshold *iv_thresholds; /* where an iv can stop, see equation_join.c */
u32 iv_threshold_cnt, iv_threshold_cap;
+ u32 leftover_epoch; /* how many leftovers started */
+ u64 exit_trip; /* the trip in which the loop can end, 0 if it is not known */
+ /* they move with the trip index, or only at first, see bpf_note_linear_xregs() */
+ DECLARE_BITMAP(linear_xregs, MAX_BPF_XREG);
+ DECLARE_BITMAP(nonlinear_xregs, MAX_BPF_XREG);
struct bpf_loop_per_pass this_pass;
/* sorted constants of the loop, the largest and the smallest constant that an insn adds */
s64 *thresholds;
@@ -994,6 +1006,8 @@ struct bpf_insn_aux_data {
u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */
unsigned int orig_idx; /* original instruction index, initialized once */
+ u32 thresholds_epoch; /* see bpf_collect_iv_thresholds() */
+ u8 iv_operand; /* the operand of the jump that tests the iv, + 1 */
/* registers alive before this instruction. */
u16 live_regs_before;
/*
@@ -1678,6 +1692,8 @@ u64 bpf_hash_apart_xregs(struct bpf_verifier_env *env, const struct bpf_verifier
u64 bpf_hash_args(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 hash);
void bpf_range_of_data_dep(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 *first, u32 *last);
+bool bpf_xreg_in_data_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 fr, u32 xregno);
bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 fr, u32 xregno);
void bpf_note_reads_from_unproven(struct bpf_verifier_env *env, u32 idx);
@@ -1731,6 +1747,7 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env);
/* verifier.c, for flow.c */
int bpf_flow_loop_enter(struct bpf_verifier_env *env, struct bpf_loop **backedge_loop,
bool *starts_leftover);
+int bpf_flow_start_leftover(struct bpf_verifier_env *env);
int bpf_flow_edge_unproven(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 idx,
int err);
#define BPF_EXIT_NEEDS_CALLER 1 /* verify the call in the frames of the caller */
@@ -2432,6 +2449,8 @@ void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *fra
void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
bool taken);
+void bpf_equations_next_trip(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ u32 depth);
bool bpf_equations_subsume(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
const struct bpf_verifier_state *cur);
void bpf_equations_print(struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
@@ -2441,6 +2460,7 @@ void bpf_equations_pre_join(struct bpf_verifier_env *env, struct bpf_verifier_st
void bpf_equations_post_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
const struct bpf_verifier_state *src);
u32 bpf_equations_bytes(const struct bpf_equations *eqs);
+void bpf_collect_iv_thresholds(struct bpf_verifier_env *env);
void scalar_bounds_sync(struct scalar *s);
bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
bool scalars_fit_in_bits(const struct scalar *a, const struct scalar *b, u32 bits);
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index 3d9df6e6845..fc05ca7dd60 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -1132,6 +1132,91 @@ void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *fra
env->eq_env->fill_bits = size * BITS_PER_BYTE;
}
+/*
+ * The state came back to the header of the loop at @depth.
+ * A trip count or may_goto ends the loop long before the trip index gets to 2^63.
+ */
+void bpf_equations_next_trip(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 depth)
+{
+ struct scalar *trip_index = &st->frame[st->curframe]->loop_stack[depth].trip_index;
+ struct bpf_xreg_equation *xreg_eq;
+
+ scalar_add_const(trip_index, 1);
+ trip_index->r64 = cnum64_intersect(trip_index->r64, cnum64_from_urange(0, S64_MAX));
+ scalar_bounds_sync(trip_index);
+ xreg_eq = bpf_find_xreg_equation(&st->equations,
+ bpf_xreg(st->curframe, BPF_XREG_TRIP + depth));
+ if (!xreg_eq)
+ return;
+ xreg_eq->off++;
+ refine_sym_from_xreg(env, st, xreg_eq);
+ propagate_bounds(env, st, false);
+ bpf_equations_clean(env, st);
+}
+
+/*
+ * @rest is a constant minus a number, and not an iv that counts down. An equation has a cost,
+ * so wait until an unproven insn needs @len.
+ */
+static bool tests_rest(struct bpf_verifier_env *env, u32 len, u32 rest)
+{
+ const struct bpf_verifier_state *st = env->cur_state;
+ const struct bpf_xreg_equation *xreg_eq, *trip;
+ u32 i;
+
+ xreg_eq = bpf_find_xreg_equation(&st->equations, bpf_reg_to_xreg(st, rest));
+ if (!xreg_eq || xreg_eq->is_ptr || xreg_eq->coef > 0)
+ return false;
+ for (i = 0; i < st->frame[st->curframe]->loop_stack_cnt; i++) {
+ trip = bpf_find_xreg_equation(&st->equations,
+ bpf_xreg(st->curframe, BPF_XREG_TRIP + i));
+ if (trip && trip->sym == xreg_eq->sym)
+ return false;
+ }
+ if (env->path_sensitive || bpf_xreg_in_data_deps(env, st, st->curframe, len))
+ return true;
+ env->imprecise |= !env->err;
+ return false;
+}
+
+/*
+ * The ranges of two numbers do not show how far they are from each other:
+ * for (i = 0..3; i != n; i++) for (p = t + 0..3; p != t + n; p++)
+ * if (len > 64 - off) return; read(buf + off, len);
+ * Give the difference a sym. bpf_equations_jump() refines it if it does not wrap.
+ * For an iv and its bound the first trip makes the equation. The other trips find it.
+ */
+static void add_distance_equation(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+ struct bpf_verifier_state *st = env->cur_state;
+ const struct loop_stack_entry *e = bpf_top_loop(st->frame[st->curframe]);
+ struct expr a, b, diff;
+ struct scalar bounds;
+ struct equation_pos pos = {};
+ u8 op = BPF_OP(insn->code);
+ bool is_ptr, is_eq = op == BPF_JEQ || op == BPF_JNE;
+
+ if (op == BPF_JSET || !compares_numbers(st, insn))
+ return;
+ if (is_eq && (!e || (e->peeled && e->trip)))
+ return;
+ if (!is_eq && !tests_rest(env, insn->dst_reg, insn->src_reg) &&
+ !tests_rest(env, insn->src_reg, insn->dst_reg))
+ return;
+ bpf_equations_clean(env, st);
+ if (!assign_sym(env, bpf_reg_to_xreg(st, insn->dst_reg), true, &a, &is_ptr) ||
+ !assign_sym(env, bpf_reg_to_xreg(st, insn->src_reg), true, &b, &is_ptr) ||
+ bpf_find_next_diff(env, &st->equations, &a, &b, &pos, &diff))
+ return;
+ bpf_eval_diff(env, st, &a, &b, &bounds);
+ /* len can get its bound from this jump */
+ if (!is_eq || !scalar_is_unknown(&bounds))
+ bpf_add_sym_equation(env, &st->equations, (struct bpf_sym_equation){
+ { -1, a.coef, -b.coef },
+ { bpf_new_sym(env, &st->equations, &bounds), a.var, b.var },
+ env->insn_idx, a.off - b.off });
+}
+
/* Note the operands. Delete the equations of the registers that the insn writes. */
void bpf_equations_before_insn(struct bpf_verifier_env *env)
{
@@ -1169,6 +1254,9 @@ void bpf_equations_before_insn(struct bpf_verifier_env *env)
&eq_env->operand_expr[i], &eq_env->is_ptr[i]);
}
}
+ if (bpf_insn_is_cond_jump(insn->code) && BPF_SRC(insn->code) == BPF_X && !st->speculative &&
+ insn->dst_reg < BPF_REG_FP && insn->src_reg < BPF_REG_FP)
+ add_distance_equation(env, insn);
if (class == BPF_JMP && op == BPF_EXIT && fr && !st->frame[fr]->in_callback_fn)
copy_xreg_equation(env, eqs, bpf_xreg(fr - 1, BPF_REG_0), bpf_xreg(fr, BPF_REG_0));
if (class == BPF_JMP && op == BPF_CALL) {
@@ -2140,6 +2228,45 @@ bool bpf_equations_subsume(struct bpf_verifier_env *env, const struct bpf_verifi
/* Join */
+static void join_trip_index(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
+ const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
+ bool widen)
+{
+ const struct loop_stack_entry *s;
+ struct loop_stack_entry *e;
+ u64 xreg, threshold;
+ struct scalar old;
+ u32 fr, i;
+
+ for (fr = 0; fr <= dst->curframe; fr++) {
+ e = dst->frame[fr]->loop_stack;
+ s = src->frame[fr]->loop_stack;
+ for (i = 0; i < dst->frame[fr]->loop_stack_cnt; i++, e++, s++) {
+ xreg = bpf_xreg(fr, BPF_XREG_TRIP + i);
+ old = e->trip_index;
+ scalar_union(&e->trip_index, &s->trip_index);
+ /* an outer loop still has to find its thresholds */
+ e->trip_widened |= s->trip_widened || (widen && old.grew_once &&
+ e->loop->header == dst->insn_idx);
+ e->exit_threshold = e->loop == leftover_loop ? s->exit_threshold :
+ max(e->exit_threshold, s->exit_threshold);
+ if (!leftover_loop || scalar_equal(&e->trip_index, &old))
+ continue;
+ threshold = widen ? bpf_guess_trip_threshold(env, dst, e, xreg) : 0;
+ e->trip_index.grew_once = true;
+ if (!threshold || threshold < cnum64_umax(e->trip_index.r64))
+ continue;
+ e->trip_widened = true;
+ if (threshold != S64_MAX)
+ e->loop->exit_trip = max(e->loop->exit_trip, threshold + 1);
+ scalar_set_unknown(&old);
+ old.r64 = cnum64_from_urange(cnum64_umin(e->trip_index.r64), threshold);
+ scalar_bounds_sync(&old);
+ scalar_union(&e->trip_index, &old);
+ }
+ }
+}
+
/*
* Make the equations of @dst hold for @src too. Call it before the xregs are joined.
* @leftover_loop: the states are at its header. @widen: @src came around it.
@@ -2168,6 +2295,7 @@ void bpf_equations_pre_join(struct bpf_verifier_env *env, struct bpf_verifier_st
src->equations.needs_refine;
swap(dst->equations, *eqs);
}
+ join_trip_index(env, dst, src, leftover_loop, widen);
}
/* The xregs of @dst are joined. Some of them are gone. */
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index c88740ffb42..b91f5cf5c49 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -190,4 +190,9 @@ void bpf_eval_diff(struct bpf_verifier_env *env, const struct bpf_verifier_state
void bpf_guess_equations(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
const struct bpf_verifier_state *src, struct bpf_loop *leftover_loop,
bool widen);
+void bpf_note_linear_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
+ const struct bpf_equations *join);
+u64 bpf_guess_trip_threshold(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
+ const struct loop_stack_entry *e, u64 xreg);
+
#endif
diff --git a/kernel/bpf/equation_join.c b/kernel/bpf/equation_join.c
index d47c4f04453..1df6a1cce03 100644
--- a/kernel/bpf/equation_join.c
+++ b/kernel/bpf/equation_join.c
@@ -485,6 +485,28 @@ static void join_bounds_by_selector(struct equation_join *j, u64 var, bool alway
join_diff(j, make_expr(1, var, 0), make_expr(max_slope, j->selector, 0), false);
}
+/* Where the trip index of @e in @dst can stop. 0: it is too early to know. */
+u64 bpf_guess_trip_threshold(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
+ const struct loop_stack_entry *e, u64 xreg)
+{
+ const struct bpf_iv_threshold *threshold;
+ u64 value = S64_MAX;
+ int kind;
+
+ /* without an equation nothing limits it */
+ if (!bpf_find_xreg_equation(&dst->equations, xreg))
+ return value;
+ if (!e->trip_index.grew_once)
+ return 0;
+ if (e->exit_threshold < value && e->exit_threshold >= cnum64_umax(e->trip_index.r64))
+ return e->exit_threshold;
+ for (kind = THRESHOLD_EXIT_JUMP; kind <= THRESHOLD_BODY_JUMP && value == S64_MAX; kind++)
+ for_each_iv_threshold(env, e->loop, threshold, kind)
+ if (threshold->value >= cnum64_umax(e->trip_index.r64))
+ value = min_t(u64, value, threshold->value);
+ return value;
+}
+
/*
* Guess the equations that hold for @dst and @src, see bpf_equations_pre_join().
* The first join shows what a trip moves. The next walk finds the thresholds.
@@ -539,3 +561,255 @@ void bpf_guess_equations(struct bpf_verifier_env *env, const struct bpf_verifier
for (i = 0, sym_cnt = j.eqs->sym_cnt; j.selector != VAR_NONE && i < sym_cnt; i++)
join_bounds_by_selector(&j, VAR_SYM | i, selector_is_trip);
}
+
+/*
+ * Which xregs move with the trip index in @dst? Some do not in the equations of the @join:
+ * i = (i + 1) & 63;
+ */
+void bpf_note_linear_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *dst,
+ const struct bpf_equations *join)
+{
+ const struct bpf_func_state *f = dst->frame[dst->curframe];
+ u64 trip_xreg = bpf_xreg(dst->curframe, BPF_XREG_TRIP + f->loop_stack_cnt - 1);
+ const struct bpf_xreg_equation *old_trip =
+ bpf_find_xreg_equation(&dst->equations, trip_xreg), *xreg_eq, *new;
+ const struct bpf_xreg_equation *new_trip = bpf_find_xreg_equation(join, trip_xreg);
+
+ if (!f->loop_stack_cnt || !old_trip || !new_trip)
+ return;
+ for_each_xreg_equation(env, &dst->equations, xreg_eq) {
+ if (xreg_eq->sym != old_trip->sym || xreg_eq->xreg >> 32 != dst->curframe ||
+ (u32)xreg_eq->xreg >= BPF_XREG_TRIP)
+ continue;
+ new = bpf_find_xreg_equation(join, xreg_eq->xreg);
+ if (!new || new->sym != new_trip->sym)
+ __set_bit((u32)xreg_eq->xreg, loop->nonlinear_xregs);
+ else if (!__test_and_set_bit((u32)xreg_eq->xreg, loop->linear_xregs))
+ env->new_linear_xreg = true;
+ }
+}
+
+/* Thresholds */
+
+/*
+ * A jump gives thresholds in many states. Keep the last one in the direction of @step.
+ * No @step: @value is for the syms of the current state only.
+ */
+static void add_iv_threshold(struct bpf_verifier_env *env, struct bpf_loop *loop, s64 value,
+ s64 step, enum iv_threshold_kind kind, u32 idx)
+{
+ struct bpf_iv_threshold *threshold;
+
+ for_each_iv_threshold(env, loop, threshold, kind)
+ if (threshold->insn_idx == env->insn_idx && threshold->idx == idx)
+ break;
+ if (threshold < loop->iv_thresholds + loop->iv_threshold_cnt) {
+ if (value == threshold->value || (step && (step > 0) == (value < threshold->value)))
+ return;
+ threshold->value = value;
+ } else if (bpf_grow_array(env, &loop->iv_thresholds, &loop->iv_threshold_cap,
+ loop->iv_threshold_cnt + 1, sizeof(*threshold))) {
+ loop->iv_thresholds[loop->iv_threshold_cnt++] =
+ (struct bpf_iv_threshold){ value, kind, env->insn_idx, idx };
+ }
+ if (env->log.level & BPF_LOG_LEVEL2)
+ bpf_log(&env->log, "loop at %d: %s has threshold %lld\n", loop->header,
+ kind <= THRESHOLD_BODY_JUMP ? "the trip index" : "a number", value);
+}
+
+/* Write @expr as a function of @trip_var. It starts from the far end of what is added. */
+static bool match_iv(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ struct expr *expr, u64 trip_var, bool *low32, bool *start_is_range)
+{
+ const struct bpf_sym_equation *sym_eq, *ext;
+ struct scalar bounds;
+ struct expr rest;
+ u32 n, i, k;
+
+ for (n = 0;
+ expr->coef && expr->var != trip_var && n < st->equations.sym_equation_cnt; n++) {
+ if (!var_is_sym(expr->var) || over_limit(env))
+ return false;
+ ext = NULL;
+ for_each_sum(env, &st->equations, sym_eq) {
+ i = find_sym_pos(sym_eq, var_symno(expr->var));
+ if (i == 3)
+ continue;
+ k = find_sym_pos(sym_eq, var_symno(trip_var));
+ if (!i && !(u8)sym_eq->coef[2])
+ ext = sym_eq;
+ if (k == 3 || !is_plus_minus_one(sym_eq->coef[i]))
+ continue;
+ rest = sym_expr(-expr->coef * sym_eq->coef[i] * sym_eq->coef[3 - i - k],
+ sym_eq->sym[3 - i - k],
+ expr->off - expr->coef * sym_eq->coef[i] * sym_eq->off);
+ bpf_eval_expr(env, st, &rest, &bounds);
+ *expr = make_expr(-expr->coef * sym_eq->coef[i] * sym_eq->coef[k],
+ trip_var, bounds.r64.base);
+ if (bounds.r64.size > U32_MAX && bounds.r32.size == U32_MAX)
+ return false;
+ *start_is_range = bounds.r64.size;
+ *low32 |= bounds.r64.size >= U32_MAX;
+ if (*low32)
+ expr->off = bounds.r32.base +
+ (expr->coef < 0 ? bounds.r32.size : 0);
+ else if (expr->coef < 0)
+ expr->off += bounds.r64.size;
+ return expr->coef;
+ }
+ if (!ext)
+ return false;
+ *expr = expr_scale(rhs_expr(ext, 0), expr->coef, expr->off);
+ }
+ return expr->coef && expr->var == trip_var;
+}
+
+/*
+ * "*@x @op @bound" holds in the trips that continue. In how many, if each adds @step to it?
+ * *@x becomes the end of the range in which they continue.
+ */
+static u64 compute_trip_count(s64 step, u64 *x, u8 op, bool jmp32, struct scalar bound)
+{
+ struct cnum64 arc;
+ struct scalar body_range;
+ u64 left;
+
+ if (op == BPF_JNE && jmp32)
+ scalar_set_const(&bound, bound.r32.base + (step > 0 ? bound.r32.size : 0));
+ else if (op == BPF_JNE)
+ scalar_set_const(&bound, bound.r64.base + (step > 0 ? bound.r64.size : 0));
+ scalar_set_unknown(&body_range);
+ scalars_refine_cond_op(&body_range, &bound, op, jmp32);
+ arc = jmp32 ? (struct cnum64){ body_range.r32.base, body_range.r32.size } : body_range.r64;
+ left = jmp32 ? (u32)(*x - arc.base) : *x - arc.base;
+ if (left > arc.size)
+ return 0;
+ *x = arc.base + (step > 0 ? arc.size : 0);
+ *x = !jmp32 ? *x : op == bpf_signed_opcode(op) && op != BPF_JNE ? (s32)*x : (u32)*x;
+ return min(div64_u64(step > 0 ? arc.size - left : left, abs64(step)), U64_MAX - 1) + 1;
+}
+
+static bool insn_is_in_loop(struct bpf_verifier_env *env, const struct bpf_loop *loop, int idx)
+{
+ int h = bpf_loop_at_index(env, idx);
+
+ return h == loop->header || bpf_is_nested_loop(env, h, loop->header);
+}
+
+/* The jump stays in the loop if operand 0 or 1, or their difference, is "op" the bound */
+static u8 loop_cond_opcode(const struct bpf_insn *insn, bool taken, int i)
+{
+ u8 op = taken ? BPF_OP(insn->code) : bpf_rev_opcode(BPF_OP(insn->code));
+
+ return !i ? op : i == 1 ? bpf_flip_opcode(op) : bpf_signed_opcode(op);
+}
+
+/* The difference starts from @x, but not further out than @bound[1] - @bound[0] can be */
+static s64 clamp_diff_start(const struct scalar *bound, s64 x, s64 step)
+{
+ s64 a = step < 0 ? cnum64_smax(bound[1].r64) : cnum64_smin(bound[1].r64);
+ s64 b = step < 0 ? cnum64_smin(bound[0].r64) : cnum64_smax(bound[0].r64);
+
+ if (check_sub_overflow(a, b, &a))
+ return x;
+ return step < 0 ? min(x, a) : max(x, a);
+}
+
+/* The jump at env->insn_idx tests a number that moves with the trip index. Find thresholds. */
+void bpf_collect_iv_thresholds(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
+ const struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];
+ const struct bpf_verifier_state *st = env->cur_state;
+ const struct bpf_func_state *f = st->frame[st->curframe];
+ bool is_ptr, is_iv[3], low32[3], start_is_range[3] = {}, stays_in_loop[2], is_body_jump;
+ struct loop_stack_entry *e = bpf_top_loop(f);
+ struct expr trip, step, expr[3];
+ struct scalar bound[3];
+ struct equation_pos pos = {};
+ bool tests_carry = false;
+ u32 i, n;
+ u64 start, x, cnt;
+ u8 op;
+
+ if (!env->eq_env || !e || e->loop->kind != BPF_LOOP_JUMPS ||
+ (env->path_sensitive ? !aux->iv_operand : !st->equations.xreg_equation_cnt ||
+ (e->trip_widened &&
+ aux->thresholds_epoch == e->loop->leftover_epoch)))
+ return;
+ expr[1] = make_expr(0, 0, insn->imm);
+ if (!bpf_xreg_expr(env, st, bpf_reg_to_xreg(st, insn->dst_reg), &expr[0], &is_ptr) ||
+ (BPF_SRC(insn->code) == BPF_X &&
+ !bpf_xreg_expr(env, st, bpf_reg_to_xreg(st, insn->src_reg), &expr[1], &is_ptr)))
+ return;
+ bpf_eval_expr(env, st, &expr[1], &bound[0]);
+ bpf_eval_expr(env, st, &expr[0], &bound[1]);
+ scalar_set_const(&bound[2], 0);
+ stays_in_loop[0] = insn_is_in_loop(env, e->loop, env->insn_idx + 1);
+ stays_in_loop[1] = insn_is_in_loop(env, e->loop, env->insn_idx + insn->off + 1);
+ is_body_jump = stays_in_loop[0] && stays_in_loop[1];
+ low32[0] = low32[1] = low32[2] = BPF_CLASS(insn->code) == BPF_JMP32;
+ x = bpf_xreg(st->curframe, BPF_XREG_TRIP + f->loop_stack_cnt - 1);
+ if (!bpf_xreg_expr(env, st, x, &trip, &is_ptr) || !var_is_sym(trip.var))
+ return;
+ if (expr[0].coef && expr[1].coef &&
+ bpf_find_next_diff(env, &st->equations, &expr[0], &expr[1], &pos, &expr[2])) {
+ /* it gets to 0 */
+ add_iv_threshold(env, e->loop, expr[0].off - expr[1].off, 0, THRESHOLD_DIFF, 0);
+ add_iv_threshold(env, e->loop, expr[1].off - expr[0].off, 0, THRESHOLD_DIFF, 1);
+ add_iv_threshold(env, e->loop, -expr[2].coef * expr[2].off, 0, THRESHOLD_DIFF, 2);
+ } else {
+ expr[2].coef = 0;
+ }
+ for (i = 0; i < 3; i++) {
+ is_iv[i] = match_iv(env, st, &expr[i], trip.var, &low32[i], &start_is_range[i]);
+ /* the step in one trip */
+ step = make_expr(0, 0, expr[i].coef);
+ is_iv[i] = is_iv[i] && bpf_expr_div_exact(&step, 0, trip.coef);
+ expr[i].coef = step.off;
+ }
+ /* they do not get closer */
+ if (is_iv[0] && is_iv[1] && !start_is_range[0] && !start_is_range[1] &&
+ expr[0].coef == expr[1].coef) {
+ /* but 'x != (u32)x' is 'x > U32_MAX', see refine_carry(). Operand i is x. */
+ i = cnum64_umax(bound[1].r64) <= U32_MAX;
+ if ((insn->code != (BPF_JMP | BPF_JNE | BPF_X) &&
+ insn->code != (BPF_JMP | BPF_JEQ | BPF_X)) ||
+ expr[0].off == expr[1].off || (u32)(expr[0].off - expr[1].off) ||
+ cnum64_umax(bound[i].r64) > U32_MAX)
+ return;
+ scalar_set_const(&bound[i], U32_MAX);
+ is_iv[!i] = is_iv[2] = low32[i] = false;
+ tests_carry = true;
+ }
+ start = e->trip_index.r64.base;
+ for (n = is_iv[0] && is_iv[1] && is_iv[2] ? 4 : 0; n < 6; n++) {
+ i = n / 2;
+ if (!is_iv[i] || !stays_in_loop[n & 1] || (is_body_jump && start_is_range[i]))
+ continue;
+ aux->thresholds_epoch = e->loop->leftover_epoch;
+ /* note_trips_left() takes the other operand as the bound */
+ if (i < 2 && !tests_carry)
+ aux->iv_operand = i + 1;
+ x = expr[i].coef * (start - trip.off) + expr[i].off;
+ if (i == 2 && start_is_range[i])
+ x = clamp_diff_start(bound, low32[i] ? (s32)x : x, expr[i].coef);
+ op = loop_cond_opcode(insn, n & 1, i);
+ if (tests_carry)
+ op = op == BPF_JNE ? BPF_JGT : BPF_JLE;
+ cnt = compute_trip_count(expr[i].coef, &x, op, low32[i], bound[i]);
+ /* its start is not known, so the trip index does not give its end */
+ if (cnt && start_is_range[i] && i < 2) {
+ add_iv_threshold(env, e->loop, x, expr[i].coef, THRESHOLD_SYM, 2 * n);
+ add_iv_threshold(env, e->loop, x + expr[i].coef, expr[i].coef,
+ THRESHOLD_SYM, 2 * n + 1);
+ }
+ if (!cnt || cnt > BPF_MAX_TRIPS)
+ continue;
+ add_iv_threshold(env, e->loop, start + cnt, 1, is_body_jump, 2 * n);
+ if (is_body_jump)
+ add_iv_threshold(env, e->loop, start + cnt + 1, 1, is_body_jump, 2 * n + 1);
+ else
+ e->exit_threshold = min(e->exit_threshold, start + cnt);
+ }
+}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 5fbbb09a665..d31a689471e 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -102,6 +102,7 @@ struct bpf_flow {
unsigned long *bb_starts; /* insn starts a basic block */
unsigned long *join_points; /* more than one edge leads to the insn */
struct bpf_loop call_loop; /* from the exit of a function to its next call */
+ u32 start_cnt; /* of passes and of path-sensitive runs */
/* of the pass */
u64 *cost; /* per function */
@@ -398,6 +399,8 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
loop->rt_bounded = flow_loop_is_bounded(env, h);
/* also when it's bounded: the main pass may find that it's not */
flow_place_guards(env, h, seen);
+ /* pass 0 does not know the trip count before it ends */
+ loop->guarded = loop->has_guards && !bpf_prog_is_offloaded(env->prog->aux);
if (aux[h].guard_at_header)
ret = 1;
}
@@ -960,6 +963,22 @@ static bool flow_same_entry(struct bpf_loop *loop, const struct flow_state *t,
flow_loop_entry(&t->st, loop)->entry_id == flow_loop_entry(&s->st, loop)->entry_id;
}
+/* @t is from an earlier walk over a leftover. The header state of @s subsumes that of @t. */
+static bool flow_is_stale(const struct flow_state *t, const struct flow_state *s)
+{
+ const struct loop_stack_entry *a, *b;
+ u32 fr, i;
+
+ for (fr = 0; !t->pending && fr <= s->st.curframe; fr++) {
+ a = t->st.frame[fr]->loop_stack;
+ b = s->st.frame[fr]->loop_stack;
+ for (i = 0; i < s->st.frame[fr]->loop_stack_cnt; i++, a++, b++)
+ if (a->entry_id == b->entry_id && a->trip < b->trip)
+ return true;
+ }
+ return false;
+}
+
/*
* All states arrive here. @s is
* - pruned, if a state subsumes it,
@@ -1016,8 +1035,13 @@ static int flow_prune_or_join(struct bpf_verifier_env *env, struct flow_state *s
joined = true;
continue;
}
+ if (!leftover_loop && s->kind == FLOW_AT && flow_is_stale(t, s)) {
+ flow_free_state(env, t);
+ continue;
+ }
joined = true;
widened = leftover_loop;
+ /* what enters did not grow in this loop */
widen = !t->pending && !enters;
flow_unlink(env, t);
err = bpf_states_join(env, &t->st, &s->st, leftover_loop, widen);
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index ad442a5d6b6..d5ec2dfb561 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -40,6 +40,16 @@ u32 bpf_frame_mask(struct bpf_verifier_env *env, unsigned long *masks,
return words * BITS_PER_LONG;
}
+bool bpf_xreg_in_data_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 fr, u32 xregno)
+{
+ unsigned long data_deps[XREG_WORDS];
+
+ return env->precision->data_deps &&
+ xregno < bpf_frame_mask(env, env->precision->data_deps, st, fr, data_deps) &&
+ test_bit(xregno, data_deps);
+}
+
bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 fr, u32 xregno)
{
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 3f4edb9bafb..bede3c12c3d 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -583,6 +583,24 @@ u64 bpf_hash_join_key(const struct bpf_verifier_env *env, const struct bpf_verif
return hash;
}
+/*
+ * If may_goto or an iterator stops the loop, its trip index can start again from a smaller
+ * number, never from a larger one: see bpf_equations_next_trip().
+ */
+static bool trip_index_safe(const struct loop_stack_entry *old, const struct loop_stack_entry *cur)
+{
+ u64 min = cnum64_umin(old->trip_index.r64);
+ struct scalar first;
+
+ if (old->peeled || scalar_subsumes(&old->trip_index, &cur->trip_index))
+ return true;
+ if (!old->loop->rt_bounded && !old->loop->this_pass.needs_guard)
+ return false;
+ scalar_set_const(&first, min);
+ return min <= cnum64_umin(cur->trip_index.r64) &&
+ scalar_subsumes(&old->trip_index, &first);
+}
+
static bool loop_stack_safe(struct bpf_func_state *old, struct bpf_func_state *cur)
{
u32 i;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index cb79c91ae1c..bf26a6dc87d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19247,6 +19247,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
env->false_reg2 = *src_reg;
env->true_reg1 = *dst_reg;
env->true_reg2 = *src_reg;
+ bpf_collect_iv_thresholds(env);
pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32);
if (pred == 1) {
/* Only follow the goto, ignore fall-through. If needed, push
@@ -20738,7 +20739,9 @@ static struct loop_stack_entry *push_loop(struct bpf_verifier_env *env,
.held_cnt = held_cnt(env->cur_state),
.entry_id = ++env->loop_entry_cnt,
.peeled = true,
+ .exit_threshold = U64_MAX,
};
+ scalar_set_const(&frame->loop_stack[cnt].trip_index, 0);
frame->loop_stack_cnt++;
return &frame->loop_stack[cnt];
}
@@ -20759,10 +20762,17 @@ static void leave_peeled_trips(struct bpf_verifier_env *env, struct loop_stack_e
{
struct bpf_func_state *frame;
struct bpf_reg_state *reg;
+ u32 i;
entry->peeled = false;
entry->loop->this_pass.has_leftover = true;
+ entry->loop->leftover_epoch++;
+ /* the leftover of the last pass had no trip count */
+ entry->loop->guarded |= entry->loop->last_pass.needs_guard && !env->cur_state->speculative;
+ entry->trip_index.grew_once = entry->trip_widened = false;
bpf_for_each_reg_in_vstate(env->cur_state, frame, reg, reg->grew_once = false);
+ for (i = 0; i < env->cur_state->equations.sym_cnt; i++)
+ env->cur_state->equations.syms[i].grew_once = false;
}
/*
@@ -20855,6 +20865,8 @@ static void loop_stack_pop(struct bpf_verifier_env *env)
verbose(env, "exiting loop %d\n",
frame->loop_stack[i].loop->header);
leave_loops(frame, new_cnt);
+ if (new_cnt != cnt)
+ bpf_equations_clean(env, env->cur_state);
}
static bool xreg_has_id(struct bpf_verifier_state *st, u32 id)
@@ -20929,6 +20941,11 @@ static int next_trip(struct bpf_verifier_env *env, struct loop_stack_entry *entr
bool *starts_leftover)
{
entry->trip++;
+ if (entry->loop->kind != BPF_LOOP_RECURSION) {
+ bpf_equations_next_trip(env, env->cur_state, entry - cur_func(env)->loop_stack);
+ if (infeasible_state_ends(env))
+ return 1;
+ }
if (env->path_sensitive)
return failing_path_trip(env, entry);
if (!entry->peeled)
@@ -20992,6 +21009,13 @@ static int enter_callback(struct bpf_verifier_env *env, bool back, struct bpf_lo
return err;
}
+/* The current state is at the header of a loop and starts its leftover */
+int bpf_flow_start_leftover(struct bpf_verifier_env *env)
+{
+ leave_peeled_trips(env, bpf_top_loop(cur_func(env)));
+ return enter_callback(env, false, NULL);
+}
+
/*
* The current state gets to env->insn_idx. Leave the loops that the insn is not in. Enter
* the loops that it is in. Returns 1 if the state ends here.
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 30/49] bpf: Get the trip count from the trip index
2026-11-20 7:42 ` [PATCH bpf-next 30/49] bpf: Get the trip count from the trip index Alexei Starovoitov
@ 2026-11-20 8:22 ` sashiko-bot
0 siblings, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:22 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] bpf: empty trip_index range and desynced var_off in
bpf_equations_next_trip()
--
commit 162fc0fc6e39fa252359de4313f88c4f96f4dc62
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Get the trip count from the trip index
This commit tracks each loop's trip index as an xreg, collects induction
variable thresholds from conditional jumps, and widens the trip index at
loop headers to bound the loop trip count.
> diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
> index 3d9df6e684528..fc05ca7dd6068 100644
> --- a/kernel/bpf/equation.c
> +++ b/kernel/bpf/equation.c
> @@ -1132,6 +1132,91 @@ void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *fra
> env->eq_env->fill_bits = size * BITS_PER_BYTE;
> }
>
> +/*
> + * The state came back to the header of the loop at @depth.
> + * A trip count or may_goto ends the loop long before the trip index gets to 2^63.
> + */
> +void bpf_equations_next_trip(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 depth)
> +{
> + struct scalar *trip_index = &st->frame[st->curframe]->loop_stack[depth].trip_index;
> + struct bpf_xreg_equation *xreg_eq;
> +
> + scalar_add_const(trip_index, 1);
> + trip_index->r64 = cnum64_intersect(trip_index->r64, cnum64_from_urange(0, S64_MAX));
> + scalar_bounds_sync(trip_index);
> + xreg_eq = bpf_find_xreg_equation(&st->equations,
> + bpf_xreg(st->curframe, BPF_XREG_TRIP + depth));
> + if (!xreg_eq)
> + return;
> + xreg_eq->off++;
> + refine_sym_from_xreg(env, st, xreg_eq);
> + propagate_bounds(env, st, false);
> + bpf_equations_clean(env, st);
> +}
[Severity: Medium]
Can bpf_equations_next_trip() leave trip_index->r64 set to CNUM64_EMPTY
with a desynchronized var_off?
If trip_index is widened to the fallback [0, S64_MAX] range and a jump in
the loop body refines an induction variable equal to trip_index to
S64_MAX, trip_index becomes the constant S64_MAX. When that path takes a
backedge, bpf_clean_verifier_state() -> bpf_equations_clean() drops the
trip_index xreg_equation because trip_index->var_off is constant.
In bpf_equations_next_trip(), scalar_add_const(trip_index, 1) then wraps
trip_index to constant S64_MIN, and intersecting trip_index->r64 with
[0, S64_MAX] produces CNUM64_EMPTY. scalar_bounds_sync() immediately
bails out on range_bounds_violation(), leaving trip_index->r64 empty while
var_off remains tnum_const(S64_MIN), and bpf_find_xreg_equation() returns
NULL without marking st->equations.infeasible. The state then continues
into flow_prune_or_join() and passes CNUM64_EMPTY to cnum64_umax() and
cnum64_umin() in join_trip_index() and trip_index_safe().
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=30
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 31/49] bpf: Give a loop a checked trip count or may_goto
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (29 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 30/49] bpf: Get the trip count from the trip index Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 32/49] bpf: Use the equations in the join rule Alexei Starovoitov
` (17 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A fixpoint proves that each insn is safe. It does not prove that a loop
ends. A state that is pruned by its own ancestor closes a cycle:
t --> ... --> header --> ... --> s
^ |
`----------- subsumes -----------'
All pruned states go through flow_check_trip_counts(). If s made more
trips than t, look at the trip index k of s:
if k <= BPF_MAX_TRIPS (2^32) the loop has a trip count, no may_goto
else the loop gets may_goto
A loop that has may_goto or an iterator needs nothing. There is no trip
count without CAP_PERFMON, as before.
may_goto is there for all states at run time. If a state went through
the loop without its edges, the pass starts again.
bpf_flow_drop_guards() removes may_goto that no state needed.
A peeled trip that an earlier trip subsumes is pruned for the price of
may_goto, see flow_prune_peeled().
may_goto at a header skips the body. So a slot that is live at its
target is live at the header.
bpf_loop_spine_end() finds the insns that dominate all back edges. It
replaces the counters in flow_place_guards().
If both edges of a jump are dead, no run gets to the jump. It becomes
'goto -1'.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 27 +++--
kernel/bpf/fixups.c | 24 ++--
kernel/bpf/flow.c | 225 +++++++++++++++++++++++++++++++----
kernel/bpf/liveness.c | 3 +
kernel/bpf/loops.c | 13 ++
kernel/bpf/verifier.c | 85 ++++++-------
6 files changed, 285 insertions(+), 92 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index be499471e12..2d4d8531ced 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -839,7 +839,9 @@ enum bpf_loop_kind {
/* What a pass finds about a loop */
struct bpf_loop_per_pass {
bool needs_guard; /* see bpf_flow_needs_guard() */
+ bool no_guard_edges; /* a state did not take the edges of may_goto */
bool has_leftover; /* the peeled trips were not enough */
+ bool peel_pruned; /* see flow_prune_peeled() */
/* a check failed in a state that the loop kept apart, with no other suspect */
bool has_unproven;
u32 last_peeled_trip; /* see flow_keep_peeled() */
@@ -872,11 +874,14 @@ struct bpf_loop {
u32 max_peel_cnt; /* peel up to this trip */
bool peeled_apart; /* the join rule holds for states that start a peeled trip */
bool disjoint_apart; /* disjoint values in data_deps stay apart, also in the leftover */
+ bool no_peel_prune; /* a larger earlier trip does not prune, see flow_prune_peeled() */
);
struct bpf_loop_per_pass last_pass; /* the last pass whose partition was kept */
bool partitioned; /* the last partition was for this loop */
bool peeled_joined; /* at the start of a peeled trip, against the join rule */
+ bool changed; /* see flow_forget_loop() */
bool guarded; /* states take the edges of may_goto */
+ bool infinite; /* see failing_path_trip() */
struct bpf_iv_threshold *iv_thresholds; /* where an iv can stop, see equation_join.c */
u32 iv_threshold_cnt, iv_threshold_cap;
u32 leftover_epoch; /* how many leftovers started */
@@ -951,6 +956,7 @@ struct bpf_insn_aux_data {
u64 cast_arena:1; /* addr_space_cast of an address in arena */
u64 cast_mov:1; /* addr_space_cast of a pointer that is not in arena, it's a move */
u64 loop_guard:1; /* may_goto is added in front of this instruction */
+ u64 guard_unwinds:1; /* it leaves the subprog */
u64 speculative_first:1; /* a speculative state came to it first */
u64 call_recorded:1; /* see record_call_fixup() */
u64 pessimized:1; /* see bpf_partition_for_pessimized() */
@@ -1366,7 +1372,7 @@ struct bpf_verifier_env {
bool imprecise; /* a state before the first error is imprecise */
bool over_limit; /* see bpf_check_limits() */
bool restart; /* see bpf_flow_next_is_no_bound() */
- bool guards_late; /* see bpf_flow_loop_gets_guards() */
+ bool guards_late; /* see bpf_flow_needs_guard() */
bool extra_values; /* a state can have constants that no run has */
bool seen_direct_write;
/* a register may point to the stack and is not PTR_TO_STACK, see bpf_reg_state::buf */
@@ -1734,15 +1740,19 @@ static inline int bpf_add_cost_and_check(struct bpf_verifier_env *env, u64 cnt,
int bpf_flow_prepare(struct bpf_verifier_env *env);
void bpf_flow_reset(struct bpf_verifier_env *env);
void bpf_flow_free(struct bpf_verifier_env *env);
+bool bpf_flow_trip_is_new(struct bpf_verifier_env *env, struct loop_stack_entry *entry);
bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx);
int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
int insn_idx, int prev_insn_idx, bool again);
int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos, u64 *diag_pos);
+int bpf_flow_needs_guard(struct bpf_verifier_env *env, struct bpf_loop *loop);
int bpf_flow_call(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
struct bpf_verifier_state *entry, int subprog, int call_idx);
int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int exit_idx);
void bpf_flow_next_is_no_bound(struct bpf_verifier_env *env, int insn_idx, int h);
bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env);
+void bpf_flow_new_pass(struct bpf_verifier_env *env);
+void bpf_flow_drop_guards(struct bpf_verifier_env *env);
int bpf_flow_place_guards(struct bpf_verifier_env *env);
/* verifier.c, for flow.c */
int bpf_flow_loop_enter(struct bpf_verifier_env *env, struct bpf_loop **backedge_loop,
@@ -2065,18 +2075,16 @@ struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
/*
* Where the program goes from may_goto and from the checks that the verifier adds:
- * - May_goto in front of a jump goes where the jump goes, with the registers and the stack
- * as they are in front of the jump. bpf_insn_successors() has that edge.
+ * - May_goto before a jump goes where the jump goes, with the registers and the stack
+ * as they are before the jump. bpf_insn_successors() has that edge.
* - A subprog that is out of budget goes to bail_insn, an exit. No such edge.
- * - May_goto in front of the head of a loop goes to where the loops around it are left: from
+ * - May_goto before the header of a loop goes to where the loops around it are left: from
* @idx to what is returned here, -1 if there is none. No such edge either. The registers
- * and the stack are what they are in front of @idx.
+ * and the stack are what they are before @idx.
*
* The main pass looks at the states that go there. What is computed from the graph before
- * the main pass has to take these edges into account where the main pass believes it:
- * - Against the direction of the edges, liveness: a register that is live at the target is
- * live at @idx. Stack that is dead at @idx stays dead: a read of it at the target is
- * rejected.
+ * the main pass has to take these edges into account where the main pass relies on it:
+ * - Against the direction of the edges, liveness: what is live at the target is live at @idx.
* - In the direction of the edges, bpf_compute_const_regs() and compute_subprog_args(): what
* holds on all edges of the graph to the target doesn't hold on this one.
*/
@@ -2425,6 +2433,7 @@ u8 bpf_rev_opcode(u8 opcode);
int bpf_compute_idoms(struct bpf_verifier_env *env);
int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
+int bpf_loop_spine_end(struct bpf_verifier_env *env, const struct bpf_loop *loop);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
u32 bpf_loop_thresholds_below(const struct bpf_loop *loop, s64 k);
int bpf_loop_collect_thresholds(struct bpf_verifier_env *env, struct bpf_loop *loop);
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index ccde914c57a..4c9f63e096d 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -608,18 +608,14 @@ static int emit_unwind_exit(struct bpf_verifier_env *env, int s, struct bpf_insn
int bpf_insert_loop_guards(struct bpf_verifier_env *env)
{
struct bpf_insn patch[2] = { BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0) };
- struct bpf_insn leave[3] = { BPF_MOV64_IMM(BPF_REG_0, 0) };
struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct bpf_insn leave[4], *insn;
struct bpf_subprog_info *sub;
- struct bpf_insn *insn;
int i, s, n, err;
- bool leaves;
for (s = 0; s < env->subprog_cnt; s++) {
sub = &env->subprog_info[s];
- n = bpf_ret_reg_pair(env, s) ? 3 : 2;
- leaves = false;
- for (i = sub->start; i < sub[1].start; i++) {
+ for (i = sub->start, n = 0; i < sub[1].start; i++) {
if (!aux[i].loop_guard)
continue;
/* A jump to itself is a back edge. It goes through may_goto too. */
@@ -627,16 +623,15 @@ int bpf_insert_loop_guards(struct bpf_verifier_env *env)
insn = bpf_patch_list_add(env, i, patch, 2);
if (IS_ERR(insn))
return PTR_ERR(insn);
- if (aux[i].guard_no_unwind)
+ if (!aux[i].guard_unwinds) {
bpf_patch_list_jump(&env->patch_list, insn - 1, aux[i].guard_to, 0);
- else
- bpf_patch_list_jump(&env->patch_list, insn - 1, sub[1].start, n);
- leaves |= !aux[i].guard_no_unwind;
+ continue;
+ }
+ n = emit_unwind_exit(env, s, leave);
+ bpf_patch_list_jump(&env->patch_list, insn - 1, sub[1].start, n);
}
- if (!leaves)
+ if (!n)
continue;
- leave[1] = BPF_MOV64_IMM(BPF_REG_2, 0);
- leave[n - 1] = budget_exit_insn(env, sub);
err = bpf_patch_list_append(env, i - 1, leave, n);
if (err)
return err;
@@ -831,6 +826,9 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
if (bpf_prog_is_offloaded(env->prog->aux))
bpf_prog_offload_replace_insn(env, i, &ja);
+ /* If both edges are dead, no run gets here. The driver keeps the old target. */
+ if (!aux_data[i + 1 + ja.off].seen)
+ ja.off = -1;
memcpy(insn, &ja, sizeof(ja));
}
}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index d31a689471e..2acf85a10d4 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -244,11 +244,11 @@ static void flow_set_guard(struct bpf_verifier_env *env, int h, int n)
aux[h].loop->has_guards = true;
}
-static void flow_place_guards(struct bpf_verifier_env *env, int h, u32 *seen)
+static void flow_place_guards(struct bpf_verifier_env *env, int h)
{
struct bpf_insn_aux_data *aux = env->insn_aux_data;
struct bpf_loop *loop = aux[h].loop;
- int i, n, o, t, found = -1;
+ int i, n, o, t;
if (loop->backedges_overflow)
goto header;
@@ -268,20 +268,9 @@ static void flow_place_guards(struct bpf_verifier_env *env, int h, u32 *seen)
return;
}
- /* seen[n]: the number of back edges that n dominates */
- for (i = 0; i < loop->backedges_cnt; i++)
- for (n = loop->backedges[i].from; n >= 0; n = n == h ? -1 : env->idoms[n])
- seen[n]++;
- for (n = loop->backedges[0].from; n >= 0 && found < 0; n = n == h ? -1 : env->idoms[n])
- if (seen[n] == loop->backedges_cnt && bpf_insn_leaves_loop(env, h, n))
- found = n;
- for (i = 0; i < loop->backedges_cnt; i++)
- for (n = loop->backedges[i].from; n >= 0; n = n == h ? -1 : env->idoms[n])
- seen[n] = 0;
- if (found >= 0) {
- flow_set_guard(env, h, found);
- return;
- }
+ for (n = bpf_loop_spine_end(env, loop); n >= 0; n = n == h ? -1 : env->idoms[n])
+ if (bpf_insn_leaves_loop(env, h, n))
+ return flow_set_guard(env, h, n);
for (i = 0; i < loop->backedges_cnt; i++)
if (loop->backedges[i].latch < 0)
@@ -367,6 +356,86 @@ bool bpf_flow_loops_lost_bound(struct bpf_verifier_env *env)
return false;
}
+/*
+ * A state goes around @loop without a trip count, so the loop gets may_goto. At run time
+ * may_goto is there for all states. This includes states with a trip count, and states that
+ * got to the end of the loop in a peeled trip. A loop before this one can use up the budget
+ * of may_goto. So if a state did not take the edges of may_goto, the pass restarts.
+ */
+int bpf_flow_needs_guard(struct bpf_verifier_env *env, struct bpf_loop *loop)
+{
+ if (loop->rt_bounded)
+ return 0;
+ if (!loop->has_guards) {
+ verbose(env, "loop at insn %d never exits\n", loop->header);
+ return -EINVAL;
+ }
+ if (!loop->this_pass.needs_guard && (env->log.level & BPF_LOG_LEVEL2))
+ verbose(env, "loop at %d has no trip count\n", loop->header);
+ loop->this_pass.needs_guard = loop->guarded = true;
+ env->guards_late |= loop->this_pass.no_guard_edges;
+ return 0;
+}
+
+/* This pass peels more trips. What the last pass found about @loop can be wrong. */
+static void flow_forget_loop(struct bpf_verifier_env *env, struct bpf_loop *loop)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ int i, n = loop->nr_insns;
+
+ loop->changed = loop->guarded = false;
+ bpf_add_cost(env, n, BPF_COST_SCAN);
+ for (i = loop->header; n--; i = bpf_flow_insn_after(env, i)) {
+ if (bpf_loop_at_index(env, i) != loop->header)
+ continue;
+ aux[i].guard_no_unwind = aux[i].guard_unset_reg = false;
+ aux[i].not_stopped = -1;
+ }
+ loop->rt_bounded = flow_loop_is_bounded(env, loop->header);
+}
+
+/* A pass starts. It verifies the main prog and the global functions. */
+void bpf_flow_new_pass(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ struct bpf_flow *flow = env->flow;
+ int i;
+
+ memset(flow->cost, 0, env->subprog_cnt * sizeof(*flow->cost));
+ flow->budget = bpf_precision_budget(env);
+ flow->budget_func_cnt = 0;
+ flow->start_cnt++;
+ for (i = 0; i < env->prog->len; i++, aux++) {
+ if (aux->callback_loop)
+ aux->callback_loop->this_pass = (struct bpf_loop_per_pass){};
+ if (aux->recursion_loop)
+ aux->recursion_loop->this_pass = (struct bpf_loop_per_pass){};
+ if (!aux->loop)
+ continue;
+ /* pass 0 is over */
+ if (flow->start_cnt == 2)
+ aux->loop->guarded = aux->loop->this_pass.needs_guard;
+ aux->loop->this_pass = (struct bpf_loop_per_pass){};
+ if (aux->loop->changed && !env->path_sensitive)
+ flow_forget_loop(env, aux->loop);
+ if (aux->loop->rt_bounded)
+ aux->loop->rt_bounded = flow_loop_is_bounded(env, i);
+ }
+}
+
+/* The pass has no error. A loop needs may_goto only if a state had no trip count. */
+void bpf_flow_drop_guards(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ int i, h;
+
+ for (i = 0; i < env->prog->len; i++) {
+ h = bpf_loop_at_index(env, i);
+ if (h >= 0)
+ aux[i].loop_guard &= aux[h].loop->this_pass.needs_guard;
+ }
+}
+
/*
* Which loops have something that stops them at run time, and where may_goto goes in the others.
* Returns 1 if a place is at the head of a loop.
@@ -376,11 +445,6 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
struct bpf_insn_aux_data *aux = env->insn_aux_data;
int len = env->prog->len, h, i, ret = 0, err = 0;
struct bpf_loop *loop;
- u32 *seen;
-
- seen = kvcalloc(len, sizeof(*seen), BPF_VERIFIER_GFP);
- if (!seen)
- return -ENOMEM;
for (i = 0; i < len; i++) {
aux[i].not_stopped = -1;
@@ -398,13 +462,12 @@ int bpf_flow_place_guards(struct bpf_verifier_env *env)
break;
loop->rt_bounded = flow_loop_is_bounded(env, h);
/* also when it's bounded: the main pass may find that it's not */
- flow_place_guards(env, h, seen);
+ flow_place_guards(env, h);
/* pass 0 does not know the trip count before it ends */
loop->guarded = loop->has_guards && !bpf_prog_is_offloaded(env->prog->aux);
if (aux[h].guard_at_header)
ret = 1;
}
- kvfree(seen);
return err ?: ret;
}
@@ -963,6 +1026,55 @@ static bool flow_same_entry(struct bpf_loop *loop, const struct flow_state *t,
flow_loop_entry(&t->st, loop)->entry_id == flow_loop_entry(&s->st, loop)->entry_id;
}
+/*
+ * @t subsumes @s. The caller prunes @s, @t continues for it. If @s made more trips it can
+ * come from @t:
+ *
+ * t --> ... --> header --> ... --> s
+ * ^ |
+ * `----------- subsumes -----------'
+ *
+ * A back edge adds a trip and a join has the larger number. So each cycle of states has such
+ * an edge for the outermost loop that it goes around. All pruned states come here.
+ * Speculation ends by itself.
+ *
+ * The loop ends if the trip index k of @s is small each time. That is the trip count:
+ *
+ * trip count: guess -> state holds k -> jumps refine k -> k is small ?
+ * yes: no guard no: add may_goto
+ *
+ * A loop that can run longer than that gets may_goto.
+ * There is no trip count without CAP_PERFMON (allow_uninit_stack).
+ */
+static int flow_check_trip_counts(struct bpf_verifier_env *env, const struct flow_state *t,
+ const struct flow_state *s)
+{
+ const struct loop_stack_entry *e;
+ u32 fr, i;
+ u64 max_trip;
+ int err;
+
+ for (fr = 0; fr <= s->st.curframe && !s->st.speculative; fr++) {
+ e = s->st.frame[fr]->loop_stack;
+ for (i = 0; i < s->st.frame[fr]->loop_stack_cnt; i++, e++) {
+ if (e->loop->kind != BPF_LOOP_JUMPS ||
+ e->trip <= t->st.frame[fr]->loop_stack[i].trip)
+ continue;
+ max_trip = cnum64_umax(e->trip_index.r64);
+ if (!e->peeled && max_trip <= BPF_MAX_TRIPS && env->allow_uninit_stack) {
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "loop at %d has trip count %llu\n",
+ e->loop->header, max_trip + 1);
+ continue;
+ }
+ err = bpf_flow_needs_guard(env, e->loop);
+ if (err)
+ return err;
+ }
+ }
+ return 0;
+}
+
/* @t is from an earlier walk over a leftover. The header state of @s subsumes that of @t. */
static bool flow_is_stale(const struct flow_state *t, const struct flow_state *s)
{
@@ -1068,6 +1180,73 @@ static int flow_prune_or_join(struct bpf_verifier_env *env, struct flow_state *s
return err;
}
+/*
+ * @s starts a peeled trip of @loop. Does it repeat the state of an earlier trip? It is compared
+ * with trip 0, 1, 2, 4, 8, ... That finds any period. If @s comes from that state, the verifier
+ * sees no end of the loop. So the loop needs may_goto.
+ * A state that has less than the earlier one can be on its way to the end of the loop:
+ * for (i = 7; i >= 0; i--) { if (i >= n) continue; ... }
+ * It is pruned for the price of may_goto. So is a state of another entry. If that causes
+ * an error, the next pass sets no_peel_prune.
+ */
+static int flow_prune_peeled(struct bpf_verifier_env *env, struct flow_state *s,
+ struct bpf_loop *loop)
+{
+ struct loop_stack_entry *entry = flow_loop_entry(&s->st, loop);
+ struct flow_state *t, *tn;
+ u32 trip = entry->trip;
+ int ret = 0;
+
+ entry->trip = 0;
+ s->kind = FLOW_PEELED;
+ flow_for_each_near(env, s, t, tn) {
+ if (!flow_same_join_key(env, t, s) || !bpf_state_subsumes(env, &t->st, &s->st))
+ continue;
+ if (!loop->rt_bounded && !loop->this_pass.needs_guard && !s->st.speculative &&
+ (!flow_same_entry(loop, t, s) || !bpf_state_subsumes(env, &s->st, &t->st))) {
+ if (loop->no_peel_prune)
+ continue;
+ loop->this_pass.peel_pruned = true;
+ }
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "from %d to %d: pruned\n", s->prev_insn_idx, s->st.insn_idx);
+ entry->trip = trip;
+ ret = flow_check_trip_counts(env, t, s) ?: 1;
+ break;
+ }
+ s->kind = FLOW_AT;
+ entry->trip = trip;
+ return ret;
+}
+
+/*
+ * The current state starts a peeled trip. An earlier trip can have all that it has in the
+ * deps. Then more peeled trips cannot help, and the leftover starts. If the earlier trip
+ * subsumes the full state, the trip counts as new: flow_prune_peeled() prunes it.
+ */
+bool bpf_flow_trip_is_new(struct bpf_verifier_env *env, struct loop_stack_entry *entry)
+{
+ struct flow_state *s = container_of(env->cur_state, struct flow_state, st), *t, *tn;
+ u32 trip = entry->trip;
+ bool is_new = true;
+
+ if (entry->loop->kind == BPF_LOOP_RECURSION || entry->loop->no_peel_prune)
+ return true;
+ entry->trip = 0;
+ s->kind = FLOW_PEELED;
+ flow_for_each_near(env, s, t, tn) {
+ if (!flow_same_join_key(env, t, s) || bpf_values_grew_in_deps(env, &t->st, &s->st))
+ continue;
+ is_new = bpf_state_subsumes(env, &t->st, &s->st);
+ if (is_new)
+ break;
+ }
+ s->kind = FLOW_AT;
+ entry->trip = trip;
+ entry->loop->this_pass.peel_pruned |= !is_new;
+ return is_new;
+}
+
/* @s is taken. @starts_leftover: the states of FLOW_LEFTOVER are joined, start the leftover */
static int flow_add(struct bpf_verifier_env *env, struct flow_state *s, bool starts_leftover)
{
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index c9fea08eada..59976ae6999 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -418,6 +418,7 @@ static inline bool update_insn(struct bpf_verifier_env *env, struct func_instanc
unsigned long *may_read, *must_write, *live_before;
struct frame_masks *fm = instance->frames[frame];
u32 rel = relative_idx(instance, insn_idx);
+ int t = bpf_head_guard_target(env, insn_idx);
struct bpf_iarray *succ;
bool changed = false;
u32 s, w;
@@ -445,6 +446,8 @@ static inline bool update_insn(struct bpf_verifier_env *env, struct func_instanc
for (w = 0; w < fm->words; w++) {
unsigned long new_before = (new_after[w] & ~must_write[w]) | may_read[w];
+ if (t >= 0)
+ new_before |= rel_mask(fm, relative_idx(instance, t), before)[w];
if (new_before != live_before[w]) {
live_before[w] = new_before;
changed = true;
diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c
index 23c443ab3a2..deda85d0953 100644
--- a/kernel/bpf/loops.c
+++ b/kernel/bpf/loops.c
@@ -310,6 +310,16 @@ static int check_loop_depth(struct bpf_verifier_env *env, int idx)
return 0;
}
+/* The last insn that dominates all back edges of @loop */
+int bpf_loop_spine_end(struct bpf_verifier_env *env, const struct bpf_loop *loop)
+{
+ int i, n = loop->backedges[0].from;
+
+ for (i = 1; i < loop->backedges_cnt; i++)
+ n = idoms_intersect(env, n, loop->backedges[i].from);
+ return n;
+}
+
/*
* The backedge->from and its latch can reside in an inner loop of @h, e.g.:
*
@@ -627,6 +637,9 @@ int bpf_loop_collect_thresholds(struct bpf_verifier_env *env, struct bpf_loop *l
if (!todo)
return -ENOMEM;
done = todo + words;
+ /* the insns of a loop follow its header, see flow_compute_order() */
+ i = loop->header;
+ n = loop->kind == BPF_LOOP_CALLBACK ? 1 : loop->nr_insns;
for (;;) {
err = err ?: bpf_add_cost_and_check(env, n, BPF_COST_XREG);
for (; n-- && i >= 0 && i < sub[1].start && !err; i = bpf_flow_insn_after(env, i)) {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index bf26a6dc87d..6373505c672 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -196,7 +196,6 @@ struct bpf_verifier_stack_elem {
struct bpf_verifier_state st;
int insn_idx;
int prev_insn_idx;
- enum bpf_guard_edge guard_edge;
bool again; /* see bpf_flow_deliver() */
struct bpf_verifier_stack_elem *next;
};
@@ -21155,87 +21154,79 @@ static int push_unwind_exit(struct bpf_verifier_env *env)
}
/*
- * The loop that is not counted and has nothing that stops it at run time gets
- * may_goto in front of this insn. When it is out of budget the program goes
- * where may_goto goes with the registers and the stack as they are here.
- * Every state that goes through the loop does, see bpf_flow_loop_gets_guards().
+ * A loop has no trip count, and nothing stops it at run time. It gets may_goto
+ * before this insn. When may_goto is out of budget, the program goes to its
+ * target with the registers and the stack that it has here.
+ * Every state that goes through the loop does that, see bpf_flow_needs_guard().
*/
static int check_loop_guard(struct bpf_verifier_env *env)
{
struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
- struct bpf_verifier_state *branch;
- struct bpf_func_state *frame = cur_func(env);
- struct loop_stack_entry *entry;
struct bpf_loop *loop;
- int h, err;
+ int err;
- h = bpf_loop_at_index(env, env->insn_idx);
/* The callback came back to the helper that calls it: the program is not in front of it. */
- if (h < 0 || !frame->loop_stack_cnt || frame->callback_depth)
+ if (!aux->guard_place || cur_func(env)->callback_depth)
return 0;
- loop = env->insn_aux_data[h].loop;
- entry = &frame->loop_stack[frame->loop_stack_cnt - 1];
- if (loop->rt_bounded || !env->insn_aux_data[env->insn_idx].guard_place ||
- entry->loop_id != h)
+ loop = env->insn_aux_data[bpf_loop_at_index(env, env->insn_idx)].loop;
+ if (loop->rt_bounded)
return 0;
- if (verifier_bug_if(!entry->terminates && !entry->guarded, env,
- "nothing stops the loop at %d", h))
- return -EFAULT;
- if (!entry->guarded && !bpf_loop_walk_may_give_up(entry))
+ if (!(env->path_sensitive ? loop->infinite : loop->guarded)) {
+ loop->this_pass.no_guard_edges = true;
return 0;
+ }
+ env->imprecise |= !env->err;
/* the device runs the insns that the driver was given, without may_goto */
if (bpf_prog_is_offloaded(env->prog->aux)) {
- verbose(env, "loop at insn %d has no known bound in an offloaded program\n", h);
+ verbose(env, "loop at insn %d has no known bound in an offloaded program\n",
+ loop->header);
+ env->cur_state->suspect_loops |= loop->suspect_bit;
return -EINVAL;
}
/*
- * A guard has one way out, and only that one is looked at.
+ * A guard has one way out, and only that one is verified.
*
* Leave the subprog, if that is possible every time the program gets here:
* do { p = lookup(); } while (!p); *p
- * The code that follows the loop counts on what the jump that leaves it has found out.
+ * The code after the loop relies on the result of the jump that leaves it.
*
* Otherwise leave the loop:
* lock(); do { ... } while (...); unlock();
- * The code that follows the loop releases what is held.
+ * The code after the loop releases what is held.
*
- * From the head of a loop to where the loop is left is not an edge of the program:
+ * The program has no edge from the header of a loop to the exit of the loop:
* for (;;) { c = *s++; if (!c) return 0; if (c == '*') return 1; }
- * The loop is left to 'exit', and only its body sets r0. Leave the subprog then, with
+ * The loop is left to 'exit', and only its body sets r0. Then leave the subprog. Use
* a plain exit if it cannot unwind, like check_recursion_guard().
*
- * It's the same way for all states that get here. As long as they leave the subprog
- * what would leave the loop is dropped. Both are dropped as long as the loop has
- * no may_goto: the state is in a walk that may get to the end of the loop.
+ * All states that get here take the same way, or the pass starts again.
+ * A speculative state does not select the way.
*/
+ if (env->cur_state->speculative)
+ goto out;
if (aux->guard_at_header && !aux->guard_unset_reg && guard_reads_unset_reg(env, aux)) {
aux->guard_unset_reg = true;
- /* The states that were here left the loop. All again. */
- if (aux->guard_no_unwind)
- env->guards_late = true;
+ env->guards_late |= aux->guard_no_unwind && aux->loop_guard;
aux->guard_no_unwind = false;
}
if (!aux->guard_no_unwind &&
!state_can_unwind(env, env->cur_state, aux->guard_unset_reg)) {
+ env->guards_late |= aux->loop_guard;
aux->guard_no_unwind = true;
- /* What would leave the loop from the states that were here is gone. All again. */
- if (aux->guard_edge_dropped && loop->guarded)
- env->guards_late = true;
}
- if (!aux->guard_no_unwind) {
+out:
+ aux->loop_guard = true;
+ aux->guard_unwinds = !aux->guard_no_unwind;
+ if (aux->guard_no_unwind)
+ err = PTR_ERR_OR_ZERO(push_stack(env, aux->guard_to, env->insn_idx, false));
+ else
err = push_unwind_exit(env);
- if (err)
- return err;
- env->head->guard_edge = BPF_GUARD_LEAVES_SUBPROG;
- }
-
- branch = push_stack(env, env->insn_aux_data[env->insn_idx].guard_to, env->insn_idx, false);
- if (IS_ERR(branch))
- return PTR_ERR(branch);
- env->head->guard_edge = BPF_GUARD_LEAVES_LOOP;
- return 0;
+ /* more peeled trips can show that the edge is dead */
+ if (!err && env->head)
+ env->head->st.suspect_loops |= loop->suspect_bit;
+ return err;
}
/* The states that the insn pushed go where they were pushed to. */
@@ -21602,8 +21593,8 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
}
}
-/* All is done again, whatever came out, see do_check_all() */
-static bool round_is_dropped(struct bpf_verifier_env *env)
+/* What the pass assumed about loops does not hold, see do_check_all() */
+static bool pass_needs_restart(struct bpf_verifier_env *env)
{
return (env->restart && bpf_flow_loops_lost_bound(env)) || env->guards_late;
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 32/49] bpf: Use the equations in the join rule
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (30 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 31/49] bpf: Give a loop a checked trip count or may_goto Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 33/49] bpf: Verify a static function once for each instance Alexei Starovoitov
` (16 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The join rule compares the ranges of the deps. Two cases need the
equations.
1. State 1 has all ranges of state 2, but only state 1 has a relation:
r1 (data) r6 (control)
state 1 0..99 0..99 r1 == r6
state 2 0..7 0..99
join 0..99 0..99 a test of r6 tells nothing
bpf_equations_join_forgets() finds such a pair. It counts as two
xregs that differ, so the states stay apart. One xreg has to be in
data_deps: the rule joins what differs in control_deps only.
2. LOOP_DISJOINT_APART keeps disjoint values apart at the header. A
counter has another value in each trip, so it keeps all trips apart:
for (i = 0; i < 400; i++) if (x > best) { best = x; at = i; }
The join has i = trip index and keeps i. So an xreg that moves
with the trip index keeps no trips of one entry apart.
bpf_note_linear_xregs() notes these xregs. If a pass notes a new
one, it does not stop at the errors of the last pass. Later trips
can show that the xreg moves with the trip index only at first.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 6 ++++++
kernel/bpf/equation.h | 1 +
kernel/bpf/equation_join.c | 32 ++++++++++++++++++++++++++++++++
kernel/bpf/precision.c | 11 +++++++++++
4 files changed, 50 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 2d4d8531ced..c2636f5a6ed 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1374,6 +1374,7 @@ struct bpf_verifier_env {
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_needs_guard() */
bool extra_values; /* a state can have constants that no run has */
+ bool new_linear_xreg; /* see bpf_same_insns_unproven() */
bool seen_direct_write;
/* a register may point to the stack and is not PTR_TO_STACK, see bpf_reg_state::buf */
bool seen_stack_slice;
@@ -2469,6 +2470,11 @@ void bpf_equations_pre_join(struct bpf_verifier_env *env, struct bpf_verifier_st
void bpf_equations_post_join(struct bpf_verifier_env *env, struct bpf_verifier_state *dst,
const struct bpf_verifier_state *src);
u32 bpf_equations_bytes(const struct bpf_equations *eqs);
+/* equation_join.c */
+bool bpf_equations_join_forgets(struct bpf_verifier_env *env, const struct bpf_verifier_state *a,
+ const struct bpf_verifier_state *b, u32 fr,
+ const unsigned long *data_deps, const unsigned long *xregs,
+ u32 bits);
void bpf_collect_iv_thresholds(struct bpf_verifier_env *env);
void scalar_bounds_sync(struct scalar *s);
bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
diff --git a/kernel/bpf/equation.h b/kernel/bpf/equation.h
index b91f5cf5c49..9430f935bde 100644
--- a/kernel/bpf/equation.h
+++ b/kernel/bpf/equation.h
@@ -54,6 +54,7 @@ struct sym_scratch {
u32 first_use; /* see link_sym_equations() */
u32 new_symno;
bool determined, by_op;
+ bool forgotten, in_data_deps; /* bpf_equations_join_forgets() */
const struct bpf_sym_equation *op; /* that makes it */
struct expr cur_expr; /* check_equations(): its value in the other state */
};
diff --git a/kernel/bpf/equation_join.c b/kernel/bpf/equation_join.c
index 1df6a1cce03..93ebf337e4c 100644
--- a/kernel/bpf/equation_join.c
+++ b/kernel/bpf/equation_join.c
@@ -574,9 +574,11 @@ void bpf_note_linear_xregs(struct bpf_verifier_env *env, const struct bpf_verifi
const struct bpf_xreg_equation *old_trip =
bpf_find_xreg_equation(&dst->equations, trip_xreg), *xreg_eq, *new;
const struct bpf_xreg_equation *new_trip = bpf_find_xreg_equation(join, trip_xreg);
+ struct bpf_loop *loop;
if (!f->loop_stack_cnt || !old_trip || !new_trip)
return;
+ loop = bpf_top_loop(f)->loop;
for_each_xreg_equation(env, &dst->equations, xreg_eq) {
if (xreg_eq->sym != old_trip->sym || xreg_eq->xreg >> 32 != dst->curframe ||
(u32)xreg_eq->xreg >= BPF_XREG_TRIP)
@@ -589,6 +591,36 @@ void bpf_note_linear_xregs(struct bpf_verifier_env *env, const struct bpf_verifi
}
}
+/* An xreg of @data_deps and an xreg of @xregs in frame @fr have one sym in @a and not in @b */
+bool bpf_equations_join_forgets(struct bpf_verifier_env *env, const struct bpf_verifier_state *a,
+ const struct bpf_verifier_state *b, u32 fr,
+ const unsigned long *data_deps, const unsigned long *xregs,
+ u32 bits)
+{
+ const struct bpf_xreg_equation *xreg_eq;
+ struct sym_scratch *sym;
+ struct expr expr;
+ bool is_ptr;
+
+ for_each_xreg_equation(env, &a->equations, xreg_eq)
+ env->eq_env->scratch[xreg_eq->sym] = (struct sym_scratch){};
+ for_each_xreg_equation(env, &a->equations, xreg_eq) {
+ if (xreg_eq->xreg >> 32 != fr || (u32)xreg_eq->xreg >= bits ||
+ !test_bit((u32)xreg_eq->xreg, xregs))
+ continue;
+ sym = &env->eq_env->scratch[xreg_eq->sym];
+ if (bpf_xreg_expr(env, b, xreg_eq->xreg, &expr, &is_ptr)) {
+ sym->forgotten |= sym->determined && expr.var != sym->cur_expr.var;
+ sym->determined = true;
+ sym->cur_expr = expr;
+ }
+ sym->in_data_deps |= test_bit((u32)xreg_eq->xreg, data_deps);
+ if (sym->forgotten && sym->in_data_deps)
+ return true;
+ }
+ return false;
+}
+
/* Thresholds */
/*
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index d5ec2dfb561..aa9f7f76256 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -336,6 +336,13 @@ void bpf_compare_states(struct bpf_verifier_env *env, const struct bpf_verifier_
bitmap_or(all, all, deps, bits);
bitmap_or(deps, deps, data_deps, bits);
bpf_add_cost(env, 4 + bitmap_weight(all, bits), BPF_COST_COPY);
+ /* r1 == r6 in one state only: two xregs differ, count both */
+ flags = (bpf_equations_join_forgets(env, old, new, fr, data_deps, deps, bits) ?
+ CMP_NEW : 0) |
+ (bpf_equations_join_forgets(env, new, old, fr, data_deps, deps, bits) ?
+ CMP_OLD : 0);
+ note_cmp(cmp, flags, true, true);
+ note_cmp(cmp, flags, true, true);
for_each_set_bit(xregno, all, bits) {
if (xregno == XREG_STACK_ARGS) {
arg_cnt = max(fa->out_stack_arg_cnt, fb->out_stack_arg_cnt);
@@ -355,6 +362,10 @@ void bpf_compare_states(struct bpf_verifier_env *env, const struct bpf_verifier_
} else {
flags = compare_slots(env, cmp, fa, fb, bpf_xreg_to_spi(xregno));
}
+ if (iv_loop && fr == old->curframe && !(flags & CMP_TYPE) &&
+ test_bit(xregno, iv_loop->linear_xregs) &&
+ !test_bit(xregno, iv_loop->nonlinear_xregs))
+ flags &= ~(CMP_DISJOINT | CMP_GAP);
note_cmp(cmp, flags, test_bit(xregno, data_deps), test_bit(xregno, deps));
if (fr == old->curframe && (flags & CMP_DISJOINT) &&
test_bit(xregno, data_deps))
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 33/49] bpf: Verify a static function once for each instance
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (31 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 32/49] bpf: Use the equations in the join rule Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 8:33 ` sashiko-bot
2026-11-20 7:42 ` [PATCH bpf-next 34/49] bpf: Partition the states after an error Alexei Starovoitov
` (15 subsequent siblings)
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A chain of calls multiplies the work if each call is verified in the
frames of its caller. 18 levels with 2 calls each are 1 << 18 chains.
A function that cannot get to the frames of its callers is verified on
its own. The old pass united two calls if their states met anywhere in
the function. Then each call got the results of both. Now the states of
two calls meet only at the first insn:
call1 f(0) --+
call2 f(1) --+--> first insn
in pass 0 join -> one instance
if r1 is in data_deps keep apart -> two instances
An instance is an entry state, its exit states and the calls that wait
for them. So the result of a call goes with its arguments. An instance
that has started does not grow. The join starts a new instance:
call1 f(0) instance 1: f(0) exits 1 -> call1
call2 f(1) instance 2: f(0..1) exits 1 -> call2, call3 at once
call3 f(2) f(0..2) exits 2 -> call2, call3 later
A call waits if it has no exit state to continue with, or if it is in
a peeled trip.
Instances can multiply too. The functions that have more than one
instance divide a budget. When a function has used its part, the next
calls are joined, see flow_may_add_instance().
A function can test a reference of its caller for NULL and return it.
bpf_flow_return() then marks the copies in the caller as not NULL. Else
'if (!n)' drops the reference, and copy still has its id:
copy = throw_if_null(n); if (!n) return copy->a;
A speculative call gets an instance too. flow_end_instance() frees the
calls that have all exit states.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 7 +-
kernel/bpf/flow.c | 317 +++++++++++++++++++++++++++++++++++
kernel/bpf/verifier.c | 20 ++-
3 files changed, 335 insertions(+), 9 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index c2636f5a6ed..44c99bf1bf5 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -607,11 +607,6 @@ struct bpf_verifier_state {
* and their packet pointers are not in the state, see bpf_flow_return().
*/
bool changed_pkt_data;
- /*
- * Not zero in a function that is verified on its own: the call that the
- * state at its entry came from, see bpf_flow_call().
- */
- u32 flow_entry;
);
/* Not zero in a function that is verified on its own, see bpf_flow_call() */
u32 instance_id;
@@ -1143,6 +1138,7 @@ struct bpf_subprog_info {
u32 callx_edge; /* 1 + the first of its env->callx_edges, 0 if none */
struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */
u32 deepest_insn; /* the insn that set stack_depth */
+ u32 first_exit; /* see bpf_flow_exit() */
u16 stack_depth; /* max. stack depth used by this function */
u16 stack_extra;
u16 arena_stack_depth; /* size of the frame in the stack in arena */
@@ -1373,6 +1369,7 @@ struct bpf_verifier_env {
bool over_limit; /* see bpf_check_limits() */
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_needs_guard() */
+ bool calls_joined; /* see flow_may_add_instance() */
bool extra_values; /* a state can have constants that no run has */
bool new_linear_xreg; /* see bpf_same_insns_unproven() */
bool seen_direct_write;
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 2acf85a10d4..3087923ba79 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -71,6 +71,12 @@ struct flow_state {
INTERVAL_TREE_DEFINE(struct flow_state, tree_node, u64, subtree_last, FLOW_FIRST, FLOW_LAST, static,
flow_tree)
+/* Instances that can wait for each other, see flow_find_ended() */
+struct flow_group {
+ u32 busy_cnt; /* pending states, and open calls to other groups */
+ struct flow_instance *instances;
+};
+
/* One check of a function: its entry state, its exit states and the calls that get them */
struct flow_instance {
u32 id;
@@ -79,8 +85,11 @@ struct flow_instance {
u32 pending_cnt; /* states */
/* calls from this instance that can get more exit states, and those that call itself */
u32 open_call_cnt, self_call_cnt;
+ struct flow_group *group;
+ struct flow_instance *next_in_group;
u64 *cost; /* of its function, see flow_may_add_instance() */
bool in_peeled_trip; /* a call of it, or of a caller, is in a peeled trip */
+ bool replaced; /* no new call finds it, see flow_grow_instance() */
bool ended; /* see flow_end_instance() */
struct flow_instance *next;
struct list_head waiting_calls; /* calls of this instance */
@@ -113,6 +122,7 @@ struct bpf_flow {
struct rb_root_cached state_tree;
u32 *nr_states; /* per insn */
struct rb_root_cached pending;
+ struct flow_group *groups; /* bpf_subprog_info.cycle is the index */
struct flow_instance **instances; /* bpf_verifier_state.instance_id is the index */
u32 instance_cnt, instance_cap;
struct flow_instance *last_instance; /* of the last pending state */
@@ -537,6 +547,7 @@ int bpf_flow_prepare(struct bpf_verifier_env *env)
flow->pos = kvcalloc(2 * len, sizeof(u32), BPF_VERIFIER_GFP);
flow->rank = kvcalloc(env->subprog_cnt + 1, sizeof(u32), BPF_VERIFIER_GFP);
flow->cost = kvcalloc(env->subprog_cnt, sizeof(u64), BPF_VERIFIER_GFP);
+ flow->groups = kvcalloc(env->subprog_cnt, sizeof(*flow->groups), BPF_VERIFIER_GFP);
flow->bb_starts = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
flow->join_points = kvcalloc(BITS_TO_LONGS(len), sizeof(long), BPF_VERIFIER_GFP);
flow->nr_states = kvcalloc(len, sizeof(u32), BPF_VERIFIER_GFP);
@@ -626,6 +637,7 @@ static void flow_clear_pending(struct bpf_flow *flow, struct flow_state *s)
return;
rb_erase_cached(&s->rb, &flow->pending);
flow_state_instance(flow, s)->pending_cnt--;
+ flow_state_instance(flow, s)->group->busy_cnt--;
s->pending = false;
}
@@ -640,6 +652,7 @@ static void flow_unlink(struct bpf_verifier_env *env, struct flow_state *s)
instance = flow_state_instance(env->flow, s);
instance->open_call_cnt--;
instance->self_call_cnt -= instance == s->callee;
+ instance->group->busy_cnt -= instance->group != s->callee->group;
s->callee = NULL;
}
if (!RB_EMPTY_NODE(&s->tree_node))
@@ -680,6 +693,7 @@ void bpf_flow_reset(struct bpf_verifier_env *env)
for (i = 0; i < flow->instance_cnt; i++) {
flow->instances[i]->group->instances = NULL;
kfree(flow->instances[i]);
+ }
env->mem -= flow->instance_cnt * INSTANCE_BYTES;
flow->instance_cnt = 0;
flow->last_instance = NULL;
@@ -695,6 +709,7 @@ void bpf_flow_free(struct bpf_verifier_env *env)
kvfree(flow->pos);
kvfree(flow->rank);
kvfree(flow->cost);
+ kvfree(flow->groups);
kvfree(flow->bb_starts);
kvfree(flow->join_points);
kvfree(flow->nr_states);
@@ -781,6 +796,7 @@ static void flow_set_pending(struct bpf_verifier_env *env, struct flow_state *s)
}
s->pending = true;
instance->pending_cnt++;
+ instance->group->busy_cnt++;
rb_add_cached(&s->rb, &flow->pending, flow_less);
}
@@ -938,6 +954,7 @@ static bool flow_loop_is_left(struct flow_state *s, const struct flow_state *nex
static struct flow_instance *flow_new_instance(struct bpf_verifier_env *env,
struct flow_state *entry)
{
+ const struct bpf_func_state *f = entry ? entry->st.frame[0] : NULL;
struct bpf_flow *flow = env->flow;
struct flow_instance **instances, *instance;
u32 cap;
@@ -957,6 +974,11 @@ static struct flow_instance *flow_new_instance(struct bpf_verifier_env *env,
INIT_LIST_HEAD(&instance->exit_states);
INIT_LIST_HEAD(&instance->done);
instance->entry_state = entry;
+ instance->group = flow->groups;
+ if (f && f->loop_stack_cnt && !f->loop_stack->peeled)
+ instance->group += env->subprog_info[f->subprogno].cycle;
+ instance->next_in_group = instance->group->instances;
+ instance->group->instances = instance;
env->mem += INSTANCE_BYTES;
instance->id = flow->instance_cnt++;
flow->instances[instance->id] = instance;
@@ -1334,6 +1356,301 @@ int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st
return flow_add(env, s, false);
}
+/* @x is an exit state of the instance of the call @w. @w can leave the instance. */
+static int flow_apply_exit(struct bpf_verifier_env *env, struct flow_state *w,
+ struct flow_state *x)
+{
+ int call_idx = w->st.insn_idx, err;
+ struct bpf_verifier_state st = {};
+
+ err = bpf_copy_verifier_state(env, &st, &w->st) ?:
+ bpf_flow_return(env, &st, &x->st, call_idx);
+ if (!err)
+ err = bpf_flow_deliver(env, &st, call_idx + 1, call_idx, false);
+ else if (err < 0 && st.speculative)
+ err = min(bpf_flow_edge_unproven(env, &st, call_idx + 1, err), 0);
+ else if (err < 0)
+ err = min(bpf_flow_edge_unproven(env, &x->st, x->prev_insn_idx, err), 0);
+ bpf_free_verifier_state(env, &st, false);
+ if (err <= 0)
+ return err;
+
+ /* The call is checked again in the frames of the caller */
+ flow_unlink(env, w);
+ flow_find(env, w);
+ flow_link(env, w);
+ return 0;
+}
+
+/* The arguments that are readable */
+static u64 flow_readable_args(const struct bpf_func_state *f)
+{
+ u64 mask = 0;
+ u32 i;
+
+ for (i = BPF_REG_1; i <= BPF_REG_5; i++)
+ mask |= (u64)(f->regs[i].type != NOT_INIT) << i;
+ for (i = 0; i < f->out_stack_arg_cnt; i++)
+ mask |= (u64)(f->stack_arg_regs[i].type != NOT_INIT) << (MAX_BPF_REG + i);
+ return mask;
+}
+
+/* The exit states of @old are exit states of @instance too */
+static int flow_copy_exit_states(struct bpf_verifier_env *env, struct flow_instance *instance,
+ struct flow_instance *old)
+{
+ struct flow_state *x, *copy;
+ int err;
+
+ list_for_each_entry(x, &old->exit_states, node) {
+ if (x->pending)
+ continue;
+ copy = flow_new_state(env, x->prev_insn_idx);
+ if (!copy)
+ return -ENOMEM;
+ err = bpf_copy_verifier_state(env, ©->st, &x->st);
+ if (err) {
+ flow_free_state(env, copy);
+ return err;
+ }
+ copy->kind = FLOW_FUNC_EXIT;
+ copy->st.instance_id = instance->id;
+ flow_find(env, copy);
+ flow_insert(env, copy);
+ }
+ return 0;
+}
+
+/* @s gets to the first insn. @t is there and does not subsume @s. */
+static int flow_grow_instance(struct bpf_verifier_env *env, struct flow_state *t,
+ struct flow_state *s, struct bpf_loop *leftover_loop)
+{
+ u64 args = flow_readable_args(t->st.frame[0]);
+ struct flow_instance *old = t->started_instance, *instance = old;
+ int err;
+
+ if (!old->entry_state) {
+ instance = flow_new_instance(env, t);
+ if (!instance)
+ return -ENOMEM;
+ instance->cost = old->cost;
+ }
+ flow_unlink(env, t);
+ err = bpf_states_join(env, &t->st, &s->st, leftover_loop ?: &env->flow->call_loop,
+ instance != old);
+ t->started_instance = instance;
+ flow_find(env, t);
+ flow_link(env, t);
+ if (args != flow_readable_args(t->st.frame[0]))
+ flow_free_list(env, &instance->exit_states);
+ else if (!err && instance != old)
+ err = flow_copy_exit_states(env, instance, old);
+ old->replaced = instance != old;
+ if (old->ended)
+ flow_free_list(env, &old->exit_states);
+ return err;
+}
+
+/*
+ * A chain of calls multiplies the instances. The functions that have more than one instance
+ * divide a budget. When a function has used its part, the next calls are joined.
+ */
+static bool flow_may_add_instance(struct bpf_verifier_env *env, int subprog)
+{
+ struct bpf_flow *flow = env->flow;
+
+ if (flow->cost[subprog] * flow->budget_func_cnt <= flow->budget)
+ return true;
+ env->calls_joined = true;
+ return false;
+}
+
+static bool flow_in_peeled_trip(const struct bpf_verifier_state *st)
+{
+ const struct loop_stack_entry *e;
+ u32 fr, i;
+
+ for (fr = 0; fr <= st->curframe; fr++)
+ for (i = 0, e = st->frame[fr]->loop_stack; i < st->frame[fr]->loop_stack_cnt; i++)
+ if (e[i].peeled && e[i].loop->kind != BPF_LOOP_RECURSION)
+ return true;
+ return false;
+}
+
+/*
+ * The call at @call_idx in @caller goes to a function that is verified on its own. @entry is
+ * the state at its first insn. Both states are taken. The first insn is the only join point
+ * for the states of two calls.
+ *
+ * call1 f(0) --+ pass 0: join -> one instance for both calls
+ * call2 f(1) --+--> first insn of f data dep r1 differs -> one instance for each call
+ *
+ * An instance that has started does not grow. The join starts a new instance:
+ *
+ * call1 f(0) instance 1: f(0) exits 1 --> call1
+ * call2 f(1) instance 2: f(0..1) exits 1 --> call2, call3 at once: the caller continues
+ * call3 f(2) f(0..2) exits 2 --> call2, call3 later, if they have more
+ *
+ * An argument can have another type or object. Then the old exit states can refer to what the
+ * new call did not pass in, so the call waits. It waits in a peeled trip too: a late exit state
+ * checks all later trips again.
+ */
+int bpf_flow_call(struct bpf_verifier_env *env, struct bpf_verifier_state *caller,
+ struct bpf_verifier_state *entry, int subprog, int call_idx)
+{
+ struct loop_stack_entry *e = bpf_top_loop(entry->frame[0]);
+ struct bpf_loop *leftover_loop = e && e->trip && !e->peeled ? e->loop : NULL;
+ struct flow_instance *instance, *from = env->flow->instances[caller->instance_id];
+ u64 *cost = &env->flow->cost[subprog];
+ struct flow_state *w, *s, *t, *tn, *x;
+ bool subsumed;
+ int err;
+
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "call at %d waits for func#%d\n", call_idx, subprog);
+
+ s = flow_new_state(env, call_idx);
+ if (!s)
+ return -ENOMEM;
+ swap(s->st, *entry);
+ s->st.insn_idx = env->subprog_info[subprog].start;
+ s->st.last_insn_idx = call_idx;
+ s->kind = FLOW_FUNC_ENTRY;
+ err = bpf_clean_verifier_state(env, &s->st);
+ if (err)
+ goto free;
+again:
+ flow_for_each_near(env, s, t, tn) {
+ if (!flow_same_join_key(env, t, s) ||
+ bpf_must_keep_apart(env, &t->st, &s->st, s->kind == FLOW_JOINED_CALLS ||
+ (leftover_loop && flow_same_entry(leftover_loop, t, s)) ?
+ BPF_JOIN_POINT_HEADER : BPF_JOIN_POINT_CALLS,
+ leftover_loop ? e : NULL))
+ continue;
+ subsumed = bpf_state_subsumes(env, &t->st, &s->st);
+ if (!subsumed && t->st.speculative != s->st.speculative)
+ continue;
+ env->imprecise |= !env->err;
+ if (!subsumed)
+ err = flow_grow_instance(env, t, s, leftover_loop);
+ flow_free_state(env, s);
+ if (err)
+ return err;
+ goto wait;
+ }
+ if (s->kind == FLOW_FUNC_ENTRY && !flow_may_add_instance(env, subprog)) {
+ s->kind = FLOW_JOINED_CALLS;
+ goto again;
+ }
+ t = s;
+ t->started_instance = flow_new_instance(env, t);
+ if (!t->started_instance) {
+ err = -ENOMEM;
+ goto free;
+ }
+ /* the first instance is free */
+ if ((*cost)++)
+ t->started_instance->cost = cost;
+ env->flow->budget_func_cnt += *cost == 2;
+ flow_link(env, t);
+wait:
+ w = flow_new_state(env, caller->last_insn_idx);
+ if (!w)
+ return -ENOMEM;
+ swap(w->st, *caller);
+ w->st.insn_idx = call_idx;
+ instance = w->callee = t->started_instance;
+ from->open_call_cnt++;
+ from->self_call_cnt += from == instance;
+ from->group->busy_cnt += from->group != instance->group;
+ flow_insert(env, w);
+ instance->in_peeled_trip |= t->kind == FLOW_FUNC_ENTRY &&
+ (from->in_peeled_trip || flow_in_peeled_trip(&w->st));
+ if (instance->entry_state &&
+ (list_empty(&instance->exit_states) || instance->in_peeled_trip))
+ flow_set_pending(env, w);
+ list_for_each_entry(x, &instance->exit_states, node)
+ if (!err && !x->pending && w->callee)
+ err = flow_apply_exit(env, w, x);
+ if (instance->ended && w->callee)
+ flow_free_state(env, w);
+ return err;
+free:
+ flow_free_state(env, s);
+ return err;
+}
+
+/* @st came to exit of the function that is verified on its own. It is taken. */
+int bpf_flow_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int exit_idx)
+{
+ struct bpf_subprog_info *sub = &env->subprog_info[st->frame[0]->subprogno];
+ struct loop_stack_entry *e = bpf_top_loop(st->frame[0]);
+ struct flow_state *x;
+ int err;
+
+ st->insn_idx = exit_idx;
+ err = bpf_clean_verifier_state(env, st);
+ if (err)
+ return err;
+ x = flow_new_state(env, exit_idx);
+ if (!x)
+ return -ENOMEM;
+ x->st = *st;
+ memset(st, 0, sizeof(*st));
+ /* all exit states are compared at the first exit */
+ sub->first_exit = sub->first_exit ?: exit_idx;
+ x->st.insn_idx = sub->first_exit;
+ x->kind = FLOW_FUNC_EXIT;
+ return flow_prune_or_join(env, x, e && !e->peeled ? e->loop : NULL, false);
+}
+
+static void flow_add_ended(struct flow_instance *instance, struct flow_instance **ended)
+{
+ if (instance->ended)
+ return;
+ instance->ended = true;
+ instance->next = *ended;
+ *ended = instance;
+}
+
+/*
+ * A state of @instance comes from another state of it, or from an exit state that one of its
+ * open calls gets. If there is none of these, @instance ended. The same holds for a group.
+ * The instances in the leftover of a recursion are a group: they can wait for each other.
+ * All other instances are group 0.
+ */
+static void flow_find_ended(struct flow_instance *instance, struct flow_instance **ended)
+{
+ struct flow_group *group = instance->group;
+
+ if (!instance->pending_cnt && instance->open_call_cnt == instance->self_call_cnt)
+ flow_add_ended(instance, ended);
+ while (!group->busy_cnt && group->instances) {
+ flow_add_ended(group->instances, ended);
+ group->instances = group->instances->next_in_group;
+ }
+}
+
+/* If an instance ended, its waiting calls have all exit states */
+static void flow_end_instance(struct bpf_verifier_env *env, struct flow_instance *instance)
+{
+ struct flow_instance *from, *ended = NULL;
+ struct flow_state *w, *wn;
+
+ flow_find_ended(instance, &ended);
+ while ((instance = ended)) {
+ ended = instance->next;
+ list_for_each_entry_safe(w, wn, &instance->waiting_calls, node) {
+ from = flow_state_instance(env->flow, w);
+ flow_free_state(env, w);
+ flow_find_ended(from, &ended);
+ }
+ flow_free_list(env, &instance->done);
+ if (instance->replaced)
+ flow_free_list(env, &instance->exit_states);
+ }
+}
+
/*
* Make the next pending state the current one. env->cur_state has to be empty.
* Returns -ENOENT when nothing is pending.
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6373505c672..4068014fedc 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1817,11 +1817,11 @@ int bpf_copy_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_st
return err;
bpf_add_cost(env, bpf_xreg_cnt(src) + src->acquired_refs, BPF_COST_COPY);
dst_state->speculative = src->speculative;
- dst_state->flow_entry = src->flow_entry;
dst_state->infeasible = src->infeasible;
dst_state->max_held_cnt = src->max_held_cnt;
dst_state->suspect_loops = src->suspect_loops;
dst_state->join_key = src->join_key;
+ dst_state->instance_id = src->instance_id;
dst_state->insn_idx = src->insn_idx;
dst_state->last_insn_idx = src->last_insn_idx;
for (i = 0; i <= src->curframe; i++) {
@@ -10927,7 +10927,7 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
mark_reg_invalid(env, reg);
}
}));
- if (env->cur_state->flow_entry)
+ if (env->cur_state->instance_id)
env->cur_state->changed_pkt_data = true;
}
@@ -11515,7 +11515,7 @@ static bool func_is_on_its_own(struct bpf_verifier_env *env, struct bpf_verifier
if (sub->changes_pkt_data || sub->has_tail_call || sub->has_ld_abs ||
sub->is_cb || sub->is_async_cb || sub->is_exception_cb)
return false;
- if (state->speculative)
+ if (env->path_sensitive)
return false;
if (sub->releases_inherited &&
(state->acquired_refs || state->active_rcu_locks || state->active_preempt_locks))
@@ -11560,6 +11560,7 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
if (!callee)
return -ENOMEM;
entry.frame[0] = callee;
+ entry.speculative = state->speculative;
entry.in_sleepable = state->in_sleepable;
entry.suspect_loops = state->suspect_loops;
entry.callers_unknown = state->callers_unknown;
@@ -11628,6 +11629,8 @@ static int exit_func_on_its_own(struct bpf_verifier_env *env)
return err ?: PROCESS_BPF_EXIT;
}
+static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, bool is_null);
+
static u32 returned_id(u32 id)
{
return bpf_id_is_xreg(id) ? BPF_RETURNED_ID(id) : id;
@@ -11646,6 +11649,7 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
struct bpf_reg_state *reg;
int err;
+ caller->speculative |= exit->speculative;
/*
* Keep only what the function held all the time. The state at its exit looks the same if
* the function released a lock and acquired it again. Else verify the function in the
@@ -11676,6 +11680,8 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
* the packet pointers of the caller is found out in the context of the caller.
*/
if (exit->changed_pkt_data) {
+ if (exit->speculative)
+ return -EACCES;
env->subprog_info[callee->subprogno].changes_pkt_data = true;
return BPF_EXIT_NEEDS_CALLER;
}
@@ -11700,6 +11706,9 @@ int bpf_flow_return(struct bpf_verifier_env *env, struct bpf_verifier_state *cal
reg->id = returned_id(reg->id);
reg->parent_id = returned_id(reg->parent_id);
reg->map_uid = returned_id(reg->map_uid);
+ /* the function saw that a reference of the caller is not NULL */
+ if (reg->id && !type_may_be_null(reg->type))
+ mark_ptr_or_null_regs(caller, ret_regs[i], false);
}
return 0;
}
@@ -20327,7 +20336,7 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
{
struct bpf_func_state *cur_frame = cur_func(env);
- if (!exception_exit && !env->cur_state->curframe && env->cur_state->flow_entry)
+ if (!exception_exit && !env->cur_state->curframe && env->cur_state->instance_id)
return exit_func_on_its_own(env);
/* We must do check_reference_leak here before
@@ -21530,6 +21539,9 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
from_unproven = reads_from_unproven(env);
fixups = insn_aux->fixups;
err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
+ /* a call or an exit gave the state to the flow, the error is from there */
+ if (err < 0 && !state->frame[0])
+ return err;
insn_aux->releases |= held_cnt(state) < held && err != PROCESS_BPF_EXIT;
insn_aux->acquires |= held_cnt(state) > held;
if (nospec_ends_path(env, state, err)) {
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 33/49] bpf: Verify a static function once for each instance
2026-11-20 7:42 ` [PATCH bpf-next 33/49] bpf: Verify a static function once for each instance Alexei Starovoitov
@ 2026-11-20 8:33 ` sashiko-bot
0 siblings, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:33 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] bpf: missing early return in do_check_block() after
check_func_callx_set()
--
commit 609a14fb5ab2e00b0fa47a3b3041980c47a05bcf
Author: Alexei Starovoitov <ast@kernel.org>
bpf: Verify a static function once for each instance
This commit verifies static functions that do not access their caller's
frames once per instance, joining call states only at the function entry
instruction within a per-function instance budget.
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 6373505c672d9..4068014fedc5d 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -21530,6 +21539,9 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
> from_unproven = reads_from_unproven(env);
> fixups = insn_aux->fixups;
> err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
> + /* a call or an exit gave the state to the flow, the error is from there */
> + if (err < 0 && !state->frame[0])
> + return err;
> insn_aux->releases |= held_cnt(state) < held && err != PROCESS_BPF_EXIT;
> insn_aux->acquires |= held_cnt(state) > held;
> if (nospec_ends_path(env, state, err)) {
[Severity: Medium]
Does checking !state->frame[0] miss negative errors returned through
check_func_callx_set()?
When a callx instruction targets a set of functions,
check_func_callx_set() passes a temporary copy as env->cur_state to
check_func_callx_one() -> call_func_on_its_own() -> bpf_flow_call() and
then restores env->cur_state = cur:
kernel/bpf/verifier.c:check_func_callx_set() {
...
if (!err) {
env->cur_state = copy;
err = check_func_callx_one(env, set->subprog[i], insn_idx);
env->cur_state = cur;
}
bpf_free_verifier_state(env, copy, true);
...
}
bpf_flow_call() consumes copy via swap(w->st, *caller), leaving
state->frame[0] in cur non-NULL. If flow_apply_exit() then returns a
negative recoverable error (such as from bpf_flow_edge_unproven()),
do_check_block() skips this early return and falls into the
error_is_recoverable(err) path for the callx instruction itself, recording
the error on the callx instruction and continuing the pass via
skip_unproven_insn() when last_unproven_left was already decremented to 0.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=33
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 34/49] bpf: Partition the states after an error
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (32 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 33/49] bpf: Verify a static function once for each instance Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 35/49] bpf: Reproduce an error on one path, with no joins Alexei Starovoitov
` (14 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
If a pass has an error, what does the next pass keep apart? This is
trace partitioning. More precision costs more, so take the cheap steps
first. bpf_partition_more() takes the first line that adds something:
ADD_DEPS the deps of the unproven insns
PEEL suspect loops: one trip, then what a budget pays
PEELED_APART suspect loops: the join rule holds at the start
of a peeled trip
TRANSITIVE_DEPS data_deps |= control_deps; add the jumps that can
skip their writes
DISJOINT_APART disjoint values in data_deps stay apart
LOOP_DISJOINT_APART suspect loops: the same at the header
PEEL_MORE suspect loops: 4, 16, 64, ... trips; no prune of
a peeled trip if the last pass pruned one
IN_FRAMES calls with an argument that is in the deps and
has an equation: if (!ok(x)) return; buf[x]
A step can be the wrong one, and use up the limit of the load: to peel
a loop of 100000 trips does not help if the error needs control_deps. So
each step but the first has a budget, and can be undone:
+-----------------+ in the budget, or it helped
+--->| partition, pass |------------------------------> keep
| +--------+--------+
| | over its part of the budget
| v
+--- undo, take the next line
| | no line left, one was too costly
| v
+--- budget *= 4, undo all since one helped
If a step was too costly for a loop, a later step is a second choice.
It is undone when the budget grows, also if it helped:
LOOP_DISJOINT_APART makes each later peel much more costly.
The first budget is two times the cost of a pass. charge_loops() gives
the cost of a state to the loops that keep it apart.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 23 ++
kernel/bpf/equation.c | 12 +
kernel/bpf/precision.c | 500 +++++++++++++++++++++++++++++++++++
kernel/bpf/precision.h | 20 ++
kernel/bpf/verifier.c | 32 ++-
5 files changed, 586 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 44c99bf1bf5..540608bcd8f 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -839,6 +839,7 @@ struct bpf_loop_per_pass {
bool peel_pruned; /* see flow_prune_peeled() */
/* a check failed in a state that the loop kept apart, with no other suspect */
bool has_unproven;
+ bool starves, starved; /* see charge_loops() */
u32 last_peeled_trip; /* see flow_keep_peeled() */
u32 peeled_cnt; /* trips that the pass peeled */
u64 cost; /* for the states that the loop keeps apart */
@@ -871,7 +872,12 @@ struct bpf_loop {
bool disjoint_apart; /* disjoint values in data_deps stay apart, also in the leftover */
bool no_peel_prune; /* a larger earlier trip does not prune, see flow_prune_peeled() */
);
+ struct bpf_loop_precision undo; /* before the last partition */
+ struct bpf_loop_precision kept; /* see keep_precision() */
struct bpf_loop_per_pass last_pass; /* the last pass whose partition was kept */
+ /* to peel all trips, estimated from a partition that was over its budget */
+ u64 full_peel_cost;
+ u16 tried; /* bits of PARTITION_*, with this budget */
bool partitioned; /* the last partition was for this loop */
bool peeled_joined; /* at the start of a peeled trip, against the join rule */
bool changed; /* see flow_forget_loop() */
@@ -1350,9 +1356,11 @@ struct bpf_verifier_env {
bool strict_alignment; /* perform strict pointer alignment checks */
bool test_reg_invariants; /* fail verification on register invariants violations */
bool uses_arena_sp; /* the program has insns with BPF_REG_ARENA_SP */
+ bool replay; /* the good pass, once more for the driver */
bool path_sensitive; /* no joins, see run_path_sensitive() */
bool for_log; /* the verdict is known, see do_check_all() */
bool fallthrough_first; /* see find_failing_path() */
+ u32 pass; /* number of the pass, see do_check_all() */
struct bpf_precision *precision;
u64 cost, mem; /* see bpf_check_limits() */
u64 max_cost, max_mem;
@@ -1696,6 +1704,8 @@ u64 bpf_hash_apart_xregs(struct bpf_verifier_env *env, const struct bpf_verifier
u64 bpf_hash_args(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 hash);
void bpf_range_of_data_dep(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 *first, u32 *last);
+bool bpf_call_needs_frames(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u16 args);
bool bpf_xreg_in_data_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 fr, u32 xregno);
bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
@@ -1707,6 +1717,15 @@ bool bpf_insn_is_unproven(struct bpf_verifier_env *env);
void bpf_note_fixups_differ(struct bpf_verifier_env *env);
void bpf_find_unproven_after_pass(struct bpf_verifier_env *env);
bool bpf_same_insns_unproven(struct bpf_verifier_env *env);
+int bpf_partition_more(struct bpf_verifier_env *env);
+bool bpf_undo_precision(struct bpf_verifier_env *env);
+void bpf_precision_new_pass(struct bpf_verifier_env *env);
+u64 bpf_pass_max_cost(struct bpf_verifier_env *env);
+u64 bpf_precision_budget(struct bpf_verifier_env *env);
+u64 bpf_precision_first_budget(struct bpf_verifier_env *env);
+int bpf_precision_init(struct bpf_verifier_env *env);
+void bpf_precision_free(struct bpf_verifier_env *env);
+
/*
* The two limits of the verifier, for the whole load. The unit of cost is one step of a scan,
* about a dozen insns of the CPU. Memory is the bytes of states and tables at a time.
@@ -1718,6 +1737,9 @@ bool bpf_same_insns_unproven(struct bpf_verifier_env *env);
#define BPF_COST_STATE 512 /* make, order and free a state */
#define BPF_MAX_COST (1ULL << 33)
#define BPF_MAX_MEM (1ULL << 31)
+/* The floors of the cost after the first error, and of the budget of a partition */
+#define BPF_PARTITION_BUDGET (BPF_MAX_COST / 3)
+#define BPF_MIN_BUDGET (BPF_MAX_COST >> 12)
/* Without CAP_BPF programs are small, and their loads wait for each other */
#define BPF_LIMITS_SHIFT_UNPRIV 5
@@ -2456,6 +2478,7 @@ void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *fra
void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
bool taken);
+u16 bpf_args_with_equation(const struct bpf_verifier_state *st);
void bpf_equations_next_trip(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
u32 depth);
bool bpf_equations_subsume(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index fc05ca7dd60..4216fd00879 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -1217,6 +1217,18 @@ static void add_distance_equation(struct bpf_verifier_env *env, const struct bpf
env->insn_idx, a.off - b.off });
}
+/* At a call bpf_equations_before_insn() moves the equations of r1-r5 to the next frame */
+u16 bpf_args_with_equation(const struct bpf_verifier_state *st)
+{
+ u16 args = 0;
+ u32 r;
+
+ for (r = BPF_REG_1; r <= BPF_REG_5; r++)
+ if (bpf_find_xreg_equation(&st->equations, bpf_xreg(st->curframe + 1, r)))
+ args |= BIT(r);
+ return args;
+}
+
/* Note the operands. Delete the equations of the registers that the insn writes. */
void bpf_equations_before_insn(struct bpf_verifier_env *env)
{
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index aa9f7f76256..42d9fd0ee72 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -62,6 +62,28 @@ bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_st
return test_bit(xregno, data_deps) || test_bit(xregno, control_deps);
}
+/*
+ * On its own a function does not get the equations of its arguments @args:
+ * if (!ok(x)) return 0; buf[x]; the test in ok() has to bound x
+ */
+bool bpf_call_needs_frames(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u16 args)
+{
+ struct bpf_precision *prec = env->precision;
+ u32 func = st->frame[st->curframe]->subprogno, idx = st->insn_idx;
+
+ args &= bpf_args_with_equation(st);
+ if (prec->data_deps) {
+ if (args & *mask_at(env, prec->in_frames, func, idx))
+ return true;
+ prec->this_pass.arg_lost_equation |=
+ args & (*mask_at(env, prec->data_deps, func, idx) |
+ *mask_at(env, prec->control_deps, func, idx));
+ }
+ env->imprecise |= args && !env->err;
+ return false;
+}
+
/* The value in a slot. NULL if the slot has bytes of several types. */
static const struct bpf_reg_state *slot_value(struct bpf_verifier_env *env,
const struct bpf_func_state *f, u32 spi,
@@ -1037,6 +1059,157 @@ static int compute_deps_of_unproven(struct bpf_verifier_env *env)
return add_control_deps(env);
}
+/* An unreadable xreg needs one type and object, whatever the numbers are. Try that first. */
+static int add_deps(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ u32 len = env->prog->len, i;
+ int ret, more = 0;
+
+ if (!bitmap_empty(prec->last_pass.bad_read_insns, len)) {
+ ret = bpf_compute_deps(env, prec->new_data_deps, prec->last_pass.bad_read_insns);
+ if (ret)
+ return ret;
+ more = add_masks(prec, prec->type_apart, prec->new_data_deps);
+ }
+ if (more)
+ bitmap_andnot(prec->last_pass.unproven_insns, prec->last_pass.unproven_insns,
+ prec->last_pass.bad_read_insns, len);
+ bitmap_or(prec->bad_read_insns_done, prec->bad_read_insns_done,
+ prec->last_pass.bad_read_insns, len);
+ ret = compute_deps_of_unproven(env);
+ for (i = 0; i < 5 * prec->words; i++)
+ prec->kept[i] |= prec->data_deps[i] & ~prec->undo[i];
+ bitmap_or(prec->earlier_unproven_insns, prec->earlier_unproven_insns,
+ prec->last_pass.unproven_insns, len);
+ return ret < 0 ? ret : ret | more;
+}
+
+/* Add the jumps that decide which value an xreg of data_deps has */
+static int add_transitive_deps(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ u32 cnt, len = env->prog->len;
+ int ret = compute_deps_of_unproven(env);
+
+ while (!ret) {
+ cnt = bitmap_weight(prec->control_seeds, len);
+ add_masks(prec, prec->new_data_deps, prec->new_control_deps);
+ bitmap_or(prec->controlled_insns, prec->controlled_insns, prec->def_insns, len);
+ bitmap_or(prec->controlled_insns, prec->controlled_insns,
+ prec->last_pass.call_insns, len);
+ bitmap_or(prec->controlled_insns, prec->controlled_insns, prec->control_seeds, len);
+ ret = add_control_deps(env);
+ if (bitmap_weight(prec->control_seeds, len) == cnt)
+ break;
+ }
+ return ret;
+}
+
+/* A copy of the deps. ADD_DEPS has no budget, so it must not add calls. */
+static int check_calls_in_frames(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ return prec->last_pass.arg_lost_equation &&
+ (add_masks(prec, prec->in_frames, prec->new_data_deps) |
+ add_masks(prec, prec->in_frames, prec->new_control_deps));
+}
+
+static int keep_disjoint_apart(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ return prec->last_pass.disjoint_joined &&
+ add_masks(prec, prec->disjoint_apart, prec->new_data_deps);
+}
+
+/* @loop is a suspect. Returns false if @kind adds nothing. */
+static bool partition_loop(struct bpf_loop *loop, u32 kind)
+{
+ bool open = loop->last_pass.has_leftover || loop->last_pass.peel_pruned ||
+ (loop->budget && loop->last_pass.cost > loop->budget);
+
+ switch (kind) {
+ case PARTITION_PEEL:
+ /* the first trip is often different from the other trips */
+ loop->max_peel_cnt = loop->peel_cnt || loop->exit_trip ? U32_MAX : 1;
+ break;
+ case PARTITION_PEEL_MORE:
+ loop->max_peel_cnt = min(4ULL * loop->peel_cnt ?: 1, U32_MAX);
+ loop->no_peel_prune |= loop->last_pass.peel_pruned;
+ break;
+ case PARTITION_PEELED_APART:
+ open = !loop->peeled_apart && loop->peeled_joined;
+ loop->peeled_apart = true;
+ loop->max_peel_cnt = U32_MAX;
+ break;
+ case PARTITION_LOOP_DISJOINT_APART:
+ open = !loop->disjoint_apart &&
+ (loop->last_pass.has_leftover || loop->peel_cnt > 1);
+ loop->peel_cnt = min(loop->peel_cnt, 1);
+ loop->disjoint_apart = true;
+ loop->max_peel_cnt = 0;
+ break;
+ }
+ return open;
+}
+
+/*
+ * The suspect loops divide half of the budget. If the cost to peel all trips is known, the
+ * loop gets that, or it waits.
+ */
+static int partition_suspect_loops(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ struct bpf_loop *loop;
+ u32 i, cnt = 0;
+ int more = 0;
+ u64 budget, bits = 0;
+
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ loop->partitioned = (prec->last_pass.suspect_loops & loop->suspect_bit) &&
+ !(loop->tried & prec->partitioned_by) &&
+ partition_loop(loop, __ffs(prec->partitioned_by));
+ if (!loop->partitioned)
+ loop->precision = loop->undo;
+ cnt += loop->partitioned;
+ bits |= loop->partitioned ? loop->suspect_bit : 0;
+ }
+ budget = div_u64(prec->budget / 2, cnt ?: 1);
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ if (!loop->partitioned)
+ continue;
+ loop->budget = max(loop->budget, budget);
+ if (prec->partitioned_by == BIT(PARTITION_PEEL) && loop->max_peel_cnt > 1)
+ loop->budget = max(loop->budget, loop->full_peel_cost);
+ if (!memcmp(&loop->precision, &loop->undo, sizeof(loop->undo)) ||
+ loop->budget > 2 * prec->budget)
+ loop->tried |= prec->partitioned_by | BIT(PARTITION_TOO_COSTLY);
+ loop->partitioned = (bits & prec->last_pass.first_suspect_loops) &&
+ !(loop->tried & prec->partitioned_by);
+ if (!loop->partitioned)
+ loop->precision = loop->undo;
+ prec->budget_left += loop->partitioned ? loop->budget - budget : 0;
+ loop->changed |= loop->partitioned;
+ more |= loop->partitioned;
+ }
+ return more;
+}
+
+static int (* const partition_by[])(struct bpf_verifier_env *env) = {
+ [PARTITION_ADD_DEPS] = add_deps,
+ [PARTITION_PEEL] = partition_suspect_loops,
+ [PARTITION_TRANSITIVE_DEPS] = add_transitive_deps,
+ [PARTITION_PEELED_APART] = partition_suspect_loops,
+ [PARTITION_DISJOINT_APART] = keep_disjoint_apart,
+ [PARTITION_LOOP_DISJOINT_APART] = partition_suspect_loops,
+ [PARTITION_PEEL_MORE] = partition_suspect_loops,
+ [PARTITION_IN_FRAMES] = check_calls_in_frames,
+};
+
static int alloc_masks(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *sub = env->subprog_info;
@@ -1090,3 +1263,330 @@ static int alloc_masks(struct bpf_verifier_env *env)
prec->loop_control_deps = masks + 17 * prec->words;
return 0;
}
+
+u64 bpf_precision_budget(struct bpf_verifier_env *env)
+{
+ return env->precision->budget;
+}
+
+u64 bpf_precision_first_budget(struct bpf_verifier_env *env)
+{
+ return max(2 * env->precision->pass_cost, BPF_MIN_BUDGET);
+}
+
+/* The limit on cost of a pass */
+u64 bpf_pass_max_cost(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ return env->cost + prec->pass_cost + prec->budget_left;
+}
+
+/*
+ * @keep: the partition helped, keep the precision. Else undo all since the last keep.
+ * If a partition was too costly for a loop, a later one is a second choice. It stays only
+ * until the budget grows.
+ */
+static void keep_precision(struct bpf_precision *prec, bool keep)
+{
+ struct bpf_loop *loop;
+ u16 second_choice = 0;
+ u32 i;
+
+ for (i = 0; keep && i < prec->loop_cnt; i++)
+ second_choice |= prec->loops[i]->tried & BIT(PARTITION_TOO_COSTLY);
+ prec->too_costly = 0;
+ for (i = 0; i < prec->loop_cnt; i++)
+ prec->loops[i]->tried &= second_choice;
+ if (second_choice)
+ return;
+ memcpy(keep ? prec->kept : prec->data_deps, keep ? prec->data_deps : prec->kept,
+ 5 * prec->words * sizeof(long));
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ if (keep) {
+ loop->kept = loop->precision;
+ loop->full_peel_cost = 0;
+ }
+ loop->changed |= !!memcmp(&loop->precision, &loop->kept, sizeof(loop->kept));
+ loop->precision = loop->kept;
+ }
+}
+
+/* The first partition that adds something. If all others are too costly, start again. */
+static int partition_next(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ int i, ret;
+ u16 wait;
+
+ prec->budget_left = prec->budget;
+ memcpy(prec->undo, prec->data_deps, 5 * prec->words * sizeof(long));
+ for (i = 0; i < prec->loop_cnt; i++) {
+ prec->loops[i]->undo = prec->loops[i]->precision;
+ prec->loops[i]->partitioned = false;
+ }
+ for (i = 0; i < ARRAY_SIZE(partition_by); i++) {
+ prec->partitioned_by = BIT(i);
+ ret = prec->too_costly & prec->partitioned_by ? 0 : partition_by[i](env);
+ ret = ret ?: prec->new_nospec;
+ if (ret)
+ return ret;
+ }
+ wait = prec->too_costly | prec->last_pass.calls_joined;
+ for (i = 0; i < prec->loop_cnt; i++)
+ if (prec->last_pass.first_suspect_loops & prec->loops[i]->suspect_bit)
+ wait |= prec->loops[i]->tried & BIT(PARTITION_TOO_COSTLY);
+ prec->partitioned_by = 0;
+ wait = wait && prec->budget < env->max_cost - env->cost;
+ prec->budget_left = prec->budget = min(4 * prec->budget, env->max_cost - env->cost);
+ prec->failing_path_due = prec->budget_grew = true;
+ return wait;
+}
+
+static void undo_loop(struct bpf_precision *prec, struct bpf_loop *loop)
+{
+ loop->precision = loop->undo;
+ loop->tried |= BIT(PARTITION_TOO_COSTLY);
+ if (!loop->this_pass.starved)
+ loop->tried |= prec->partitioned_by;
+ if (prec->partitioned_by == BIT(PARTITION_PEEL) && loop->this_pass.peeled_cnt > 1 &&
+ loop->disjoint_apart == loop->kept.disjoint_apart)
+ loop->full_peel_cost = div_u64(loop->this_pass.cost * 9 / 8,
+ loop->this_pass.peeled_cnt - 1) *
+ min(loop->exit_trip, U32_MAX);
+ loop->changed = true;
+}
+
+/*
+ * The first unproven insn of the last pass has no error now, and no old error came back.
+ * A pass that is over its budget possibly did not get to that insn.
+ */
+static bool partition_helped(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ return prec->first_unproven_insn < env->prog->len &&
+ !test_bit(prec->first_unproven_insn, prec->this_pass.unproven_insns) &&
+ (!test_bit(env->err_insn, prec->earlier_unproven_insns) ||
+ (test_bit(env->err_insn, prec->last_unproven_insns) &&
+ !prec->this_pass.over_budget));
+}
+
+/* The pass is over its budget. Returns true if another partition replaces the last one. */
+bool bpf_undo_precision(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ bool over_budget = false, again = false;
+ struct bpf_loop *loop;
+ u32 i;
+
+ prec->this_pass.over_budget = env->over_limit;
+ prec->this_pass.calls_joined = env->calls_joined;
+ if (!env->over_limit || env->cost > env->max_cost)
+ return false;
+ if (prec->partitioned_by <= BIT(PARTITION_ADD_DEPS) ||
+ (env->err && (prec->last_pass.over_budget || partition_helped(env)))) {
+ env->over_limit = !env->err;
+ return false;
+ }
+ env->over_limit = false;
+ memcpy(prec->data_deps, prec->undo, 5 * prec->words * sizeof(long));
+ for (i = 0; i < prec->loop_cnt; i++)
+ over_budget |= prec->loops[i]->partitioned &&
+ !bpf_loop_has_budget(prec->loops[i]);
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ if (loop->partitioned)
+ undo_loop(prec, loop);
+ /* if the old budget was enough, the pass possibly did not get to the loop */
+ if (loop->partitioned && over_budget && bpf_loop_has_budget(loop))
+ loop->tried &= ~prec->partitioned_by;
+ /* peeled_apart did not help. The partition can fit without it. */
+ if (loop->peeled_apart && !loop->kept.peeled_apart) {
+ loop->peeled_apart = false;
+ loop->tried |= BIT(PARTITION_PEELED_APART) |
+ BIT(PARTITION_TOO_COSTLY);
+ loop->changed = again = true;
+ }
+ }
+ if (partition_by[__ffs(prec->partitioned_by)] != partition_suspect_loops && !again)
+ prec->too_costly |= prec->partitioned_by;
+ return partition_next(env) > 0;
+}
+
+/*
+ * Keep more apart in the next pass. Take the first line that adds something:
+ *
+ * ADD_DEPS the deps of the unproven insns; for an unreadable xreg
+ * type_apart first
+ * PEEL suspect loops: one trip, then as many as the budget pays for
+ * PEELED_APART suspect loops: the join rule holds at the start of a peeled trip
+ * TRANSITIVE_DEPS data_deps |= control_deps; add the jumps that can skip their writes
+ * DISJOINT_APART disjoint values in data_deps stay apart
+ * LOOP_DISJOINT_APART suspect loops: the same, also at the header of the leftover;
+ * it replaces peeling
+ * PEEL_MORE suspect loops: 4, 16, 64, ... trips for each state that enters
+ * IN_FRAMES calls with an argument that is in the deps and has an equation
+ *
+ * All but ADD_DEPS can be undone:
+ *
+ * +-----------------+ in the budget, or it helped
+ * +--->| partition, pass |------------------------------------> keep
+ * | +--------+--------+
+ * | | over its part of the budget
+ * | v
+ * +--- undo, take the next line
+ * | | no line left, one was too costly
+ * | v
+ * +--- budget *= 4, undo all since one helped, see keep_precision()
+ *
+ * Returns 1 if more is kept apart.
+ */
+int bpf_partition_more(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ bool helped, undone = false;
+ struct bpf_loop *loop;
+ int ret;
+ u32 i;
+
+ ret = prec->data_deps ? 0 : alloc_masks(env);
+ if (ret)
+ return ret;
+ if (!test_bit(env->err_insn, prec->this_pass.unproven_insns))
+ env->err_insn = find_first_bit(prec->this_pass.unproven_insns, env->prog->len);
+ helped = partition_helped(env);
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ if (loop->partitioned && !helped &&
+ (!bpf_loop_has_budget(loop) || loop->this_pass.starves)) {
+ undo_loop(prec, loop);
+ undone = true;
+ continue;
+ }
+ /* the next PARTITION_PEEL_MORE sets no_peel_prune */
+ if (loop->partitioned && loop->this_pass.peeled_cnt < loop->max_peel_cnt &&
+ !(prec->partitioned_by == BIT(PARTITION_PEEL_MORE) &&
+ loop->this_pass.peel_pruned))
+ loop->tried |= prec->partitioned_by;
+ if (loop->max_peel_cnt && bpf_loop_has_budget(loop)) {
+ loop->peel_cnt = max(loop->peel_cnt, loop->this_pass.peeled_cnt);
+ loop->max_peel_cnt = 0;
+ if (!loop->peeled_apart && !loop->disjoint_apart)
+ loop->budget = 0;
+ }
+ loop->last_pass = loop->this_pass;
+ }
+ /* what this pass found is not valid after the undo */
+ if (undone)
+ return partition_next(env);
+ if (!prec->this_pass.over_budget)
+ prec->pass_cost = max(prec->pass_cost, env->cost - prec->start_cost);
+ if (prec->partitioned_by <= BIT(PARTITION_ADD_DEPS))
+ prec->budget = max(prec->budget, bpf_precision_first_budget(env));
+ swap(prec->this_pass, prec->last_pass);
+ bitmap_copy(prec->last_unproven_insns, prec->last_pass.unproven_insns, env->prog->len);
+ if (helped)
+ keep_precision(prec, true);
+ prec->failing_path_due |= helped;
+ prec->first_unproven_insn = env->err_insn;
+ return find_calls_to_unproven(env) ?: partition_next(env);
+}
+
+void bpf_precision_new_pass(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ bitmap_zero(prec->this_pass.unproven_insns, env->prog->len);
+ bitmap_zero(prec->this_pass.bad_read_insns, env->prog->len);
+ bitmap_zero(prec->this_pass.call_insns, env->prog->len);
+ bitmap_zero(prec->this_pass.unproven_funcs, env->subprog_cnt);
+ prec->this_pass.suspect_loops = prec->this_pass.first_suspect_loops =
+ prec->this_pass.disjoint_joined = prec->this_pass.arg_lost_equation = 0;
+ if (prec->apart_xregno)
+ memset(prec->apart_xregno, 0, env->prog->len * sizeof(u16));
+ if (prec->data_deps && !env->replay && (prec->budget_grew || env->for_log)) {
+ keep_precision(prec, false);
+ prec->partitioned_by = prec->pass_cost = 0;
+ }
+ prec->budget_grew = prec->new_nospec = false;
+ prec->start_cost = env->cost;
+ prec->last_unproven_left = bitmap_weight(prec->last_unproven_insns, env->prog->len);
+ /* ADD_DEPS has no alternative, so it has no limit of its own */
+ if (prec->partitioned_by > BIT(PARTITION_ADD_DEPS) && !env->replay)
+ env->pass_max_cost = bpf_pass_max_cost(env);
+}
+
+static int add_loop(struct bpf_precision *prec, struct bpf_loop *loop)
+{
+ struct bpf_loop **loops = prec->loops;
+
+ if (!loop || loop->suspect_bit)
+ return 0;
+ if (is_power_of_2(prec->loop_cnt + 1)) {
+ loops = kvrealloc(loops, 2 * (prec->loop_cnt + 1) * sizeof(*loops),
+ BPF_VERIFIER_GFP);
+ if (!loops)
+ return -ENOMEM;
+ prec->loops = loops;
+ }
+ loop->suspect_bit = BIT_ULL(prec->loop_cnt % 64);
+ loops[prec->loop_cnt++] = loop;
+ return 0;
+}
+
+int bpf_precision_init(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ u32 words = BITS_TO_LONGS(env->prog->len), i;
+ struct bpf_precision *prec;
+ unsigned long *bits;
+ int err = 0;
+
+ prec = env->precision = kvzalloc(sizeof(*prec) + 20 * words * sizeof(long),
+ BPF_VERIFIER_GFP);
+ if (!prec)
+ return -ENOMEM;
+ bits = (void *)(prec + 1);
+ prec->this_pass.unproven_insns = bits;
+ prec->this_pass.call_insns = bits + words;
+ prec->this_pass.bad_read_insns = bits + 2 * words;
+ prec->last_pass.unproven_insns = bits + 3 * words;
+ prec->last_pass.call_insns = bits + 4 * words;
+ prec->last_pass.bad_read_insns = bits + 5 * words;
+ prec->held_insns = bits + 6 * words;
+ prec->err_insns = bits + 7 * words;
+ prec->earlier_unproven_insns = bits + 8 * words;
+ prec->control_seeds = bits + 9 * words;
+ prec->controlled_insns = bits + 10 * words;
+ prec->bad_read_insns_done = bits + 11 * words;
+ prec->controlled_insns_done = bits + 12 * words;
+ prec->def_insns = bits + 13 * words;
+ prec->last_unproven_insns = bits + 14 * words;
+ prec->bad_value_insns = bits + 15 * words;
+ prec->this_pass.unproven_funcs = bits + 16 * words;
+ prec->last_pass.unproven_funcs = bits + 17 * words;
+ prec->pass_cost = env->cost;
+ prec->budget_left = prec->budget = bpf_precision_first_budget(env);
+ prec->first_unproven_insn = prec->loop_carried_at = U32_MAX;
+ for (i = 0; i < env->prog->len && !err; i++, aux++)
+ err = add_loop(prec, aux->loop) ?: add_loop(prec, aux->callback_loop) ?:
+ add_loop(prec, aux->recursion_loop);
+ return err;
+}
+
+void bpf_precision_free(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ if (!prec)
+ return;
+ kvfree(prec->func);
+ kvfree(prec->loops);
+ kvfree(prec->hops_to_err);
+ kvfree(prec->seen);
+ kvfree(prec->data_deps);
+ kvfree(prec);
+ env->precision = NULL;
+}
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index e057e452ec1..7b27c27c8d9 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -22,6 +22,8 @@ struct precision_per_pass {
unsigned long *unproven_funcs; /* of functions */
u64 suspect_loops, first_suspect_loops; /* of the unproven states, of the first one */
bool disjoint_joined; /* disjoint values in data_deps were joined */
+ bool over_budget, calls_joined;
+ bool arg_lost_equation; /* see bpf_call_needs_frames() */
};
/* callx calls one of the functions whose address is taken. They count as one function. */
@@ -51,8 +53,18 @@ struct bpf_precision {
unsigned long *bad_value_insns; /* unproven, its value is possibly the cause */
unsigned long *last_unproven_insns; /* in the last pass */
unsigned long *bad_read_insns_done;
+ unsigned long *earlier_unproven_insns; /* in an earlier pass */
unsigned long *control_seeds, *controlled_insns, *controlled_insns_done, *def_insns;
+ u32 first_unproven_insn; /* of the last pass */
u32 last_unproven_left; /* last_unproven_insns whose check did not fail again */
+ struct bpf_loop **loops;
+ u32 loop_cnt;
+ u64 start_cost, pass_cost; /* env->cost before the pass, the cost of a pass */
+ /* the cost that a partition can add to it, and that this one can add */
+ u64 budget, budget_left;
+ u8 partitioned_by, too_costly; /* bits of PARTITION_* */
+ bool failing_path_due; /* see bpf_failing_path_is_due() */
+ bool budget_grew; /* see partition_next() */
bool new_nospec;
u16 *apart_xregno; /* per insn, see bpf_range_of_data_dep() */
@@ -60,6 +72,7 @@ struct bpf_precision {
/* The path-sensitive run */
unsigned long *err_insns; /* insns at which the failing path can end */
u32 *hops_to_err, *hops_to_exit; /* per insn, see compute_hops() */
+ u64 *seen; /* hash set, see bpf_failing_path_was_here() */
unsigned long *loop_control_deps; /* see bpf_jumps_see_same() */
u32 loop_carried_at;
unsigned long loop_carried[XREG_WORDS];
@@ -72,6 +85,13 @@ struct state_cmp {
u32 disjoint_apart; /* what differs in these xregs */
};
+enum {
+ PARTITION_ADD_DEPS, PARTITION_PEEL, PARTITION_PEELED_APART, PARTITION_TRANSITIVE_DEPS,
+ PARTITION_DISJOINT_APART, PARTITION_LOOP_DISJOINT_APART, PARTITION_PEEL_MORE,
+ PARTITION_IN_FRAMES,
+ PARTITION_TOO_COSTLY, /* in bpf_loop.tried */
+};
+
u32 bpf_frame_mask(struct bpf_verifier_env *env, unsigned long *masks,
const struct bpf_verifier_state *st, u32 fr, unsigned long *out);
const struct bpf_reg_state *bpf_xreg_value(struct bpf_verifier_env *env,
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4068014fedc..c79537dd671 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -11540,7 +11540,7 @@ static bool func_is_on_its_own(struct bpf_verifier_env *env, struct bpf_verifier
if (arg_needs_caller(env, state, sub, &caller->stack_arg_regs[i]))
return false;
}
- return true;
+ return sub->recursive || !bpf_call_needs_frames(env, state, live);
}
static int enter_recursion(struct bpf_verifier_env *env, struct bpf_func_state *callee);
@@ -21611,6 +21611,36 @@ static bool pass_needs_restart(struct bpf_verifier_env *env)
return (env->restart && bpf_flow_loops_lost_bound(env)) || env->guards_late;
}
+/*
+ * The loops that keep the current state apart pay for the work since the last state, but not
+ * a loop that the state enters. A trip that is peeled on the budget of a loop belongs to that
+ * loop. Such trips can use up the budget of an inner loop.
+ */
+static void charge_loops(struct bpf_verifier_env *env, u64 cost)
+{
+ const struct bpf_verifier_state *st = env->cur_state;
+ const struct loop_stack_entry *e;
+ struct bpf_loop *starved_loop = NULL;
+ int fr, i;
+
+ for (fr = st->curframe; env->pass && fr >= 0; fr--) {
+ for (i = st->frame[fr]->loop_stack_cnt - 1; i >= 0; i--) {
+ e = &st->frame[fr]->loop_stack[i];
+ if ((e->peeled || e->loop->disjoint_apart) &&
+ (e->trip || e->loop->header != st->insn_idx))
+ e->loop->this_pass.cost += cost;
+ if (e->peeled && e->trip >= e->loop->peel_cnt) {
+ if (starved_loop && e->loop->partitioned)
+ starved_loop->this_pass.starved =
+ e->loop->this_pass.starves = true;
+ return;
+ }
+ if (e->loop->partitioned && !bpf_loop_has_budget(e->loop))
+ starved_loop = e->loop;
+ }
+ }
+}
+
static int do_check(struct bpf_verifier_env *env)
{
bool level2 = env->log.level & BPF_LOG_LEVEL2;
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 35/49] bpf: Reproduce an error on one path, with no joins
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (33 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 34/49] bpf: Partition the states after an error Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 36/49] bpf: Do not let a join raise what a program requires Alexei Starovoitov
` (13 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
After a join a pass cannot tell a real error from a false one. If the
error is real, more passes are wasted work, and the log of the joined
state does not show the bug:
R6=scalar(smin=0,smax=max) R7=scalar(smin=-22,smax=0)
find_failing_path() tries to reproduce the error. It checks one path
after another until the check of an insn in err_insns fails. This is a
path-sensitive run. It has the same checks as a pass, and goes depth
first, the shortest way first:
- There are no joins. A call stays in the frames of its caller. Each
trip is peeled.
- A state is dropped if a state with the same values in the deps was at
the insn.
- If only the jump on the iv of a loop depends on the xregs that moved,
the path skips trips, see below.
- If the jumps of a loop see the same values in two trips, the loop is
infinite. The path continues where may_goto leaves it.
- A run that is cut by its budget starts again, fall-through edge first.
If it is cut again, or by its limit on trips, it starts again with the
next limit on trips.
The path on which the check fails is the failing path. If the run made
no guess, see below, the passes that are left get a small budget. So a
wrong program is rejected fast. And the log has the full path. Only log
level 2 has the paths that the run dropped.
A path can have errors that do not end it: in a speculative state, or
in the BTF check of the arguments of a static function. Such an error
gets no "Verification failed" report, see env->no_report. A missing
stack argument ends the path, so report it at the first check.
The paths that continue from a header share a copy of the state there,
see bpf_header_state.
A path has to get through a long loop to an error after it:
for (i = 0; i < 100000; i++) { if (rnd() && n < 8) n++; }
a[n] = 0;
It makes no joins, so it has no equation for the trip index. Two trips
give the step:
trip 3: the jump sees i = 3 step 1
trip 4: the jump sees i = 4 99996 trips left
failing_path_trip() compares the state with the saved state of an
earlier trip. If the other jumps do not depend on the xregs that moved,
bpf_fast_forward_xregs() moves them to the last trips.
The nonlinear_xregs of the loop are not moved: the step of two trips
does not tell where '(i + 1) & 63' is after 99996 trips.
Two paths can have the same ranges, and only one of them has r1 == r6.
The other one can be the path that fails. So the hash of a state has
the xregs that share a sym, see bpf_equations_hash().
Three guesses of the run can make a path that the program cannot take.
If such a path cuts the budget, a valid program can be rejected. So a
guess sets env->run_has_guess, and the budget is not cut.
1. The skip. Two trips show a step also for 'sum += i':
for (i = 0; i < 1000; i++) sum += i;
buf[sum - 499500] valid, this is buf[0]
trip i sum real sum
2 2 1 1 saved
4 4 6 6 since trip 2: i +2, sum +5
skip 998 2491 497503 497 times: (996 left - 1) / 2
exit 1000 4488 499500 buf[-495012] fails
bpf_steps_are_constant() looks at the xregs that move and at the
operands of the jump on the iv, such as 'i < n / i'. A copy has to
have the step of its source.
2. "The loop is infinite". Two trips give more than 2^32 trips also for
this loop of 512 trips:
u64 i = 600; do { i = (i + 1) & 1023; } while (i != 88);
The next runs keep this answer. So it is also a guess if the run made
a guess before it.
3. The target of may_goto. All states have the same one. A state that
only a join makes can hold a lock. Then the target is the code after
the loop, also for a path with no lock.
The run accepts nothing. It is not the verdict either: a path has ranges
too, and it can fail at an insn that a pass proves.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 37 +++++
kernel/bpf/diagnostics.c | 2 +-
kernel/bpf/equation_join.c | 202 +++++++++++++++++++++++++++
kernel/bpf/failing_path.c | 259 +++++++++++++++++++++++++++++++++++
kernel/bpf/flow.c | 60 +++++++-
kernel/bpf/precision.c | 4 +
kernel/bpf/precision.h | 2 +
kernel/bpf/verifier.c | 186 ++++++++++++++++++++++++-
8 files changed, 748 insertions(+), 4 deletions(-)
create mode 100644 kernel/bpf/failing_path.c
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 540608bcd8f..03edbd63664 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -445,6 +445,7 @@ static inline u64 bpf_xreg(u32 frameno, u32 xregno)
*/
struct loop_stack_entry {
struct bpf_loop *loop;
+ struct bpf_header_state *header_state; /* see failing_path_trip() */
/* How many times the state came back to the header. A join has the larger number. */
u32 trip;
/* The state that entered the loop. What comes back to the header is for that state. */
@@ -455,6 +456,9 @@ struct loop_stack_entry {
struct scalar trip_index;
bool trip_widened; /* it was moved to a threshold, or it does not grow */
u64 exit_threshold; /* by the jumps that the state passed and that leave the loop */
+ /* Path-sensitive run: the jump that tests the iv, the value it saw, the trips left */
+ u32 iv_jump_idx;
+ u64 iv_prev, trips_left;
};
/* The states of trip 0, 1, 2, 4, 8, ... are saved. A compare with them finds any period. */
@@ -632,6 +636,12 @@ static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state
return &frame->stack[spi];
}
+/* The state in which a path started a trip, see failing_path_trip() */
+struct bpf_header_state {
+ struct bpf_verifier_state st;
+ u32 refcnt;
+};
+
static inline struct bpf_reg_state *
bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask)
{
@@ -882,7 +892,9 @@ struct bpf_loop {
bool peeled_joined; /* at the start of a peeled trip, against the join rule */
bool changed; /* see flow_forget_loop() */
bool guarded; /* states take the edges of may_goto */
+ bool was_infinite; /* at the start of the last path-sensitive run */
bool infinite; /* see failing_path_trip() */
+ bool infinite_is_guess; /* see fast_forward_trips() */
struct bpf_iv_threshold *iv_thresholds; /* where an iv can stop, see equation_join.c */
u32 iv_threshold_cnt, iv_threshold_cap;
u32 leftover_epoch; /* how many leftovers started */
@@ -1360,6 +1372,9 @@ struct bpf_verifier_env {
bool path_sensitive; /* no joins, see run_path_sensitive() */
bool for_log; /* the verdict is known, see do_check_all() */
bool fallthrough_first; /* see find_failing_path() */
+ bool run_has_guess; /* see run_path_sensitive() */
+ u32 failing_path_budget_shift; /* see func_failing_path_budget() */
+ u32 failing_path_max_trips; /* see failing_path_trip() */
u32 pass; /* number of the pass, see do_check_all() */
struct bpf_precision *precision;
u64 cost, mem; /* see bpf_check_limits() */
@@ -1375,6 +1390,8 @@ struct bpf_verifier_env {
u32 err_insn;
bool imprecise; /* a state before the first error is imprecise */
bool over_limit; /* see bpf_check_limits() */
+ bool too_deep; /* see failing_path_trip() */
+ bool run_over_budget; /* in one of its functions, see find_failing_path() */
bool restart; /* see bpf_flow_next_is_no_bound() */
bool guards_late; /* see bpf_flow_needs_guard() */
bool calls_joined; /* see flow_may_add_instance() */
@@ -1520,6 +1537,7 @@ struct bpf_verifier_env {
int *idoms;
struct bpf_equation_env *eq_env;
struct bpf_flow *flow;
+ bool no_report; /* an error does not end the path, see diag_write() */
};
static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@@ -1723,6 +1741,14 @@ void bpf_precision_new_pass(struct bpf_verifier_env *env);
u64 bpf_pass_max_cost(struct bpf_verifier_env *env);
u64 bpf_precision_budget(struct bpf_verifier_env *env);
u64 bpf_precision_first_budget(struct bpf_verifier_env *env);
+bool bpf_failing_path_is_due(struct bpf_verifier_env *env);
+int bpf_failing_path_prepare(struct bpf_verifier_env *env, bool repeat);
+bool bpf_failing_path_may_end_at(struct bpf_verifier_env *env, u32 idx);
+u32 bpf_failing_path_hops(struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
+int bpf_failing_path_was_here(struct bpf_verifier_env *env, const struct bpf_verifier_state *st);
+int bpf_jumps_see_same(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *new, int iv_jump_idx,
+ const unsigned long *iv_xregs);
int bpf_precision_init(struct bpf_verifier_env *env);
void bpf_precision_free(struct bpf_verifier_env *env);
@@ -1762,6 +1788,8 @@ void bpf_flow_reset(struct bpf_verifier_env *env);
void bpf_flow_free(struct bpf_verifier_env *env);
bool bpf_flow_trip_is_new(struct bpf_verifier_env *env, struct loop_stack_entry *entry);
bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx);
+int bpf_flow_save_header_state(struct bpf_verifier_env *env, struct loop_stack_entry *entry);
+void bpf_flow_put_header_state(struct loop_stack_entry *entry);
int bpf_flow_deliver(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
int insn_idx, int prev_insn_idx, bool again);
int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos, u64 *diag_pos);
@@ -2214,6 +2242,7 @@ struct bpf_call_arg_meta {
/* Only set by subprog */
bool subprog_may_change_pkt;
+ bool subprog_is_static;
/* Only set by kfunc */
bool r0_rdonly;
@@ -2454,6 +2483,7 @@ int bpf_compute_idoms(struct bpf_verifier_env *env);
int bpf_compute_loops(struct bpf_verifier_env *env);
int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx);
int bpf_loop_spine_end(struct bpf_verifier_env *env, const struct bpf_loop *loop);
+int bpf_flow_insn_after(struct bpf_verifier_env *env, int idx);
bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header);
u32 bpf_loop_thresholds_below(const struct bpf_loop *loop, s64 k);
int bpf_loop_collect_thresholds(struct bpf_verifier_env *env, struct bpf_loop *loop);
@@ -2495,7 +2525,14 @@ bool bpf_equations_join_forgets(struct bpf_verifier_env *env, const struct bpf_v
const struct bpf_verifier_state *b, u32 fr,
const unsigned long *data_deps, const unsigned long *xregs,
u32 bits);
+u32 bpf_equations_hash(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u32 fr,
+ const unsigned long *xregs, u32 bits);
void bpf_collect_iv_thresholds(struct bpf_verifier_env *env);
+bool bpf_steps_are_constant(struct bpf_verifier_env *env, const struct loop_stack_entry *entry,
+ const unsigned long *iv_xregs, bool infinite);
+void bpf_fast_forward_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ struct bpf_verifier_state *cur, u64 factor, unsigned long *iv_xregs);
+
void scalar_bounds_sync(struct scalar *s);
bool scalar_subsumes(const struct scalar *old, const struct scalar *cur);
bool scalars_fit_in_bits(const struct scalar *a, const struct scalar *b, u32 bits);
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c
index 2cb9eceb408..0e12078ffe1 100644
--- a/kernel/bpf/diagnostics.c
+++ b/kernel/bpf/diagnostics.c
@@ -372,7 +372,7 @@ static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...)
{
va_list args;
- if (!bpf_diag_enabled(env))
+ if (!bpf_diag_enabled(env) || env->no_report)
return;
va_start(args, fmt);
diff --git a/kernel/bpf/equation_join.c b/kernel/bpf/equation_join.c
index 93ebf337e4c..47db8f8ae2b 100644
--- a/kernel/bpf/equation_join.c
+++ b/kernel/bpf/equation_join.c
@@ -12,6 +12,7 @@
#include <linux/bpf.h>
#include <linux/cnum.h>
#include <linux/filter.h>
+#include <linux/jhash.h>
#include <linux/slab.h>
#include <linux/tnum.h>
#include "equation.h"
@@ -621,6 +622,27 @@ bool bpf_equations_join_forgets(struct bpf_verifier_env *env, const struct bpf_v
return false;
}
+/* Which xregs of @xregs in frame @fr have one sym? */
+u32 bpf_equations_hash(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u32 fr,
+ const unsigned long *xregs, u32 bits)
+{
+ const struct bpf_xreg_equation *xreg_eq;
+ u32 i, hash = 0, *first;
+
+ for (i = 0; i < st->equations.sym_cnt; i++)
+ env->eq_env->scratch[i].new_symno = U32_MAX;
+ for_each_xreg_equation(env, &st->equations, xreg_eq) {
+ if (xreg_eq->xreg >> 32 != fr || (u32)xreg_eq->xreg >= bits ||
+ !test_bit((u32)xreg_eq->xreg, xregs))
+ continue;
+ first = &env->eq_env->scratch[xreg_eq->sym].new_symno;
+ if (*first != U32_MAX)
+ hash = jhash_2words(*first, xreg_eq->xreg, hash);
+ *first = min_t(u32, *first, xreg_eq->xreg);
+ }
+ return hash;
+}
+
/* Thresholds */
/*
@@ -721,6 +743,64 @@ static u64 compute_trip_count(s64 step, u64 *x, u8 op, bool jmp32, struct scalar
return min(div64_u64(step > 0 ? arc.size - left : left, abs64(step)), U64_MAX - 1) + 1;
}
+/* A path-sensitive run has no equations for the trip index. Two trips give the step. */
+static void note_trips_left(struct bpf_verifier_env *env, struct loop_stack_entry *e,
+ const struct scalar *x, u8 op, bool jmp32, const struct scalar *bound,
+ bool is_body_jump)
+{
+ u64 value = jmp32 ? x->r32.base : x->r64.base, far_end;
+ s64 step = jmp32 ? (s32)(value - e->iv_prev) : value - e->iv_prev;
+
+ if (e->iv_jump_idx && e->iv_jump_idx != env->insn_idx)
+ return;
+ far_end = value + (step < 0 ? 0 : jmp32 ? x->r32.size : x->r64.size);
+ if (e->iv_jump_idx)
+ e->trips_left = step ? compute_trip_count(step, &far_end, op, jmp32, *bound) : 0;
+ if (e->iv_jump_idx && step && is_body_jump && !e->trips_left)
+ e->trips_left = compute_trip_count(step, &far_end, bpf_rev_opcode(op),
+ jmp32, *bound);
+ e->iv_jump_idx = env->insn_idx;
+ e->iv_prev = value;
+}
+
+static void move_bounds(struct bpf_verifier_env *env, struct scalar *s, bool is_ptr, u64 off)
+{
+ struct scalar step;
+
+ scalar_set_const(&step, off);
+ scalar_add(s, s, &step);
+ /* see refine_ptr() */
+ if (is_ptr)
+ bpf_blur_far_offset(env, container_of(&s->var_off, struct bpf_reg_state, var_off));
+ else
+ scalar_bounds_sync(s);
+}
+
+/* Some xregs of @cur moved since @old. Note them in @iv_xregs, move them @factor times as far. */
+void bpf_fast_forward_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ struct bpf_verifier_state *cur, u64 factor, unsigned long *iv_xregs)
+{
+ u32 xregno, cnt = bpf_spi_to_xreg(bpf_stack_nr_slots(cur->frame[cur->curframe]));
+ struct scalar *a, *b;
+ u64 xreg;
+ bool is_ptr;
+
+ bpf_add_cost(env, cnt, BPF_COST_COPY);
+ for (xregno = 0; xregno < cnt; xregno++) {
+ xreg = bpf_xreg(cur->curframe, xregno);
+ a = bpf_find_var_bounds(env, old, xreg, &is_ptr);
+ b = bpf_find_var_bounds(env, cur, xreg, &is_ptr);
+ if (!a || !b || a->r64.size != b->r64.size || a->r64.base == b->r64.base ||
+ test_bit(xregno,
+ bpf_top_loop(cur->frame[cur->curframe])->loop->nonlinear_xregs))
+ continue;
+ __set_bit(xregno, iv_xregs);
+ move_bounds(env, b, is_ptr, (b->r64.base - a->r64.base) * factor);
+ if (factor)
+ bpf_equations_forget_xreg(env, cur, xreg);
+ }
+}
+
static bool insn_is_in_loop(struct bpf_verifier_env *env, const struct bpf_loop *loop, int idx)
{
int h = bpf_loop_at_index(env, idx);
@@ -728,6 +808,118 @@ static bool insn_is_in_loop(struct bpf_verifier_env *env, const struct bpf_loop
return h == loop->header || bpf_is_nested_loop(env, h, loop->header);
}
+/* The slot that @insn fills from or spills to. 0 if there is none. */
+static u32 slot_of_insn(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+ u8 class = BPF_CLASS(insn->code), base = class == BPF_LDX ? insn->src_reg : insn->dst_reg;
+
+ if (class != BPF_LD && class <= BPF_STX && base == BPF_REG_FP && insn->off < 0 &&
+ !(insn->off % BPF_REG_SIZE) && -insn->off <= env->stack_limit &&
+ BPF_MODE(insn->code) != BPF_ATOMIC && bpf_size_to_bytes(BPF_SIZE(insn->code)) >= 4)
+ return bpf_spi_to_xreg(-insn->off / BPF_REG_SIZE - 1);
+ return 0;
+}
+
+/* If the operand has a constant step, the result has one too. @prev: in the same block. */
+static bool keeps_constant_step(const struct bpf_insn *insn, const struct bpf_insn *prev,
+ bool infinite)
+{
+ u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
+
+ if (bpf_is_ldimm64(insn))
+ return true;
+ if (class != BPF_ALU64 && (class != BPF_ALU || infinite))
+ return false;
+ if (op == BPF_MOV)
+ return !insn->off || (!infinite && insn->off == 32);
+ if (BPF_SRC(insn->code) == BPF_X)
+ return false;
+ if (op == BPF_ADD || op == BPF_SUB)
+ return true;
+ /* x <<= 32; x >>= 32; */
+ return !infinite && insn->imm == 32 &&
+ (op == BPF_LSH || ((op == BPF_RSH || op == BPF_ARSH) && prev && prev->imm == 32 &&
+ prev->code == (BPF_ALU64 | BPF_LSH | BPF_K) &&
+ prev->dst_reg == insn->dst_reg));
+}
+
+/* How far @xregno moved since the saved trip */
+static u64 step_of_xreg(struct bpf_verifier_env *env, const struct loop_stack_entry *entry,
+ const unsigned long *iv_xregs, u32 xregno)
+{
+ u64 xreg = bpf_xreg(env->cur_state->curframe, xregno);
+ bool is_ptr;
+
+ if (!test_bit(xregno, iv_xregs))
+ return 0;
+ return bpf_find_var_bounds(env, env->cur_state, xreg, &is_ptr)->r64.base -
+ bpf_find_var_bounds(env, &entry->header_state->st, xreg, &is_ptr)->r64.base;
+}
+
+/*
+ * Two trips gave the steps. Does each trip add the same constant to the operands of the jump on
+ * the iv, and to @iv_xregs? The code shows it, or a pass saw it at the header.
+ * i += 1; x = (u32)i; yes
+ * sum += i; x = n / i; no
+ * a = b; b = 0; no, if a moved and b did not
+ * @infinite: above 2^32 trips. Only the jump counts, and 32-bit insns wrap.
+ */
+bool bpf_steps_are_constant(struct bpf_verifier_env *env, const struct loop_stack_entry *entry,
+ const unsigned long *iv_xregs, bool infinite)
+{
+ const struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ const struct bpf_insn *insn, *prev = NULL;
+ const struct bpf_loop *loop = entry->loop;
+ DECLARE_BITMAP(not_linear, MAX_BPF_XREG);
+ u16 no_step = 0; /* an insn of the loop writes it with no constant step */
+ u16 new_step = 0; /* the last write in this basic block has a constant step */
+ u16 last_round, uses_no_step;
+ u64 steps[MAX_BPF_EXT_REG];
+ bool has_step;
+ u32 slot;
+ int n;
+
+ bitmap_andnot(not_linear, iv_xregs, loop->linear_xregs, MAX_BPF_XREG);
+ do {
+ last_round = no_step;
+ bpf_add_cost(env, loop->nr_insns, BPF_COST_SCAN);
+ for (n = 0; n < MAX_BPF_EXT_REG; n++)
+ steps[n] = step_of_xreg(env, entry, iv_xregs, n);
+ for (n = loop->header; n >= 0 && insn_is_in_loop(env, loop, n);
+ prev = insn, n = bpf_flow_insn_after(env, n)) {
+ insn = &env->prog->insnsi[n];
+ if (bpf_flow_is_bb_start(env, n)) {
+ new_step = 0;
+ prev = NULL;
+ }
+ uses_no_step = aux[n].use_regs & no_step & ~new_step;
+ if (n == entry->iv_jump_idx && (uses_no_step & ~aux[n].const_reg_mask))
+ return false;
+ slot = slot_of_insn(env, insn);
+ has_step = !slot ? keeps_constant_step(insn, prev, infinite) :
+ test_bit(bpf_xreg_to_spi(slot),
+ aux[loop->header].live_stack_before) &&
+ !test_bit(slot, not_linear) &&
+ (!infinite || BPF_SIZE(insn->code) == BPF_DW);
+ /* a copy that the run made has the step of its source */
+ if (has_step && aux[n].seen && (aux[n].def_regs & ~aux[n].use_regs))
+ steps[insn->dst_reg] =
+ slot ? step_of_xreg(env, entry, iv_xregs, slot) :
+ aux[n].use_regs ? steps[insn->src_reg] : 0;
+ new_step |= aux[n].def_regs;
+ if (has_step && !uses_no_step)
+ continue;
+ new_step &= ~aux[n].def_regs;
+ no_step |= aux[n].def_regs;
+ }
+ for_each_set_bit(n, iv_xregs, MAX_BPF_EXT_REG)
+ if (steps[n] != step_of_xreg(env, entry, iv_xregs, n))
+ no_step |= BIT(n);
+ } while (no_step != last_round);
+ not_linear[0] &= no_step | ~GENMASK(BPF_REG_FP, BPF_REG_0);
+ return infinite || bitmap_empty(not_linear, MAX_BPF_XREG);
+}
+
/* The jump stays in the loop if operand 0 or 1, or their difference, is "op" the bound */
static u8 loop_cond_opcode(const struct bpf_insn *insn, bool taken, int i)
{
@@ -756,6 +948,7 @@ void bpf_collect_iv_thresholds(struct bpf_verifier_env *env)
const struct bpf_func_state *f = st->frame[st->curframe];
bool is_ptr, is_iv[3], low32[3], start_is_range[3] = {}, stays_in_loop[2], is_body_jump;
struct loop_stack_entry *e = bpf_top_loop(f);
+ const struct bpf_iv_threshold *threshold;
struct expr trip, step, expr[3];
struct scalar bound[3];
struct equation_pos pos = {};
@@ -781,6 +974,15 @@ void bpf_collect_iv_thresholds(struct bpf_verifier_env *env)
stays_in_loop[1] = insn_is_in_loop(env, e->loop, env->insn_idx + insn->off + 1);
is_body_jump = stays_in_loop[0] && stays_in_loop[1];
low32[0] = low32[1] = low32[2] = BPF_CLASS(insn->code) == BPF_JMP32;
+ if (env->path_sensitive) {
+ if (is_body_jump)
+ for_each_iv_threshold(env, e->loop, threshold, THRESHOLD_EXIT_JUMP)
+ return;
+ i = aux->iv_operand - 1;
+ return note_trips_left(env, e, &bound[!i],
+ loop_cond_opcode(insn, stays_in_loop[1], i), low32[i],
+ &bound[i], is_body_jump);
+ }
x = bpf_xreg(st->curframe, BPF_XREG_TRIP + f->loop_stack_cnt - 1);
if (!bpf_xreg_expr(env, st, x, &trip, &is_ptr) || !var_is_sym(trip.var))
return;
diff --git a/kernel/bpf/failing_path.c b/kernel/bpf/failing_path.c
new file mode 100644
index 00000000000..6f3be5d900c
--- /dev/null
+++ b/kernel/bpf/failing_path.c
@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Help for the path-sensitive run, see run_path_sensitive(). If the error of a pass is real,
+ * the next passes get a small budget and the log gets a path to the error. The run reproduces
+ * the error with no joins. The path on which the check fails again is the failing path.
+ *
+ * entry --> ... --> state --+--> 2 hops --> unproven insn goes first
+ * +--> 9 hops --> unproven insn
+ * +--> no path is dropped
+ */
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/filter.h>
+#include <linux/slab.h>
+
+#include "precision.h"
+
+#define NO_PATH (U32_MAX / 2)
+
+static bool shorter(u32 *hops, u32 via)
+{
+ if (via + 1 >= *hops)
+ return false;
+ *hops = via + 1;
+ return true;
+}
+
+/* @to follows @from in one function */
+static bool shorter_via(struct bpf_precision *prec, u32 from, u32 to)
+{
+ return shorter(&prec->hops_to_err[from], prec->hops_to_err[to]) |
+ shorter(&prec->hops_to_exit[from], prec->hops_to_exit[to]);
+}
+
+/*
+ * For each insn, the number of edges to
+ * - an unproven insn, in its function or in a function that it calls,
+ * - the exit of its function.
+ * The first unproven insn is nearer than the other ones.
+ */
+static int compute_hops(struct bpf_verifier_env *env)
+{
+ struct bpf_subprog_info *sub = env->subprog_info;
+ u32 len = env->prog->len, taken = NO_PATH, i, k, n, s;
+ struct bpf_precision *prec = env->precision;
+ struct bpf_insn_aux_data *aux;
+ struct bpf_iarray *succ;
+ bool changed = true;
+ int t, err = 0;
+
+ for (i = 0; i < len; i++) {
+ prec->hops_to_err[i] = !test_bit(i, prec->err_insns) ? NO_PATH :
+ i == prec->first_unproven_insn ? 0 : len;
+ prec->hops_to_exit[i] = env->prog->insnsi[i].code == (BPF_JMP | BPF_EXIT) ?
+ 0 : NO_PATH;
+ }
+ while (changed && !err) {
+ changed = false;
+ /* callees first */
+ for (k = 0; k < env->subprog_cnt; k++) {
+ s = env->subprog_topo_order[k];
+ for (i = sub[s].postorder_start; i < sub[s + 1].postorder_start; i++) {
+ n = env->cfg.insn_postorder[i];
+ aux = &env->insn_aux_data[n];
+ succ = bpf_insn_successors(env, n);
+ iarray_for_each(t, succ)
+ changed |= shorter_via(prec, n, t);
+ /* see check_loop_guard() and check_recursion_guard() */
+ if (aux->guard_place)
+ changed |= shorter_via(prec, n, aux->guard_to);
+ if ((aux->guard_place || (sub[s].cycle && n == sub[s].start)) &&
+ sub[s].bail_insn >= 0)
+ changed |= shorter_via(prec, n, sub[s].bail_insn);
+ if (!bpf_calls_function(env, n))
+ continue;
+ t = bpf_insn_callee(env, n);
+ changed |= shorter(&prec->hops_to_err[n],
+ t > 0 ? prec->hops_to_err[sub[t].start] : taken);
+ }
+ if (sub[s].addr_taken)
+ changed |= shorter(&taken, prec->hops_to_err[sub[s].start]);
+ }
+ err = bpf_add_cost_and_check(env, len, BPF_COST_XREG);
+ }
+ return err;
+}
+
+bool bpf_failing_path_may_end_at(struct bpf_verifier_env *env, u32 idx)
+{
+ return idx < env->prog->len && test_bit(idx, env->precision->err_insns);
+}
+
+/* The number of edges from @st to an unproven insn. U32_MAX if there is no path. */
+u32 bpf_failing_path_hops(struct bpf_verifier_env *env, const struct bpf_verifier_state *st)
+{
+ const struct bpf_precision *prec = env->precision;
+ u32 idx = st->insn_idx, fr = st->curframe;
+ u64 hops = prec->hops_to_err[idx], out = 0;
+
+ for (; fr; fr--) {
+ out += prec->hops_to_exit[idx];
+ idx = st->frame[fr]->callsite + !st->frame[fr]->in_callback_fn;
+ hops = min(hops, out + prec->hops_to_err[idx]);
+ }
+ return hops < NO_PATH ? hops : U32_MAX;
+}
+
+static u64 hash_value(const struct bpf_reg_state *reg, u64 hash)
+{
+ return bpf_hash64(reg, offsetof(struct bpf_reg_state, grew_once), hash);
+}
+
+/* The join_key, the values of the deps, and which of them have one sym */
+static u64 hash_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *st)
+{
+ unsigned long all[XREG_WORDS], control_deps[XREG_WORDS];
+ u64 hash = bpf_hash_join_key(env, st);
+ const struct bpf_reg_state *reg;
+ const struct bpf_func_state *f;
+ struct bpf_reg_state tmp;
+ u32 fr, xregno, i, bits;
+
+ for (fr = 0; fr <= st->curframe; fr++) {
+ f = st->frame[fr];
+ bpf_frame_mask(env, env->precision->control_deps, st, fr, control_deps);
+ bits = bpf_frame_mask(env, env->precision->type_apart, st, fr, all);
+ bitmap_or(all, all, control_deps, bits);
+ bpf_add_cost(env, 2 * bitmap_weight(all, bits), BPF_COST_XREG);
+ hash += bpf_equations_hash(env, st, fr, all, bits);
+ for_each_set_bit(xregno, all, bits) {
+ for (i = 0; xregno == XREG_STACK_ARGS && i < f->out_stack_arg_cnt; i++)
+ hash = hash_value(&f->stack_arg_regs[i], hash);
+ reg = bpf_xreg_value(env, f, xregno, &tmp);
+ if (reg)
+ hash = hash_value(reg, hash + xregno);
+ i = bpf_xreg_to_spi(xregno);
+ if (i < bpf_stack_nr_slots(f))
+ hash = bpf_hash64(bpf_stack_slot(f, i)->slot_type,
+ BPF_REG_SIZE, hash);
+ }
+ }
+ return hash;
+}
+
+static bool add_to_set(u64 *set, u32 bits, u64 hash)
+{
+ u32 i;
+
+ for (i = hash & (BIT(bits) - 1); set[i]; i = (i + 1) & (BIT(bits) - 1))
+ if (set[i] == hash)
+ return false;
+ set[i] = hash;
+ return true;
+}
+
+/* Returns 1 if a path was at this insn with the same hash_state(). A collision drops a path. */
+int bpf_failing_path_was_here(struct bpf_verifier_env *env, const struct bpf_verifier_state *st)
+{
+ struct bpf_precision *prec = env->precision;
+ u32 size = BIT(prec->seen_bits), i;
+ u64 *set;
+
+ if (2 * prec->seen_cnt + 2 > size) {
+ set = kvcalloc(2 * size, sizeof(*set), BPF_VERIFIER_GFP);
+ if (!set)
+ return -ENOMEM;
+ for (i = 0; prec->seen && i < size; i++)
+ if (prec->seen[i])
+ add_to_set(set, prec->seen_bits + 1, prec->seen[i]);
+ env->mem += size * sizeof(*set);
+ kvfree(prec->seen);
+ prec->seen = set;
+ prec->seen_bits++;
+ }
+ if (!add_to_set(prec->seen, prec->seen_bits, hash_state(env, st) | 1))
+ return 1;
+ prec->seen_cnt++;
+ return 0;
+}
+
+/* The insn at @idx selects what runs next, and a path can get to it more than once */
+static bool selects_path_again(struct bpf_verifier_env *env, int idx)
+{
+ struct bpf_insn *insn = &env->prog->insnsi[idx];
+
+ if (env->insn_aux_data[idx].calls_callback)
+ return true;
+ if (idx < env->subprog_info[1].start && bpf_loop_at_index(env, idx) < 0)
+ return false;
+ return bpf_insn_is_cond_jump(insn->code) || insn_is_gotox(insn) ||
+ (bpf_calls_function(env, idx) && !bpf_pseudo_call(insn));
+}
+
+/*
+ * A path came around a loop from @old to @new. Do the jumps that it can get to again see the
+ * same values? Then it can take the same edges again.
+ * @iv_jump_idx: ignore the jump that tests the iv. -1 if there is none.
+ * @iv_xregs: each trip adds a constant to these xregs. Are all other loop-carried xregs the
+ * same too, and not computed from these?
+ */
+int bpf_jumps_see_same(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
+ const struct bpf_verifier_state *new, int iv_jump_idx,
+ const unsigned long *iv_xregs)
+{
+ const struct bpf_insn_aux_data *aux = &env->insn_aux_data[old->insn_idx];
+ struct bpf_precision *prec = env->precision;
+ unsigned long *masks = iv_jump_idx < 0 ? prec->loop_control_deps : prec->new_data_deps;
+ u32 *made_for = iv_jump_idx < 0 ? &prec->loop_control_deps_for : &prec->new_data_deps_for;
+ DECLARE_BITMAP(loop_carried, MAX_BPF_XREG) = {};
+ u32 key = 2 * iv_jump_idx + !iv_xregs, i;
+ struct state_cmp cmp;
+ int err;
+
+ if (iv_xregs) {
+ loop_carried[0] = aux->live_regs_before & XREG_REGS;
+ bpf_add_slots(loop_carried, aux->live_stack_before, MAX_BPF_XREG);
+ bitmap_andnot(loop_carried, loop_carried, iv_xregs, MAX_BPF_XREG);
+ }
+ if (*made_for == key) {
+ bpf_compare_states(env, old, new, masks, NULL, &cmp);
+ /* masks that were made for other @iv_xregs can miss an xreg */
+ if (cmp.all || bitmap_subset(loop_carried, prec->loop_carried, MAX_BPF_XREG))
+ return !cmp.all;
+ }
+ for (i = 0; i < env->prog->len; i++)
+ __assign_bit(i, prec->control_seeds,
+ i != iv_jump_idx && selects_path_again(env, i));
+ if (iv_xregs) {
+ bitmap_copy(prec->loop_carried, loop_carried, MAX_BPF_XREG);
+ prec->loop_carried_at = old->insn_idx;
+ }
+ err = bpf_compute_deps(env, masks, prec->control_seeds);
+ prec->loop_carried_at = U32_MAX;
+ if (err)
+ return err;
+ *made_for = key;
+ bpf_compare_states(env, old, new, masks, NULL, &cmp);
+ return !cmp.all;
+}
+
+/* The masks or the budget are new */
+bool bpf_failing_path_is_due(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+
+ return prec->failing_path_due || prec->partitioned_by == BIT(PARTITION_ADD_DEPS);
+}
+
+/* @repeat: start like the last run */
+int bpf_failing_path_prepare(struct bpf_verifier_env *env, bool repeat)
+{
+ struct bpf_precision *prec = env->precision;
+
+ prec->failing_path_due = false;
+ if (prec->seen)
+ memset(prec->seen, 0, sizeof(*prec->seen) << prec->seen_bits);
+ prec->seen_cnt = prec->new_data_deps_for = 0;
+ return repeat ? 0 : compute_hops(env);
+}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index 3087923ba79..a1ac4709b68 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -101,12 +101,14 @@ struct flow_instance {
struct flow_kept {
struct list_head node;
- struct bpf_verifier_state st;
+ struct bpf_flow *flow;
+ struct bpf_header_state header_state;
};
struct bpf_flow {
/* computed once */
u32 *pos; /* position of the insn in the order */
+ s32 *insn_after; /* the next insn in the order, or -1 */
u32 *rank; /* callers go first */
unsigned long *bb_starts; /* insn starts a basic block */
unsigned long *join_points; /* more than one edge leads to the insn */
@@ -127,6 +129,7 @@ struct bpf_flow {
u32 instance_cnt, instance_cap;
struct flow_instance *last_instance; /* of the last pending state */
u64 priority_gen;
+ struct list_head unused; /* struct flow_kept */
u32 seq;
u32 live_states;
};
@@ -145,7 +148,7 @@ struct bpf_flow {
static int flow_compute_order(struct bpf_verifier_env *env, struct bpf_flow *flow)
{
struct bpf_insn_aux_data *aux = env->insn_aux_data;
- int len = env->prog->len, i, s, n, h, top, depth;
+ int len = env->prog->len, i, s, n, h, top, depth, prev = -1;
int *next, *head, *tail, *stack;
unsigned long *linked;
u32 pos = 0;
@@ -195,6 +198,10 @@ static int flow_compute_order(struct bpf_verifier_env *env, struct bpf_flow *flo
continue;
}
flow->pos[n] = pos++;
+ flow->insn_after[n] = -1;
+ if (prev >= 0)
+ flow->insn_after[prev] = n;
+ prev = n;
if (aux[n].loop && head[n] >= 0) {
stack[depth++] = n;
n = head[n];
@@ -213,6 +220,11 @@ static int flow_compute_order(struct bpf_verifier_env *env, struct bpf_flow *flo
return err;
}
+int bpf_flow_insn_after(struct bpf_verifier_env *env, int idx)
+{
+ return env->flow->insn_after[idx];
+}
+
/* Does the insn at @idx stop the loop @h? */
static bool flow_is_bound(struct bpf_verifier_env *env, int h, int idx)
{
@@ -541,6 +553,7 @@ int bpf_flow_prepare(struct bpf_verifier_env *env)
if (!flow)
return -ENOMEM;
env->flow = flow;
+ INIT_LIST_HEAD(&flow->unused);
flow->state_tree = flow->pending = RB_ROOT_CACHED;
flow->call_loop.kind = BPF_LOOP_RECURSION;
@@ -558,6 +571,7 @@ int bpf_flow_prepare(struct bpf_verifier_env *env)
return -ENOMEM;
}
+ flow->insn_after = flow->pos + len;
/* subprog_topo_order[] has callees first */
for (i = 0; i < env->subprog_cnt; i++)
flow->rank[env->subprog_topo_order[i]] = env->subprog_cnt - i;
@@ -677,6 +691,17 @@ static void flow_free_list(struct bpf_verifier_env *env, struct list_head *list)
flow_free_state(env, s);
}
+static void flow_free_unused(struct bpf_verifier_env *env)
+{
+ struct flow_kept *k;
+
+ while ((k = list_first_entry_or_null(&env->flow->unused, struct flow_kept, node))) {
+ list_del(&k->node);
+ bpf_free_verifier_state(env, &k->header_state.st, false);
+ kfree(k);
+ }
+}
+
/* What a check leaves behind. */
void bpf_flow_reset(struct bpf_verifier_env *env)
{
@@ -694,6 +719,7 @@ void bpf_flow_reset(struct bpf_verifier_env *env)
flow->instances[i]->group->instances = NULL;
kfree(flow->instances[i]);
}
+ flow_free_unused(env);
env->mem -= flow->instance_cnt * INSTANCE_BYTES;
flow->instance_cnt = 0;
flow->last_instance = NULL;
@@ -723,6 +749,35 @@ bool bpf_flow_is_bb_start(struct bpf_verifier_env *env, int insn_idx)
return test_bit(insn_idx, env->flow->bb_starts);
}
+/* The paths that continue from the current state share a copy of it */
+int bpf_flow_save_header_state(struct bpf_verifier_env *env, struct loop_stack_entry *entry)
+{
+ struct flow_kept *k = kzalloc_obj(*k, BPF_VERIFIER_GFP);
+
+ if (!k)
+ return -ENOMEM;
+ bpf_flow_put_header_state(entry);
+ entry->header_state = NULL;
+ k->flow = env->flow;
+ k->header_state.refcnt = 1;
+ if (bpf_copy_verifier_state(env, &k->header_state.st, env->cur_state)) {
+ list_add(&k->node, &k->flow->unused);
+ return -ENOMEM;
+ }
+ entry->header_state = &k->header_state;
+ return 0;
+}
+
+void bpf_flow_put_header_state(struct loop_stack_entry *entry)
+{
+ struct flow_kept *k;
+
+ if (!entry->header_state || --entry->header_state->refcnt)
+ return;
+ k = container_of(entry->header_state, struct flow_kept, header_state);
+ list_add(&k->node, &k->flow->unused);
+}
+
/* A peeled trip is complete before the next one starts. The leftover is last. */
static int flow_cmp_trips(const struct bpf_func_state *fa, const struct bpf_func_state *fb)
{
@@ -1693,6 +1748,7 @@ int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos
flow_free_state(env, t);
}
+ flow_free_unused(env);
env->insn_idx = s->st.insn_idx;
if (s->kind == FLOW_FUNC_EXIT) {
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index 42d9fd0ee72..59e425431c6 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -1486,6 +1486,10 @@ int bpf_partition_more(struct bpf_verifier_env *env)
if (prec->partitioned_by <= BIT(PARTITION_ADD_DEPS))
prec->budget = max(prec->budget, bpf_precision_first_budget(env));
swap(prec->this_pass, prec->last_pass);
+ /* only the unproven insns of this pass: no check failed at the other ones */
+ bitmap_or(prec->last_unproven_insns, prec->last_pass.unproven_insns,
+ prec->last_pass.bad_read_insns, env->prog->len);
+ bitmap_and(prec->err_insns, prec->err_insns, prec->last_unproven_insns, env->prog->len);
bitmap_copy(prec->last_unproven_insns, prec->last_pass.unproven_insns, env->prog->len);
if (helped)
keep_precision(prec, true);
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index 7b27c27c8d9..39284d1df97 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -73,7 +73,9 @@ struct bpf_precision {
unsigned long *err_insns; /* insns at which the failing path can end */
u32 *hops_to_err, *hops_to_exit; /* per insn, see compute_hops() */
u64 *seen; /* hash set, see bpf_failing_path_was_here() */
+ u32 seen_cnt, seen_bits;
unsigned long *loop_control_deps; /* see bpf_jumps_see_same() */
+ u32 loop_control_deps_for, new_data_deps_for;
u32 loop_carried_at;
unsigned long loop_carried[XREG_WORDS];
};
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c79537dd671..c4a953cf643 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1787,6 +1787,8 @@ void bpf_free_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_s
static int copy_func_state(struct bpf_func_state *dst,
const struct bpf_func_state *src)
{
+ u32 i;
+
leave_loops(dst, 0);
memcpy(dst, src, offsetof(struct bpf_func_state, stack));
/* Instruction accounting is path-local, not part of verifier state. */
@@ -1797,6 +1799,9 @@ static int copy_func_state(struct bpf_func_state *dst,
if (!dst->loop_stack)
return -ENOMEM;
dst->loop_stack_cnt = src->loop_stack_cnt;
+ for (i = 0; i < src->loop_stack_cnt; i++)
+ if (src->loop_stack[i].header_state)
+ src->loop_stack[i].header_state->refcnt++;
return copy_stack_state(dst, src);
}
@@ -10359,8 +10364,10 @@ static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_met
err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name,
meta->btf, proto_slots == nargs ? args : NULL);
if (err)
- return err;
+ return meta->subprog_is_static ? -ENODATA : err;
}
+ /* the other errors only make BTF unreliable, see btf_check_subprog_call() */
+ env->no_report |= meta->subprog_is_static;
for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) {
const struct btf_type *t;
@@ -11293,6 +11300,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, s
memset(&meta, 0, sizeof(meta));
meta.btf = btf;
meta.subprog_may_change_pkt = sub->changes_pkt_data;
+ meta.subprog_is_static = !bpf_subprog_is_global(env, subprog);
meta.func_name = bpf_subprog_name(env, subprog);
ret = btf_prepare_func_args(env, subprog);
@@ -11331,6 +11339,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, s
*
* Returns:
* EFAULT - there is a verifier bug. Abort verification.
+ * ENODATA - a static function loads a stack argument that is not there.
* Other errors - there is a type mismatch or BTF is not available.
* 0 - BTF matches with what bpf_reg_state expects.
*/
@@ -11338,6 +11347,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog)
{
struct bpf_prog *prog = env->prog;
struct btf *btf = prog->aux->btf;
+ bool no_report = env->no_report;
u32 btf_id;
int err;
@@ -11352,6 +11362,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog)
return -EINVAL;
err = btf_check_func_arg_match(env, subprog, btf);
+ env->no_report = no_report;
/* Compiler optimizations can remove arguments from static functions
* or mismatched type can be passed into a global function.
* Mark a static function as unreliable from BTF point of view. The other call is unproven.
@@ -11796,6 +11807,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
err = btf_check_subprog_call(env, subprog);
if (err == -EFAULT)
return err;
+ if (err == -ENODATA)
+ return -EINVAL;
if (bpf_subprog_is_global(env, subprog)) {
struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog);
const char *sub_name = bpf_subprog_name(env, subprog);
@@ -11922,6 +11935,8 @@ static int check_func_callx_one(struct bpf_verifier_env *env, int subprog, int *
err = btf_check_subprog_call(env, subprog);
if (err == -EFAULT)
return err;
+ if (err == -ENODATA)
+ return -EINVAL;
/* bpf_compute_live_registers() counts on it */
if (verifier_bug_if(!env->subprog_info[subprog].addr_taken, env,
@@ -20932,6 +20947,86 @@ static int check_held_in_leftover(struct bpf_verifier_env *env,
return held_cnt(st) > st->max_held_cnt;
}
+/*
+ * A path-sensitive run is in a loop with a trip count. If only the jump on the iv depends on
+ * the xregs that moved, the path can take the same edges again and again:
+ * for (i = 0; i < 100000; i++) { if (rnd() && n < 8) n++; }
+ * Two trips show a step also for 'sum += i', and more than 2^32 trips also for a ring index.
+ * If the steps are not constant, the skip or "infinite" is a guess.
+ * Returns 1 if it skipped these trips.
+ */
+static int fast_forward_trips(struct bpf_verifier_env *env, struct loop_stack_entry *entry)
+{
+ struct bpf_verifier_state *cur = env->cur_state, *old = &entry->header_state->st;
+ u32 period = entry->trip - bpf_top_loop(old->frame[old->curframe])->trip;
+ bool infinite = entry->trips_left > BPF_MAX_TRIPS, guess;
+ DECLARE_BITMAP(iv_xregs, MAX_BPF_XREG) = {};
+ int err;
+
+ /* the answer needs a walk over the program */
+ if ((infinite ? entry->loop->infinite : entry->trips_left <= period) ||
+ (u64)entry->trip * entry->loop->nr_insns < env->prog->len)
+ return 0;
+ bpf_fast_forward_xregs(env, old, cur, 0, iv_xregs);
+ err = bpf_jumps_see_same(env, old, cur, entry->iv_jump_idx, infinite ? NULL : iv_xregs);
+ if (err <= 0)
+ return err;
+ guess = !bpf_steps_are_constant(env, entry, iv_xregs, infinite);
+ if (infinite) {
+ entry->loop->infinite = true;
+ entry->loop->infinite_is_guess = guess || env->run_has_guess;
+ return 0;
+ }
+ env->run_has_guess |= guess;
+ bpf_fast_forward_xregs(env, old, cur, div_u64(entry->trips_left - 1, period), iv_xregs);
+ entry->iv_jump_idx = 0;
+ entry->trips_left = 1;
+ return 1;
+}
+
+/*
+ * A path of a path-sensitive run starts a trip. Compare it with the state in which it started
+ * trip 0, 1, 2, 4, ...
+ * If the jumps see the same values, the loop is infinite and needs may_goto.
+ * With a trip count the path makes that many trips. Else it makes up to
+ * env->failing_path_max_trips. Returns 1 if it ends here.
+ */
+static int failing_path_trip(struct bpf_verifier_env *env, struct loop_stack_entry *entry)
+{
+ struct bpf_verifier_state *cur = env->cur_state, *old = &entry->header_state->st;
+ struct bpf_loop *loop = entry->loop;
+ struct loop_stack_entry mine;
+ int err;
+
+ if (entry->trip > env->failing_path_max_trips &&
+ (!entry->trips_left || entry->trips_left > BPF_MAX_TRIPS)) {
+ env->too_deep = true;
+ return 1;
+ }
+ if (entry->header_state) {
+ mine = *entry;
+ *entry = *bpf_top_loop(old->frame[old->curframe]);
+ err = bpf_same_join_key(old, cur);
+ *entry = mine;
+ if (err)
+ err = entry->trips_left ? fast_forward_trips(env, entry) :
+ bpf_jumps_see_same(env, old, cur, -1, NULL);
+ if (err < 0)
+ return err;
+ if (err && entry->trips_left)
+ return bpf_flow_save_header_state(env, entry);
+ if (err && !loop->infinite && !loop->rt_bounded) {
+ loop->infinite = true;
+ /* the next runs keep it */
+ loop->infinite_is_guess = env->run_has_guess;
+ return bpf_flow_needs_guard(env, loop);
+ } else if (err && !bpf_values_grew_in_deps(env, old, cur)) {
+ return 1;
+ }
+ }
+ return bpf_trip_is_saved(entry) ? bpf_flow_save_header_state(env, entry) : 0;
+}
+
/*
* The cpu can speculate past the jump that the equations decide.
* The removal of dead code can delete an edge only from a conditional jump.
@@ -21186,6 +21281,7 @@ static int check_loop_guard(struct bpf_verifier_env *env)
}
env->imprecise |= !env->err;
+ env->run_has_guess |= loop->infinite_is_guess;
/* the device runs the insns that the driver was given, without may_goto */
if (bpf_prog_is_offloaded(env->prog->aux)) {
verbose(env, "loop at insn %d has no known bound in an offloaded program\n",
@@ -21225,6 +21321,9 @@ static int check_loop_guard(struct bpf_verifier_env *env)
env->guards_late |= aux->loop_guard;
aux->guard_no_unwind = true;
}
+ /* another state selected the target, possibly a state that only a join makes */
+ env->run_has_guess |= aux->guard_no_unwind &&
+ state_can_unwind(env, env->cur_state, aux->guard_unset_reg);
out:
aux->loop_guard = true;
aux->guard_unwinds = !aux->guard_no_unwind;
@@ -21538,7 +21637,9 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
held = held_cnt(state);
from_unproven = reads_from_unproven(env);
fixups = insn_aux->fixups;
+ env->no_report = state->speculative;
err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
+ env->no_report = false;
/* a call or an exit gave the state to the flow, the error is from there */
if (err < 0 && !state->frame[0])
return err;
@@ -21679,6 +21780,7 @@ static int do_check(struct bpf_verifier_env *env)
/* bpf_flow_edge_unproven() noted it with the state that failed */
if (err && !bpf_insn_is_unproven(env))
note_error(env, err);
+ env->run_over_budget |= !err;
return err;
}
@@ -22764,6 +22866,16 @@ static void free_states(struct bpf_verifier_env *env)
bpf_flow_reset(env);
}
+/* A multiple of the shortest path. It doubles with each pass. A function gets half. */
+static u64 func_failing_path_budget(struct bpf_verifier_env *env)
+{
+ u64 hops = (bpf_failing_path_hops(env, env->cur_state) + 1ULL) * BPF_COST_STATE;
+
+ env->failing_path_budget = min(hops << env->failing_path_budget_shift,
+ env->failing_path_budget);
+ return env->failing_path_budget / 2;
+}
+
static int do_check_common(struct bpf_verifier_env *env, int subprog,
enum bpf_func_context context)
{
@@ -22920,6 +23032,9 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog,
}
}
+ state->insn_idx = sub->start;
+ if (env->path_sensitive)
+ env->failing_path_max_cost = env->cost + func_failing_path_budget(env);
ret = do_check(env);
out:
if (ret && !env->err)
@@ -23156,6 +23271,75 @@ static int check_after_pass(struct bpf_verifier_env *env)
return 0;
}
+/*
+ * If the error of a pass is real, do_check_all() limits the next passes and logs a path to it.
+ * To see if it is real, reproduce it with no joins. Check one path after another, depth first
+ * from the entry of the function, until the check of an insn in err_insns fails:
+ * - there are no joins, a call stays in the instance of its caller, each trip is peeled
+ * - a state is dropped if a state with the same values in the deps was at the insn
+ * - only a loop that is found to be infinite gets may_goto, see failing_path_trip()
+ * That path is the failing path. The run accepts nothing. It is not the verdict: a path has
+ * ranges too, and a check can fail on it where the joined state is proven.
+ * run_has_guess: the run made a guess, on any of its paths. Then the failing path can be false,
+ * see fast_forward_trips() and check_loop_guard().
+ * Returns the error on the path, 0 if there is none.
+ */
+static int run_path_sensitive(struct bpf_verifier_env *env, struct bpf_snapshot *initial,
+ bool repeat)
+{
+ u64 budget = env->failing_path_budget;
+ int ret;
+
+ env->path_sensitive = true;
+ env->run_has_guess = false;
+ env->err = 0;
+ ret = bpf_failing_path_prepare(env, repeat) ?: check_funcs(env, initial);
+ env->path_sensitive = false;
+ /* for the repeat */
+ if (env->err)
+ env->failing_path_budget = budget;
+ return ret == -EINTR ? ret : env->err;
+}
+
+/*
+ * The path-sensitive runs, in their order:
+ *
+ * trips of a loop without a trip count 1, any number, 3, 7, ...
+ * which edge goes first the shortest path; if over budget, the fall-through
+ *
+ * A function gets half of the budget that is left. @last: the budget is twice the cost so far.
+ */
+static int find_failing_path(struct bpf_verifier_env *env, struct bpf_snapshot *initial, bool last)
+{
+ u32 trips;
+ u64 start;
+ int ret;
+
+ if (env->log.level & BPF_LOG_LEVEL2)
+ verbose(env, "path-sensitive run after pass %u\n", env->pass);
+ env->failing_path_budget_shift = last ? 20 : min(env->pass + 3, 20);
+ env->failing_path_budget = last ? 2 * env->cost : bpf_precision_budget(env) / 4;
+ env->failing_path_max_trips = trips = 1;
+ env->fallthrough_first = false;
+ for (;;) {
+ start = env->cost;
+ ret = run_path_sensitive(env, initial, false);
+ if (ret || env->over_limit || env->bug ||
+ env->cost - start >= env->failing_path_budget)
+ return ret;
+ env->failing_path_budget -= env->cost - start;
+ env->fallthrough_first = !env->fallthrough_first && env->run_over_budget;
+ if (env->fallthrough_first)
+ continue;
+ if (env->no_mem || (!env->run_over_budget && !env->too_deep))
+ return 0;
+ if (env->failing_path_max_trips == 1)
+ env->failing_path_max_trips = U32_MAX;
+ else
+ env->failing_path_max_trips = trips = 2 * trips + 1;
+ }
+}
+
/*
* What it took to verify a static subprog on its own is not in insns_total of its callers.
* Add it, once per caller. The main prog and global subprogs have all of it already.
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 36/49] bpf: Do not let a join raise what a program requires
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (34 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 35/49] bpf: Reproduce an error on one path, with no joins Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 37/49] bpf: Verify in passes Alexei Starovoitov
` (12 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A pass can have no error and still ask for too much:
(off, len) = (0, 64) or (56, 8) each path needs 64 bytes
join: off = 0..56, len = 8..64 the pass asks for 120 bytes
Such bounds are checked at attach time or at run time. The program
loads, and then it cannot attach or run.
Mark the insn as pessimized if it raises such a bound after an
imprecise join, and an operand has a range. See struct
prog_requirements. A barrier and a lost fastcall pattern count too. The
verifier then keeps a snapshot of the pass and partitions more as for
an unproven insn, in a small budget. If a later pass has an error, the
snapshot is the result.
r0 = *(u64 *)(ctx + 8) no range, a join cannot change it
read(ctx + off, len) a join can add values to off and len
One case is not covered: a join makes a dead path reachable, and an
insn on that path has constant operands. The bound stays larger.
kprobe_override and the bounds on buffers that a tail call sets move to
the main pass, so they are compared too.
The driver of an offloaded program cannot forget what it saw. So it
sees only a pass without errors: that pass runs once more for the
driver (env->replay). Before that, each insn after an imprecise join is
pessimized. An insn that the driver refuses is unproven. prepare() now
runs before each pass for the driver, and destroy() between two such
passes. An offloaded program gets no fastcall rewrite.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 4 ++
kernel/bpf/fixups.c | 6 ---
kernel/bpf/offload.c | 3 ++
kernel/bpf/precision.c | 44 +++++++++++++++++
kernel/bpf/precision.h | 1 +
kernel/bpf/verifier.c | 96 ++++++++++++++++++++++++++++++++----
6 files changed, 138 insertions(+), 16 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 03edbd63664..b60662593ef 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -884,6 +884,7 @@ struct bpf_loop {
);
struct bpf_loop_precision undo; /* before the last partition */
struct bpf_loop_precision kept; /* see keep_precision() */
+ struct bpf_loop_precision good; /* see bpf_restore_precision() */
struct bpf_loop_per_pass last_pass; /* the last pass whose partition was kept */
/* to peel all trips, estimated from a partition that was over its budget */
u64 full_peel_cost;
@@ -1388,6 +1389,7 @@ struct bpf_verifier_env {
struct_group_tagged(bpf_env_per_pass, per_pass,
int err; /* the first error */
u32 err_insn;
+ bool pessimized; /* see bpf_partition_for_pessimized() */
bool imprecise; /* a state before the first error is imprecise */
bool over_limit; /* see bpf_check_limits() */
bool too_deep; /* see failing_path_trip() */
@@ -1734,6 +1736,8 @@ void bpf_note_unproven_insn(struct bpf_verifier_env *env,
bool bpf_insn_is_unproven(struct bpf_verifier_env *env);
void bpf_note_fixups_differ(struct bpf_verifier_env *env);
void bpf_find_unproven_after_pass(struct bpf_verifier_env *env);
+int bpf_partition_for_pessimized(struct bpf_verifier_env *env);
+void bpf_restore_precision(struct bpf_verifier_env *env);
bool bpf_same_insns_unproven(struct bpf_verifier_env *env);
int bpf_partition_more(struct bpf_verifier_env *env);
bool bpf_undo_precision(struct bpf_verifier_env *env);
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 4c9f63e096d..3bec52051a9 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -2629,8 +2629,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
prog->dst_needed = 1;
if (insn->imm == BPF_FUNC_get_prandom_u32)
bpf_user_rnd_init_once();
- if (insn->imm == BPF_FUNC_override_return)
- prog->kprobe_override = 1;
if (insn->imm == BPF_FUNC_tail_call) {
/* If we tail call into other programs, we
* cannot make any assumptions since they can
@@ -2641,10 +2639,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (!bpf_allow_tail_call_in_subprogs(env))
prog->aux->stack_depth = MAX_BPF_STACK;
prog->aux->max_pkt_offset = MAX_PACKET_OFF;
- prog->aux->max_rdonly_access = U32_MAX;
- prog->aux->max_rdwr_access = U32_MAX;
- if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE)
- prog->aux->max_tp_access = U32_MAX;
/* mark bpf_tail_call as different opcode to avoid
* conditional branch in the interpreter for every normal
diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index f7bfff1760c..572d828a478 100644
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -315,6 +315,9 @@ int bpf_prog_offload_verifier_prep(struct bpf_prog *prog)
down_read(&bpf_devs_lock);
offload = prog->aux->offload;
if (offload) {
+ /* the driver starts again, see do_check_all() */
+ if (offload->dev_state)
+ offload->offdev->ops->destroy(prog);
ret = offload->offdev->ops->prepare(prog);
offload->dev_state = !ret;
}
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index 59e425431c6..c46f18f04f1 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -1498,6 +1498,50 @@ int bpf_partition_more(struct bpf_verifier_env *env)
return find_calls_to_unproven(env) ?: partition_next(env);
}
+/*
+ * The pass has no error. But an imprecise state can raise a bound that is checked at attach
+ * time or at run time. It can also ask for a barrier. Partition as if such an insn is
+ * unproven. Returns 1 if more is kept apart.
+ */
+int bpf_partition_for_pessimized(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ int ret;
+ u32 i;
+
+ if (!env->pessimized)
+ return 0;
+ prec->good_budget = prec->budget;
+ for (i = 0; i < prec->loop_cnt; i++)
+ prec->loops[i]->good = prec->loops[i]->precision;
+ for (i = 0; i < env->prog->len; i++)
+ if (env->insn_aux_data[i].pessimized)
+ note_unproven_after_pass(env, i);
+ ret = bpf_partition_more(env);
+ if (ret <= 0 && prec->data_deps)
+ bpf_restore_precision(env);
+ return ret;
+}
+
+/* Go back to the precision of the last pass without errors */
+void bpf_restore_precision(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ struct bpf_loop *loop;
+ u32 i;
+
+ memcpy(prec->data_deps, prec->undo, 5 * prec->words * sizeof(long));
+ for (i = 0; i < 5 * prec->words; i++)
+ prec->kept[i] &= prec->data_deps[i];
+ prec->budget = prec->good_budget;
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ loop->precision = loop->good;
+ loop->partitioned = false;
+ loop->changed = true;
+ }
+}
+
void bpf_precision_new_pass(struct bpf_verifier_env *env)
{
struct bpf_precision *prec = env->precision;
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index 39284d1df97..f833b4642bc 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -62,6 +62,7 @@ struct bpf_precision {
u64 start_cost, pass_cost; /* env->cost before the pass, the cost of a pass */
/* the cost that a partition can add to it, and that this one can add */
u64 budget, budget_left;
+ u64 good_budget; /* see bpf_restore_precision() */
u8 partitioned_by, too_costly; /* bits of PARTITION_* */
bool failing_path_due; /* see bpf_failing_path_is_due() */
bool budget_grew; /* see partition_next() */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c4a953cf643..cde5c14e730 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3602,9 +3602,21 @@ static bool is_bpf_st_mem(struct bpf_insn *insn)
return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
}
+/* See bpf_partition_for_pessimized() */
+static void note_pessimized_insn(struct bpf_verifier_env *env, u32 idx)
+{
+ if (!env->imprecise)
+ return;
+ env->insn_aux_data[idx].pessimized = true;
+ env->pessimized = true;
+ if (!env->err)
+ env->err_insn = idx;
+}
+
/* See comment for mark_fastcall_pattern_for_call() */
static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
- struct bpf_func_state *state, int insn_idx, int off)
+ struct bpf_func_state *state, int insn_idx, int off,
+ bool var_off)
{
struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
struct bpf_insn_aux_data *aux = env->insn_aux_data;
@@ -3622,6 +3634,10 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
* happens at most once per subprogram
*/
for (i = subprog->start; i < (subprog + 1)->start; ++i) {
+ /* the spills can be too much for the stack, see bpf_find_unproven_after_pass() */
+ aux[insn_idx].pessimized |= aux[i].fastcall_spills_num && env->imprecise;
+ if (aux[i].fastcall_spills_num && var_off)
+ note_pessimized_insn(env, insn_idx);
aux[i].fastcall_spills_num = 0;
aux[i].fastcall_pattern = 0;
}
@@ -7058,7 +7074,9 @@ static int check_stack_access_within_bounds(
* them. An empty range touches no stack and keeps the rewrite enabled.
*/
if (min_off < max_off)
- check_fastcall_stack_contract(env, state, env->insn_idx, min_off);
+ check_fastcall_stack_contract(env, state, env->insn_idx, min_off,
+ !tnum_is_const(reg->var_off) ||
+ env->prev_insn_idx == env->insn_idx);
/* Note that there is no stack access with offset zero, so the needed stack
* size is -min_off, not -min_off+1.
@@ -13387,6 +13405,19 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
env->prog->call_get_func_ip = true;
}
+ /* a call that only a speculative state reaches is dead code */
+ if (func_id == BPF_FUNC_override_return && !env->cur_state->speculative)
+ env->prog->kprobe_override = 1;
+
+ if (func_id == BPF_FUNC_tail_call && !env->cur_state->speculative) {
+ /* the other program can access all of it */
+ if (env->prog->expected_attach_type == BPF_TRACE_ITER)
+ env->prog->aux->max_rdonly_access =
+ env->prog->aux->max_rdwr_access = U32_MAX;
+ if (env->prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE)
+ env->prog->aux->max_tp_access = U32_MAX;
+ }
+
if (func_id == BPF_FUNC_tail_call) {
/*
* The program which a tail call jumps to returns for this one,
@@ -20725,6 +20756,21 @@ static bool reads_from_unproven(struct bpf_verifier_env *env)
return false;
}
+/* The operands of @insn that have more than one value, without the number that is stored */
+static unsigned long operands_with_range(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+ unsigned long use = cur_aux(env)->use_regs & GENMASK(BPF_REG_9, BPF_REG_0), ranges = 0;
+ struct bpf_reg_state *regs = cur_regs(env);
+ int i;
+
+ if (BPF_CLASS(insn->code) == BPF_STX && !bpf_atomic_is_load_acq(insn))
+ use &= BIT(insn->dst_reg);
+ for_each_set_bit(i, &use, BPF_REG_FP)
+ if (regs[i].r64.size)
+ ranges |= BIT(i);
+ return ranges;
+}
+
static void mark_from_unproven(struct bpf_verifier_env *env, int idx, bool from_unproven)
{
struct bpf_reg_state *regs = cur_regs(env);
@@ -21494,6 +21540,26 @@ static int skip_unproven_insn(struct bpf_verifier_env *env, const struct bpf_ins
return 0;
}
+/* These decide where the program can be attached, or what it reads */
+struct prog_requirements {
+ u32 val[12];
+};
+
+static struct prog_requirements get_prog_requirements(struct bpf_verifier_env *env)
+{
+ const struct bpf_prog *prog = env->prog;
+ const struct bpf_prog_aux *aux = prog->aux;
+
+ return (struct prog_requirements){{
+ env->ops->convert_ctx_access ? 0 : aux->max_ctx_offset,
+ aux->max_tp_access, aux->max_rdonly_access, aux->max_rdwr_access,
+ aux->min_rdwr_write, aux->max_rdwr_write, aux->kprobe_write_ctx,
+ prog->type == BPF_PROG_TYPE_EXT && env->seen_direct_write,
+ prog->type == BPF_PROG_TYPE_PERF_EVENT && prog->call_get_stack,
+ prog->tstamp_type_access, prog->kprobe_override, prog->enforce_expected_attach_type,
+ }};
+}
+
static void free_pushed_states(struct bpf_verifier_env *env)
{
struct bpf_verifier_stack_elem *elem;
@@ -21513,7 +21579,10 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
struct bpf_insn *insns = env->prog->insnsi;
int insn_cnt = env->prog->len;
bool do_print_state = true, from_unproven;
+ struct prog_requirements prog_requirements, new_requirements;
bool again = prev_insn_idx == env->insn_idx;
+ unsigned long ranges;
+ bool stack_args;
u32 held;
mark_verifier_state_scratched(env);
@@ -21584,11 +21653,14 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
env->prev_log_pos = env->log.end_pos;
}
- if (bpf_prog_is_offloaded(env->prog->aux)) {
+ if (env->replay) {
err = bpf_prog_offload_verify_insn(env, env->insn_idx,
env->prev_insn_idx);
if (err)
return err;
+ } else if (bpf_prog_is_offloaded(env->prog->aux)) {
+ /* the driver cannot forget what an imprecise state shows */
+ note_pessimized_insn(env, env->insn_idx);
}
sanitize_mark_insn_seen(env);
@@ -21626,6 +21698,9 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
goto process_bpf_exit;
insn_aux->speculative_first |= !insn_aux->seen;
+ ranges = operands_with_range(env, insn);
+ stack_args = cur_func(env)->out_stack_arg_cnt;
+
if (err)
return err;
@@ -21637,9 +21712,15 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
held = held_cnt(state);
from_unproven = reads_from_unproven(env);
fixups = insn_aux->fixups;
+ prog_requirements = get_prog_requirements(env);
env->no_report = state->speculative;
err = do_check_insn(env, &do_print_state) ?: bpf_equations_after_insn(env);
env->no_report = false;
+ new_requirements = get_prog_requirements(env);
+ /* a join adds values only to a range */
+ if ((ranges || stack_args || again) &&
+ memcmp(&prog_requirements, &new_requirements, sizeof(new_requirements)))
+ note_pessimized_insn(env, prev_insn_idx);
/* a call or an exit gave the state to the flow, the error is from there */
if (err < 0 && !state->frame[0])
return err;
@@ -25296,12 +25377,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
- if (bpf_prog_is_offloaded(env->prog->aux)) {
- ret = bpf_prog_offload_verifier_prep(env->prog);
- if (ret)
- goto skip_full_check;
- }
-
ret = bpf_check_cfg(env);
if (ret < 0)
goto skip_full_check;
@@ -25380,7 +25455,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
- ret = mark_fastcall_patterns(env);
+ /* the device runs the insns that the driver was given */
+ ret = bpf_prog_is_offloaded(env->prog->aux) ? 0 : mark_fastcall_patterns(env);
if (ret < 0)
goto skip_full_check;
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 37/49] bpf: Verify in passes
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (35 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 36/49] bpf: Do not let a join raise what a program requires Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 38/49] bpf: Make a returned pointer to the stack unreadable Alexei Starovoitov
` (11 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Put the parts together. do_check_all() runs passes until a pass has no
error, or until there is no more precision:
pass 0: join all numbers
|
+---------> run a pass
| |
| error ? --- no ---> pessimized ? --- no ---> ACCEPT
| | yes | yes
| v v
| good pass before ? save the good pass,
| | no | yes partition, cut the budget
| | v |
| | ACCEPT, use that pass +---> run a pass
| v
| partition
| |
| failing path ? --- yes ---> log it; if no guess, cut the budget
| | no |
| +<-------------------------+
| v
+- yes - is more kept apart ? --- no ---> REJECT
A good pass has no error. It is a complete proof, with any precision.
If no state before the error was imprecise, the error is real and the
verifier stops. The path-sensitive run needs the deps, so it comes after
the partition. A failing path with a guess does not cut the budget. If a
pass has a real error, the log has that error and not the guess.
A valid program that no precision can prove must not use up the limit.
After the first error the limit is the larger of BPF_PARTITION_BUDGET
and two times the cost so far.
A program of 1M insns needs 1.2 GB for its tables, so BPF_MAX_MEM is
large. If the states of a small program explode, they must not use all
of it. With 2 GB a program of 500 insns takes 30 seconds before it is
rejected. So the passes can add 256 MB to the tables, and 32 KB for each
insn. That is one state with a full stack.
The later passes and the path-sensitive runs have no log, and make no
text for diagnostics. The one that gives the log runs once more. That
work is not in the limit on cost. Log level 2 has all of them.
A later pass must not hide a bug of the verifier. bpf_note_bug() sets
env->bug and turns the log on. The load ends with -EFAULT.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 13 ++
kernel/bpf/diagnostics.c | 4 +-
kernel/bpf/failing_path.c | 8 ++
kernel/bpf/precision.c | 4 +
kernel/bpf/verifier.c | 239 +++++++++++++++++++++++++++++++++++
5 files changed, 267 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b60662593ef..40faa1b3f39 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -12,6 +12,7 @@
#include <linux/math64.h> /* for div_s64_rem() */
#include <linux/freezer.h>
#include <linux/sched/signal.h>
+#include <linux/sizes.h>
/* No memory fails the load. It does not wake up the OOM killer. */
#define BPF_GFP_MAYFAIL (__GFP_RETRY_MAYFAIL | __GFP_NOWARN)
@@ -1435,6 +1436,7 @@ struct bpf_verifier_env {
bool no_rcu_read_lock;
bool signature;
u8 bug:1; /* ends the load, see bpf_note_bug() */
+ u8 log_level:7; /* of the user */
u32 insn_aux_data_len;
struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
struct bpf_patch_list patch_list;
@@ -1604,6 +1606,13 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
u32 insn_off,
const char *prefix_fmt, ...);
+/* No later pass can hide a bug of the verifier. Its message goes into the log. */
+static inline void bpf_note_bug(struct bpf_verifier_env *env)
+{
+ env->bug = true;
+ env->log.level = env->log_level;
+}
+
#define verifier_bug_if(cond, env, fmt, args...) \
({ \
bool __cond = (cond); \
@@ -1614,6 +1623,7 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
#define verifier_bug(env, fmt, args...) \
({ \
BPF_WARN_ONCE(1, "verifier bug: " fmt "\n", ##args); \
+ bpf_note_bug(env); \
bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \
})
@@ -1767,6 +1777,9 @@ void bpf_precision_free(struct bpf_verifier_env *env);
#define BPF_COST_STATE 512 /* make, order and free a state */
#define BPF_MAX_COST (1ULL << 33)
#define BPF_MAX_MEM (1ULL << 31)
+/* What the passes can add to the tables. A loop keeps a state at each join point. */
+#define BPF_PASS_MEM_BASE SZ_256M
+#define BPF_PASS_MEM_PER_INSN SZ_32K /* a state with a full stack */
/* The floors of the cost after the first error, and of the budget of a partition */
#define BPF_PARTITION_BUDGET (BPF_MAX_COST / 3)
#define BPF_MIN_BUDGET (BPF_MAX_COST >> 12)
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c
index 0e12078ffe1..8972b429bbc 100644
--- a/kernel/bpf/diagnostics.c
+++ b/kernel/bpf/diagnostics.c
@@ -255,7 +255,7 @@ static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size)
size_t capacity, available;
char *buf;
- if (!diag || !size || size > INT_MAX)
+ if (!diag || !bpf_diag_enabled(env) || !size || size > INT_MAX)
return NULL;
if (!list_empty(&diag->fmt_chunks)) {
@@ -406,6 +406,8 @@ void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos)
if (!diag)
return;
+ /* no text is in use between two blocks */
+ diag_fmt_restore(env, (struct diag_fmt_mark){});
log = &diag->log;
end_seq = log_end(log);
if (WARN_ON_ONCE(log_pos > end_seq))
diff --git a/kernel/bpf/failing_path.c b/kernel/bpf/failing_path.c
index 6f3be5d900c..76278529202 100644
--- a/kernel/bpf/failing_path.c
+++ b/kernel/bpf/failing_path.c
@@ -250,7 +250,15 @@ bool bpf_failing_path_is_due(struct bpf_verifier_env *env)
int bpf_failing_path_prepare(struct bpf_verifier_env *env, bool repeat)
{
struct bpf_precision *prec = env->precision;
+ struct bpf_loop *loop;
+ u32 i;
+ for (i = 0; i < prec->loop_cnt; i++) {
+ loop = prec->loops[i];
+ if (repeat)
+ loop->infinite = loop->was_infinite;
+ loop->was_infinite = loop->infinite;
+ }
prec->failing_path_due = false;
if (prec->seen)
memset(prec->seen, 0, sizeof(*prec->seen) << prec->seen_bits);
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index c46f18f04f1..7fc1e93aef1 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -1545,6 +1545,7 @@ void bpf_restore_precision(struct bpf_verifier_env *env)
void bpf_precision_new_pass(struct bpf_verifier_env *env)
{
struct bpf_precision *prec = env->precision;
+ u32 i;
bitmap_zero(prec->this_pass.unproven_insns, env->prog->len);
bitmap_zero(prec->this_pass.bad_read_insns, env->prog->len);
@@ -1555,6 +1556,9 @@ void bpf_precision_new_pass(struct bpf_verifier_env *env)
if (prec->apart_xregno)
memset(prec->apart_xregno, 0, env->prog->len * sizeof(u16));
if (prec->data_deps && !env->replay && (prec->budget_grew || env->for_log)) {
+ /* the last error came before the last partition */
+ for (i = 0; env->for_log && i < 5 * prec->words; i++)
+ prec->kept[i] &= prec->undo[i];
keep_precision(prec, false);
prec->partitioned_by = prec->pass_cost = 0;
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index cde5c14e730..33f7b17b2bf 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2443,6 +2443,8 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
return 0;
out:
+ if (env->test_reg_invariants)
+ bpf_note_bug(env);
verbose(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} "
"r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u\n",
ctx, msg,
@@ -23352,6 +23354,41 @@ static int check_after_pass(struct bpf_verifier_env *env)
return 0;
}
+/* Returns the first error */
+static int check_funcs(struct bpf_verifier_env *env, struct bpf_snapshot *initial)
+{
+ u32 level = env->log.level;
+ int ret;
+
+ restore_snapshot(env, initial);
+ kvfree(env->callx_edges);
+ kvfree(env->callx_index.slots);
+ env->callx_edges = NULL;
+ env->callx_index = (struct bpf_index){};
+ bpf_flow_new_pass(env);
+ ret = do_check_main(env) ?: do_check_subprogs(env) ?: bpf_check_limits(env);
+ env->log.level = level;
+ env->pass_max_cost = U64_MAX;
+ return ret == -EINTR ? ret : env->err ?: ret;
+}
+
+static int check_pass(struct bpf_verifier_env *env, struct bpf_snapshot *initial)
+{
+ int ret;
+
+ if (env->pass && (env->log.level & BPF_LOG_LEVEL2))
+ verbose(env, "pass %u\n", env->pass);
+ bpf_precision_new_pass(env);
+ ret = check_funcs(env, initial);
+ if (ret || pass_needs_restart(env))
+ return ret;
+ ret = check_after_pass(env);
+ if (ret)
+ bpf_find_unproven_after_pass(env);
+ env->err = ret;
+ return ret;
+}
+
/*
* If the error of a pass is real, do_check_all() limits the next passes and logs a path to it.
* To see if it is real, reproduce it with no joins. Check one path after another, depth first
@@ -23363,6 +23400,7 @@ static int check_after_pass(struct bpf_verifier_env *env)
* ranges too, and a check can fail on it where the joined state is proven.
* run_has_guess: the run made a guess, on any of its paths. Then the failing path can be false,
* see fast_forward_trips() and check_loop_guard().
+ * @repeat: see log_failing_path()
* Returns the error on the path, 0 if there is none.
*/
static int run_path_sensitive(struct bpf_verifier_env *env, struct bpf_snapshot *initial,
@@ -23379,6 +23417,8 @@ static int run_path_sensitive(struct bpf_verifier_env *env, struct bpf_snapshot
/* for the repeat */
if (env->err)
env->failing_path_budget = budget;
+ if (env->err && env->run_has_guess)
+ verbose(env, "this path has a guess, its values can be wrong\n");
return ret == -EINTR ? ret : env->err;
}
@@ -23421,6 +23461,204 @@ static int find_failing_path(struct bpf_verifier_env *env, struct bpf_snapshot *
}
}
+/* Level 2 has the log of all passes. Else the log has one pass, or the failing path. */
+struct pass_log {
+ u32 level, len_max;
+ u64 pos;
+};
+
+static void restart_log(struct bpf_verifier_env *env, struct pass_log *log)
+{
+ env->log.level = log->level;
+ /* the log is dropped when a function is found to be safe */
+ log->pos = min(log->pos, env->log.end_pos);
+ if (log->level & BPF_LOG_LEVEL2)
+ return;
+ bpf_vlog_reset(&env->log, log->pos);
+ env->log.len_max = log->len_max;
+}
+
+/* Repeat the last run. The verdict does not depend on the log, so it has no cost. */
+static int log_failing_path(struct bpf_verifier_env *env, struct bpf_snapshot *initial,
+ struct pass_log *log)
+{
+ u32 insns = env->insn_processed, states = env->total_states;
+ u64 cost = env->cost;
+ int i, ret;
+
+ restart_log(env, log);
+ ret = run_path_sensitive(env, initial, true);
+ env->log.level = 0;
+ env->cost = cost;
+ env->insn_processed = insns;
+ env->total_states = states;
+ /* @initial has the sums from before the run, see copy_snapshot() */
+ for (i = 0; i < env->subprog_cnt; i++) {
+ env->subprog_info[i].insns_total = initial->subprogs[i].insns_total;
+ env->subprog_info[i].insns_self = initial->subprogs[i].insns_self;
+ }
+ return ret;
+}
+
+/*
+ * pass 0: join all numbers
+ * |
+ * +---------> run a pass
+ * | |
+ * | error ? --- no ---> pessimized ? --- no ---> ACCEPT
+ * | | yes | yes
+ * | v v
+ * | good pass before ? save the good pass,
+ * | | no | yes partition, cut the budget
+ * | | v |
+ * | | ACCEPT, use that pass +---> run a pass
+ * | v
+ * | partition
+ * | |
+ * | failing path ? --- yes ---> log it; if no guess, cut the budget
+ * | | no |
+ * | +<-------------------------+
+ * | v
+ * +- yes - is more kept apart ? --- no ---> REJECT
+ *
+ * Each pass starts from the snapshot @initial. The good pass is in the snapshot @good_pass.
+ * The path-sensitive run needs the deps, so it comes after the partition.
+ * There is no more precision when the budget is used up, or when no state was imprecise.
+ *
+ * A pass assumes what stops each loop, and checks it, see pass_needs_restart(). If the
+ * assumption is wrong, the pass starts again.
+ *
+ * The log has the failing path. If there is none, or if it has a guess and the error of a pass
+ * is real, the log has the basic block of an error. Below level 2 the later passes and the
+ * path-sensitive runs write no log. The one that gives the log runs once more.
+ */
+static int do_check_all(struct bpf_verifier_env *env)
+{
+ bool offload = bpf_prog_is_offloaded(env->prog->aux), limit = false, last, replay;
+ struct pass_log log = { env->log.level, env->log.len_max, env->log.end_pos };
+ bool quiet = log.level && !(log.level & BPF_LOG_LEVEL2), logged = false, replayed = false;
+ struct bpf_snapshot *initial, *good_pass = NULL;
+ int ret, more, path_err = 0, err = 0;
+ bool guess = false, real;
+ u64 max_cost = 0;
+
+ initial = save_snapshot(env, NULL);
+ if (!initial || bpf_precision_init(env)) {
+ bpf_precision_free(env);
+ kvfree(initial);
+ return -ENOMEM;
+ }
+ env->max_mem = min(env->max_mem, env->mem + BPF_PASS_MEM_BASE +
+ (u64)BPF_PASS_MEM_PER_INSN * env->prog->len);
+ for (;; env->pass += !env->for_log) {
+ if (!env->pass || !quiet)
+ restart_log(env, &log);
+ ret = check_pass(env, initial);
+ if (quiet)
+ env->log.level = 0;
+ replay = env->replay;
+ env->replay = false;
+ if (ret == -EINTR || env->bug || env->no_mem) {
+ err = ret;
+ break;
+ }
+ if (bpf_undo_precision(env))
+ continue;
+ if (ret && good_pass) {
+ restore_snapshot(env, good_pass);
+ err = 0;
+ if (!offload)
+ break;
+ bpf_restore_precision(env);
+ } else if (!ret) {
+ bpf_flow_drop_guards(env);
+ if (pass_needs_restart(env))
+ continue;
+ if (env->for_log && (!offload || replay))
+ break;
+ max_cost = good_pass ? max_cost : env->max_cost;
+ if (!replayed && !env->for_log && bpf_partition_for_pessimized(env) > 0) {
+ env->max_cost = min(env->max_cost, env->cost + BPF_MIN_BUDGET);
+ good_pass = save_snapshot(env, good_pass);
+ if (good_pass)
+ continue;
+ }
+ if (!offload || replay) {
+ err = 0;
+ break;
+ }
+ }
+ if (!ret || good_pass) {
+ /* The driver sees a pass without errors. It can refuse an insn. */
+ kvfree(good_pass);
+ good_pass = NULL;
+ env->max_cost = max_cost;
+ env->replay = replayed = true;
+ ret = bpf_prog_offload_verifier_prep(env->prog);
+ if (!ret)
+ continue;
+ err = ret;
+ break;
+ }
+ if (env->for_log)
+ break;
+ err = path_err ?: ret;
+ limit = !env->err;
+ if (limit)
+ goto reject;
+ /* a valid program that no precision can prove must not use up the limit */
+ env->max_cost = min(env->max_cost, max(2 * env->cost, BPF_PARTITION_BUDGET));
+ last = real = !env->imprecise;
+ more = bpf_partition_more(env);
+ if (more < 0 && more != -E2BIG)
+ err = more;
+ limit = more < 0;
+ last |= more != 1;
+ if (!limit && !path_err && (last || bpf_failing_path_is_due(env))) {
+ path_err = find_failing_path(env, initial, last);
+ if (path_err) {
+ err = path_err;
+ guess = env->run_has_guess;
+ logged = quiet && log_failing_path(env, initial, &log) == err;
+ }
+ if (path_err && !guess)
+ env->max_cost = min(env->max_cost, env->cost +
+ 2 * bpf_precision_first_budget(env));
+ }
+ if (real && guess && err == path_err && err != -EINTR) {
+ err = ret;
+ logged = false;
+ }
+ if (err == -EINTR || env->bug)
+ break;
+ if (!last)
+ continue;
+reject:
+ if (logged || !quiet || (limit && !env->pass))
+ break;
+ /* Run a pass with an error once more, for the log. It has a limit of its own. */
+ env->for_log = true;
+ quiet = false;
+ env->max_cost = 2 * env->cost;
+ }
+ env->log.level = log.level;
+ /* after an abort the error and the log can be wrong */
+ if (err && env->aborted)
+ err = -EINTR;
+ if (env->bug)
+ err = -EFAULT;
+ if (!err)
+ restart_log(env, &log);
+ if (err && (!path_err || guess) && (limit || (env->for_log && !env->err)))
+ verbose(env, "the limit on cost or memory ended pass %u\n", env->pass);
+ if (!err && env->sleepable_async_cb)
+ mark_sleepable_async_cb(env);
+ bpf_precision_free(env);
+ kvfree(initial);
+ kvfree(good_pass);
+ return err;
+}
+
/*
* What it took to verify a static subprog on its own is not in insns_total of its callers.
* Add it, once per caller. The main prog and global subprogs have all of it already.
@@ -25264,6 +25502,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
* and supplied buffer to store the verification trace
*/
ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
+ env->log_level = env->log.level;
if (ret)
goto err_free_env;
ret = bpf_diag_init(env);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 38/49] bpf: Make a returned pointer to the stack unreadable
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (36 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 37/49] bpf: Verify in passes Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 39/49] bpf: Keep two states apart if a live xreg has two types Alexei Starovoitov
` (10 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
A void static function does not set r0. So r0 can hold a pointer to the
stack at the exit:
static void f(void) { char buf[8]; r0 = buf; ...; }
f(); <- nothing reads r0
The exit failed with "cannot return stack pointer to the caller".
Let the read fail, not the exit: the register is unreadable in the
caller.
old: exit of f() -> fails
new: exit of f() -> r0 is unreadable -> a read fails
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
Documentation/bpf/kfuncs.rst | 4 ++--
kernel/bpf/diagnostics.c | 3 +++
kernel/bpf/diagnostics.h | 1 +
kernel/bpf/verifier.c | 40 ++++++++++++++++++------------------
4 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst
index c7da603cab7..335dbb833bf 100644
--- a/Documentation/bpf/kfuncs.rst
+++ b/Documentation/bpf/kfuncs.rst
@@ -704,8 +704,8 @@ from a pointer is a leak, so the verifier rejects it with::
Initialize the whole return value, for example ``struct pair p = {};``, to
avoid this. A static subprogram is exempt from the scalar-only rule: it is
verified inline, so an unassigned R2 is simply passed back to the caller as
-uninitialized and only a caller that reads it fails. A stack pointer left in
-R2 is still rejected there, just as one in R0 is.
+uninitialized and only a caller that reads it fails. The same holds for a
+stack pointer left in R2 or in R0.
.. _BPF_kfunc_lifecycle_expectations:
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c
index 8972b429bbc..e17689d638b 100644
--- a/kernel/bpf/diagnostics.c
+++ b/kernel/bpf/diagnostics.c
@@ -2233,6 +2233,9 @@ static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_h
"call invalidated this outgoing stack argument" :
"call invalidated this caller-saved register";
break;
+ case BPF_DIAG_MOD_RET_STACK_PTR:
+ reason = "the caller cannot read a returned pointer to the stack";
+ break;
case BPF_DIAG_MOD_WRITE:
if (target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT)
reason = "a later stack write overwrote this spilled value";
diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h
index cb255218c41..c0303c4aab4 100644
--- a/kernel/bpf/diagnostics.h
+++ b/kernel/bpf/diagnostics.h
@@ -26,6 +26,7 @@ enum bpf_diag_mod_reason {
BPF_DIAG_MOD_SLICE_BUF,
BPF_DIAG_MOD_NON_OWN_REF,
BPF_DIAG_MOD_CALLER_SAVED,
+ BPF_DIAG_MOD_RET_STACK_PTR,
};
enum bpf_diag_context_kind {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 33f7b17b2bf..e29a72eba02 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -11629,21 +11629,32 @@ static int call_func_on_its_own(struct bpf_verifier_env *env, int subprog, int i
return err ?: PROCESS_BPF_EXIT;
}
+/*
+ * A void function can leave a pointer to the stack in r0. The caller cannot read it. That holds
+ * for the stack of a caller too: for stack liveness a call returns no pointer to the stack.
+ */
+static void hide_returned_stack_ptrs(struct bpf_verifier_env *env, int subprog,
+ struct bpf_reg_state *regs)
+{
+ u32 i, nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
+
+ for (i = 0; i < nregs; i++) {
+ if (regs[ret_regs[i]].type != PTR_TO_STACK)
+ continue;
+ verbose(env, "the caller cannot read the stack pointer in R%d\n", ret_regs[i]);
+ bpf_diag_record_scrub(env, ®s[ret_regs[i]], BPF_DIAG_MOD_RET_STACK_PTR);
+ bpf_mark_reg_not_init(env, ®s[ret_regs[i]]);
+ }
+}
+
/* The current state is at exit of a function that is verified on its own. */
static int exit_func_on_its_own(struct bpf_verifier_env *env)
{
struct bpf_verifier_state *state = env->cur_state;
struct bpf_func_state *callee = state->frame[0];
- u32 i, nregs;
int err;
- nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1;
- for (i = 0; i < nregs; i++) {
- if (callee->regs[ret_regs[i]].type != PTR_TO_STACK)
- continue;
- verbose(env, "cannot return stack pointer to the caller\n");
- return -EINVAL;
- }
+ hide_returned_stack_ptrs(env, callee->subprogno, callee->regs);
state->unwound |= bpf_is_unwind_exit(&env->prog->insnsi[env->insn_idx]);
/* Invalidate callee-local dynptrs and their slices before the frame goes away. */
@@ -12402,18 +12413,6 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
callee = state->frame[state->curframe];
r0 = &callee->regs[BPF_REG_0];
nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1;
- for (i = 0; i < nregs; i++) {
- if (callee->regs[ret_regs[i]].type != PTR_TO_STACK)
- continue;
- /* technically it's ok to return caller's stack pointer
- * (or caller's caller's pointer) back to the caller,
- * since these pointers are valid. Only current stack
- * pointer will be invalid as soon as function exits,
- * but let's be conservative
- */
- verbose(env, "cannot return stack pointer to the caller\n");
- return -EINVAL;
- }
caller = state->frame[state->curframe - 1];
if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx]) || state->unwound) {
@@ -12472,6 +12471,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
caller->regs[regno] = callee->regs[regno];
bpf_diag_mod_end(env);
}
+ hide_returned_stack_ptrs(env, callee->subprogno, caller->regs);
}
/* Invalidate callee-local dynptrs and their slices before the frame goes away. */
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 39/49] bpf: Keep two states apart if a live xreg has two types
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (37 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 38/49] bpf: Make a returned pointer to the stack unreadable Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 40/49] bpf: Check the body of a counted loop once Alexei Starovoitov
` (9 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Pass 0 joins all states. A join of two types or two objects is
unreadable. If the xreg is live, the program probably reads it. Then the
pass has an error, and the price is two more runs:
r6 = 0; r6 = map value;
\ /
if (r6 == 0) goto out;
old: join -> r6 !read_ok -> path-sensitive run -> pass 1: apart
new: pass 0: apart
The same holds for a packet pointer that has checked bytes in one state
and has none in the other state.
bpf_hash_live_types() adds the types and objects of the live xregs to
the hash of a state. So such states do not meet.
- Numbers are joined as before.
- The arguments of the calls of an instance are joined as before, else
the calls do not share it.
- N such xregs can give 1 << N states. So the rule has a budget in each
pass: 1/64 of the limits on cost and memory. The cost of a state
that is alone does not count, see bpf_flow_next(). A pass that is over
the budget joins from there on, and so do the next passes. Then an
error selects the xregs.
- Offloaded programs are joined as before. The replay for the driver
has to join where the good pass did.
insns processed
program before after
profiler3 kprobe__vfs_link 7969 3865
test_cls_redirect_subprogs 5726 3091
xdp_synproxy_kern syncookie_xdp 8136 7630
bpf_flow flow_dissector_0 1363 701
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 2 ++
kernel/bpf/flow.c | 18 +++++++++++
kernel/bpf/precision.c | 60 ++++++++++++++++++++++++++++++++++++
kernel/bpf/precision.h | 3 ++
4 files changed, 83 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 40faa1b3f39..ba31a2e45fc 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1729,6 +1729,8 @@ bool bpf_must_keep_apart(struct bpf_verifier_env *env, const struct bpf_verifier
const struct loop_stack_entry *entry);
bool bpf_values_grew_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
const struct bpf_verifier_state *new);
+u64 bpf_hash_live_types(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u64 hash, u64 apart_cost);
u64 bpf_hash_apart_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u64 hash, bool consts);
u64 bpf_hash_args(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u64 hash);
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index a1ac4709b68..ffae6cf95f9 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -58,6 +58,7 @@ struct flow_state {
enum flow_kind kind;
bool pending;
bool late_exit_reaches; /* a late exit state can get to it */
+ bool from_apart; /* it comes from a state that was not alone */
/* it is kept until the states leave this loop, see flow_loop_is_left() */
u8 kept_frameno, kept_depth;
/* the order of pending states, see flow_set_pending() */
@@ -118,6 +119,7 @@ struct bpf_flow {
/* of the pass */
u64 *cost; /* per function */
u64 budget, charged_cost;
+ u64 apart_cost; /* of states that are not alone, see bpf_flow_next() */
u32 budget_func_cnt;
/* of the check */
@@ -128,6 +130,7 @@ struct bpf_flow {
struct flow_instance **instances; /* bpf_verifier_state.instance_id is the index */
u32 instance_cnt, instance_cap;
struct flow_instance *last_instance; /* of the last pending state */
+ bool last_is_apart; /* it adds to @apart_cost */
u64 priority_gen;
struct list_head unused; /* struct flow_kept */
u32 seq;
@@ -426,6 +429,7 @@ void bpf_flow_new_pass(struct bpf_verifier_env *env)
memset(flow->cost, 0, env->subprog_cnt * sizeof(*flow->cost));
flow->budget = bpf_precision_budget(env);
flow->budget_func_cnt = 0;
+ flow->apart_cost = 0;
flow->start_cnt++;
for (i = 0; i < env->prog->len; i++, aux++) {
if (aux->callback_loop)
@@ -631,6 +635,7 @@ static struct flow_state *flow_new_state(struct bpf_verifier_env *env, int prev_
INIT_LIST_HEAD(&s->node);
s->prev_insn_idx = prev_insn_idx;
s->seq = ++flow->seq;
+ s->from_apart = flow->last_is_apart;
s->log_pos = env->log.end_pos;
s->diag_pos = bpf_diag_event_log_save(env);
bpf_add_cost(env, 1, BPF_COST_STATE);
@@ -723,6 +728,7 @@ void bpf_flow_reset(struct bpf_verifier_env *env)
env->mem -= flow->instance_cnt * INSTANCE_BYTES;
flow->instance_cnt = 0;
flow->last_instance = NULL;
+ flow->last_is_apart = false;
}
void bpf_flow_free(struct bpf_verifier_env *env)
@@ -870,6 +876,9 @@ static struct flow_state *flow_find(struct bpf_verifier_env *env, struct flow_st
hash = bpf_hash_apart_xregs(env, &s->st, bpf_hash_join_key(env, &s->st) + s->kind,
s->kind == FLOW_PEELED);
+ /* the arguments are joined, else calls do not share an instance */
+ if (s->kind != FLOW_FUNC_ENTRY && s->kind != FLOW_JOINED_CALLS)
+ hash = bpf_hash_live_types(env, &s->st, hash, env->flow->apart_cost);
if (s->kind == FLOW_FUNC_ENTRY && (!e || e->peeled))
hash = bpf_hash_args(env, &s->st, hash);
if (s->kind == FLOW_AT ? !aux->loop && !aux->callback_loop :
@@ -1722,6 +1731,8 @@ int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos
again:
if (flow->last_instance && flow->last_instance->cost)
*flow->last_instance->cost += env->cost - flow->charged_cost;
+ if (flow->last_is_apart)
+ flow->apart_cost += env->cost - flow->charged_cost;
flow->charged_cost = env->cost;
if (flow->last_instance)
flow_end_instance(env, flow->last_instance);
@@ -1731,6 +1742,13 @@ int bpf_flow_next(struct bpf_verifier_env *env, int *prev_insn_idx, u64 *log_pos
s = rb_entry(n, struct flow_state, rb);
flow_clear_pending(flow, s);
flow->last_instance = flow_state_instance(flow, s);
+ /*
+ * Is another state at this insn? In a loop the states go one by one, and only the join
+ * points of a leftover keep them. So a yes holds up to a join point that is not peeled.
+ */
+ if (test_bit(s->st.insn_idx, flow->join_points) && !flow_in_peeled_trip(&s->st))
+ s->from_apart = false;
+ flow->last_is_apart = s->from_apart || flow->nr_states[s->st.insn_idx] > 1;
instance = s->callee;
if (instance) {
if (instance->entry_state && !instance->priority) {
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index 7fc1e93aef1..cafa7805295 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -121,6 +121,9 @@ const struct bpf_reg_state *bpf_xreg_value(struct bpf_verifier_env *env,
slot_value(env, f, bpf_xreg_to_spi(xregno), tmp);
}
+/* The budget of bpf_hash_live_types() in one pass */
+#define LIVE_TYPES_PART 64 /* of the limits */
+
static u64 hash_type(const struct bpf_reg_state *reg, u64 hash)
{
if (type_is_pkt_pointer(reg->type) || base_type(reg->type) == PTR_TO_ARENA)
@@ -128,6 +131,54 @@ static u64 hash_type(const struct bpf_reg_state *reg, u64 hash)
return bpf_hash64(reg, offsetof(typeof(*reg), var_off), hash);
}
+/*
+ * A join of two types or objects is unreadable. A join of a packet pointer that has checked
+ * bytes with one that has none gives no access. If the xreg is live, an error is probable, and
+ * the price is one more pass. So such states stay apart from the start:
+ *
+ * r6 = 0; r6 = map value;
+ * \ /
+ * if (r6 == 0) goto out; two states
+ *
+ * N such xregs can give 1 << N states. If a pass is over the budget, it joins from there on,
+ * and so do the next passes. Then an error selects the xregs, see add_deps().
+ * @apart_cost: the cost of this pass without the states that were alone.
+ */
+u64 bpf_hash_live_types(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u64 hash, u64 apart_cost)
+{
+ struct bpf_precision *prec = env->precision;
+ const struct bpf_insn_aux_data *aux;
+ const struct bpf_reg_state *reg;
+ unsigned long live[XREG_WORDS];
+ struct bpf_reg_state tmp;
+ u32 fr, xregno, bits;
+
+ if (env->path_sensitive)
+ return hash;
+ prec->live_types_over |= apart_cost > prec->live_types_cost ||
+ env->mem > prec->start_mem + prec->live_types_mem;
+ for (fr = 0; !prec->live_types_over && fr <= st->curframe; fr++) {
+ aux = &env->insn_aux_data[bpf_frame_insn_idx(st, fr)];
+ bits = bpf_spi_to_xreg(bpf_stack_nr_slots(st->frame[fr]));
+ bitmap_zero(live, bits);
+ live[0] = aux->live_regs_before & XREG_REGS;
+ bpf_add_slots(live, aux->live_stack_before, bits);
+ bpf_add_cost(env, bitmap_weight(live, bits), BPF_COST_COPY);
+ for_each_set_bit(xregno, live, bits) {
+ reg = bpf_xreg_value(env, st->frame[fr], xregno, &tmp);
+ /* a slot with no spill hashes like a slot that is not allocated */
+ if (!reg || reg == &tmp || reg->type == SCALAR_VALUE)
+ continue;
+ hash = hash_type(reg, hash + fr * MAX_BPF_XREG + xregno);
+ if (type_is_pkt_pointer(reg->type) && pkt_has_bytes_left(reg) &&
+ pkt_bytes_left(reg) > 0)
+ hash = ~hash;
+ }
+ }
+ return hash;
+}
+
/*
* States with another type or object in data_deps stay apart. Hash these like the join_key.
* @consts: find a state with the same constants in all deps.
@@ -1391,6 +1442,7 @@ bool bpf_undo_precision(struct bpf_verifier_env *env)
return false;
}
env->over_limit = false;
+ prec->live_types_over = false;
memcpy(prec->data_deps, prec->undo, 5 * prec->words * sizeof(long));
for (i = 0; i < prec->loop_cnt; i++)
over_budget |= prec->loops[i]->partitioned &&
@@ -1563,7 +1615,11 @@ void bpf_precision_new_pass(struct bpf_verifier_env *env)
prec->partitioned_by = prec->pass_cost = 0;
}
prec->budget_grew = prec->new_nospec = false;
+ /* the pass for the log has to join where the last pass did */
+ prec->join_live_types |= prec->live_types_over && !env->for_log;
+ prec->live_types_over = prec->join_live_types;
prec->start_cost = env->cost;
+ prec->start_mem = env->mem;
prec->last_unproven_left = bitmap_weight(prec->last_unproven_insns, env->prog->len);
/* ADD_DEPS has no alternative, so it has no limit of its own */
if (prec->partitioned_by > BIT(PARTITION_ADD_DEPS) && !env->replay)
@@ -1620,6 +1676,10 @@ int bpf_precision_init(struct bpf_verifier_env *env)
prec->this_pass.unproven_funcs = bits + 16 * words;
prec->last_pass.unproven_funcs = bits + 17 * words;
prec->pass_cost = env->cost;
+ prec->live_types_cost = env->max_cost / LIVE_TYPES_PART;
+ prec->live_types_mem = env->max_mem / LIVE_TYPES_PART;
+ /* the replay for the driver has to join where the good pass did */
+ prec->join_live_types = bpf_prog_is_offloaded(env->prog->aux);
prec->budget_left = prec->budget = bpf_precision_first_budget(env);
prec->first_unproven_insn = prec->loop_carried_at = U32_MAX;
for (i = 0; i < env->prog->len && !err; i++, aux++)
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index f833b4642bc..3af6f10eed7 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -67,6 +67,9 @@ struct bpf_precision {
bool failing_path_due; /* see bpf_failing_path_is_due() */
bool budget_grew; /* see partition_next() */
bool new_nospec;
+ /* see bpf_hash_live_types() */
+ u64 live_types_cost, live_types_mem, start_mem;
+ bool live_types_over, join_live_types; /* in this pass, in all next passes */
u16 *apart_xregno; /* per insn, see bpf_range_of_data_dep() */
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 40/49] bpf: Check the body of a counted loop once
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (38 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 39/49] bpf: Keep two states apart if a live xreg has two types Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 41/49] bpf: Refine the copies of a number that AND or OR splits Alexei Starovoitov
` (8 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The leftover needs three walks over the body. Two walks only collect
what the third walk needs:
walk 1 i = 0 shows the step, 150
walk 2 i = 0, 150 shows the threshold, k <= 3
walk 3 i = 150 * k is the proof
Often the code shows the step and the bound. The spine is the insns
that each trip runs once. find_steps() follows the xregs along it:
header -> a -> b -> ... -> back edge on the spine: x += constant
`-> c -' off the spine: x is inexact
bpf_guess_first_trips() makes the states that trips 1 and 2 possibly
give, and joins them with the state that enters. So walk 3 is first.
It is a guess. The state only grows, and the state that comes back is
compared with it as before. The guess is not used if
- a carried xreg is a sum, or the last value of an iv, and its start
is known: only two walks show its equation,
- 'i != n' has a range, see add_distance_equation(),
- a partition needs an xreg that the loop changes,
- a state had no trip count for the loop.
A wrong guess can cost the trip count. Then the loop gets may_goto. It
can also move a register that const_fold.c knows as a constant. So
do_check_block() does not compare the two after env->extra_values is
set.
insns processed
program bpf-next without with this patch
pyperf600 37285 109729 37284
pyperf180 22165 43304 22164
pyperf600_nounroll 1289 1758 1288
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 6 +
kernel/bpf/equation_join.c | 409 +++++++++++++++++++++++++++++++++++
kernel/bpf/fixups.c | 1 +
kernel/bpf/flow.c | 6 +
kernel/bpf/precision.c | 15 ++
5 files changed, 437 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index ba31a2e45fc..16eac538f43 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -897,6 +897,8 @@ struct bpf_loop {
bool was_infinite; /* at the start of the last path-sensitive run */
bool infinite; /* see failing_path_trip() */
bool infinite_is_guess; /* see fast_forward_trips() */
+ bool no_first_trips_guess; /* a state had no trip count, @steps can be wrong */
+ struct bpf_loop_steps *steps; /* see bpf_guess_first_trips() */
struct bpf_iv_threshold *iv_thresholds; /* where an iv can stop, see equation_join.c */
u32 iv_threshold_cnt, iv_threshold_cap;
u32 leftover_epoch; /* how many leftovers started */
@@ -1738,6 +1740,8 @@ void bpf_range_of_data_dep(struct bpf_verifier_env *env, const struct bpf_verifi
u32 *first, u32 *last);
bool bpf_call_needs_frames(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u16 args);
+bool bpf_xregs_are_partitioned(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 fr, const unsigned long *xregs);
bool bpf_xreg_in_data_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
u32 fr, u32 xregno);
bool bpf_xreg_in_deps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
@@ -2547,6 +2551,8 @@ bool bpf_equations_join_forgets(struct bpf_verifier_env *env, const struct bpf_v
u32 bpf_equations_hash(struct bpf_verifier_env *env, const struct bpf_verifier_state *st, u32 fr,
const unsigned long *xregs, u32 bits);
void bpf_collect_iv_thresholds(struct bpf_verifier_env *env);
+int bpf_guess_first_trips(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ struct bpf_loop *loop);
bool bpf_steps_are_constant(struct bpf_verifier_env *env, const struct loop_stack_entry *entry,
const unsigned long *iv_xregs, bool infinite);
void bpf_fast_forward_xregs(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
diff --git a/kernel/bpf/equation_join.c b/kernel/bpf/equation_join.c
index 47db8f8ae2b..a01f0fdc746 100644
--- a/kernel/bpf/equation_join.c
+++ b/kernel/bpf/equation_join.c
@@ -1047,3 +1047,412 @@ void bpf_collect_iv_thresholds(struct bpf_verifier_env *env)
e->exit_threshold = min(e->exit_threshold, start + cnt);
}
}
+
+/* Steps */
+
+#define ORIGIN_CONST (U16_MAX - 1)
+#define ORIGIN_NONE U16_MAX
+
+/* The value that @xregno has at the header, plus @off */
+struct origin {
+ s64 off, step;
+ u16 xregno;
+ bool exact; /* on all paths */
+ bool changed; /* by more than a constant, or not on all paths */
+ bool shifted; /* by 32 bits, to truncate */
+ bool low32; /* the high bits are 0, or */
+ bool sext; /* copies of the sign */
+};
+
+struct bpf_loop_steps {
+ DECLARE_BITMAP(changed_xregs, MAX_BPF_XREG);
+ DECLARE_BITMAP(nonlinear_xregs, MAX_BPF_XREG);
+ DECLARE_BITMAP(sum_xregs, MAX_BPF_XREG); /* sums, and the last values of ivs */
+ u32 jump_cnt, cnt;
+ /* the operands of the jumps that leave the loop, then the xregs that move */
+ struct {
+ u32 insn_idx;
+ struct origin op[2];
+ } at[];
+};
+
+struct scan {
+ struct bpf_verifier_env *env;
+ struct bpf_loop_steps *steps;
+ struct origin xregs[MAX_BPF_XREG];
+ struct origin before[MAX_BPF_XREG]; /* the writes off the spine */
+ DECLARE_BITMAP(written, MAX_BPF_XREG);
+ bool on_spine;
+};
+
+static const struct origin no_origin = { .xregno = ORIGIN_NONE };
+
+static struct origin const_origin(s64 value)
+{
+ return (struct origin){ .off = value, .xregno = ORIGIN_CONST, .exact = true };
+}
+
+static u64 extend_low32(u64 x, bool sext)
+{
+ return sext ? (s64)(s32)x : (u32)x;
+}
+
+static struct origin truncate_origin(struct origin x, bool sext)
+{
+ x.low32 = x.xregno != ORIGIN_CONST;
+ x.sext = sext;
+ if (!x.low32)
+ x.off = extend_low32(x.off, sext);
+ return x;
+}
+
+/* The result of an insn has the origin that changes most */
+static int origin_rank(const struct scan *scan, const struct origin *x)
+{
+ return x->xregno >= ORIGIN_CONST ? 0 : 1 + test_bit(x->xregno, scan->steps->changed_xregs);
+}
+
+static struct origin alu_origin(const struct scan *scan, const struct bpf_insn *insn)
+{
+ struct origin dst = scan->xregs[insn->dst_reg], src = const_origin(insn->imm);
+ u8 op = BPF_OP(insn->code);
+
+ if (BPF_SRC(insn->code) == BPF_X)
+ src = scan->xregs[insn->src_reg];
+ if (op == BPF_MOV && (!insn->off || insn->off == 32))
+ return insn->off ? truncate_origin(src, true) : src;
+ if (op == BPF_MOV)
+ dst = no_origin;
+ if (src.xregno == ORIGIN_CONST && src.off == 32 && BPF_CLASS(insn->code) == BPF_ALU64 &&
+ (dst.shifted ? op == BPF_RSH || op == BPF_ARSH : op == BPF_LSH)) {
+ dst.shifted = !dst.shifted;
+ return dst.shifted ? dst : truncate_origin(dst, op == BPF_ARSH);
+ }
+ if (origin_rank(scan, &dst) < origin_rank(scan, &src)) {
+ swap(dst, src);
+ dst.exact &= op == BPF_ADD;
+ }
+ dst.off += op == BPF_ADD ? src.off : -src.off;
+ dst.exact &= (op == BPF_ADD || op == BPF_SUB) && !dst.shifted && !src.shifted &&
+ src.xregno == ORIGIN_CONST && src.exact;
+ dst.changed |= !dst.exact;
+ return dst.xregno < ORIGIN_CONST || dst.exact ? dst : no_origin;
+}
+
+static void write_origin(struct scan *scan, u32 xregno, struct origin value)
+{
+ struct origin *before = &scan->before[xregno];
+
+ bpf_add_cost(scan->env, 1, BPF_COST_COPY);
+ if (!scan->on_spine && !__test_and_set_bit(xregno, scan->written))
+ *before = scan->xregs[xregno];
+ before->changed |= value.xregno == before->xregno &&
+ (value.changed || value.off != before->off);
+ scan->xregs[xregno] = value;
+}
+
+/* Paths meet. Some of them went around the writes. */
+static void merge_origins(struct scan *scan)
+{
+ struct origin *old, *new;
+ u32 i;
+
+ for_each_set_bit(i, scan->written, MAX_BPF_XREG) {
+ bpf_add_cost(scan->env, 1, BPF_COST_COPY);
+ old = &scan->before[i];
+ new = &scan->xregs[i];
+ if (old->xregno == new->xregno && old->off == new->off && old->exact &&
+ new->exact && old->shifted == new->shifted)
+ continue;
+ new->xregno = new->xregno < ORIGIN_CONST ? new->xregno : old->xregno;
+ new->changed |= old->changed;
+ new->exact = false;
+ }
+ if (scan->on_spine)
+ bitmap_zero(scan->written, MAX_BPF_XREG);
+}
+
+static void scan_insn(struct scan *scan, int idx)
+{
+ struct bpf_verifier_env *env = scan->env;
+ const struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx];
+ const struct bpf_insn *insn = &env->prog->insnsi[idx];
+ struct origin *xregs = scan->xregs, value = no_origin;
+ u32 i, slot = slot_of_insn(env, insn);
+ unsigned long def = aux->def_regs;
+ u8 class = BPF_CLASS(insn->code);
+
+ if (class == BPF_ALU || class == BPF_ALU64)
+ value = alu_origin(scan, insn);
+ else if (slot)
+ value = class == BPF_ST ? const_origin(insn->imm) :
+ xregs[class == BPF_LDX ? slot : insn->src_reg];
+ if (class == BPF_ALU || (slot && BPF_SIZE(insn->code) == BPF_W))
+ value = truncate_origin(value,
+ class == BPF_LDX && BPF_MODE(insn->code) == BPF_MEMSX);
+ if (slot && class != BPF_LDX)
+ return write_origin(scan, slot, value);
+ for_each_set_bit(i, &def, BPF_REG_FP)
+ write_origin(scan, i, value);
+ for_each_set_bit(i, aux->may_write_mask, MAX_BPF_STACK_SLOTS)
+ write_origin(scan, bpf_spi_to_xreg(i), no_origin);
+}
+
+/* What one trip adds to the origin of @op. False if that is not a constant. */
+static bool find_step(const struct origin *xregs, struct origin *op)
+{
+ const struct origin *end = op->xregno < ORIGIN_CONST ? &xregs[op->xregno] : op;
+
+ op->step = end == op ? 0 : end->off;
+ return op->exact && !op->shifted && end->exact && !end->shifted &&
+ end->xregno == op->xregno;
+}
+
+static bool is_invariant(const struct origin *xregs, u32 xregno)
+{
+ return xregs[xregno].xregno == xregno && xregs[xregno].exact && !xregs[xregno].off;
+}
+
+/*
+ * The spine is the insns that each trip runs once. Follow the xregs along it:
+ *
+ * header -> a -> b -> ... -> back edge on the spine: x = x at the header + constant
+ * `-> c -' off the spine: a write makes x inexact
+ */
+static void scan_loop(struct scan *scan, const struct bpf_loop *loop, const unsigned long *live,
+ const int *spine, u32 top)
+{
+ struct bpf_verifier_env *env = scan->env;
+ const struct bpf_insn_aux_data *aux = &env->insn_aux_data[loop->header];
+ unsigned long consts = aux->const_reg_mask;
+ struct bpf_loop_steps *steps = scan->steps;
+ struct origin *xregs = scan->xregs;
+ int n, prev = -1, h = loop->header;
+ const struct bpf_insn *insn;
+ u32 i;
+
+ bpf_add_cost(env, loop->nr_insns + MAX_BPF_XREG, BPF_COST_XREG);
+ bitmap_zero(scan->written, MAX_BPF_XREG);
+ steps->cnt = 0;
+ for (i = 0; i < MAX_BPF_XREG; i++)
+ xregs[i] = !test_bit(i, live) ? no_origin :
+ (struct origin){ .xregno = i, .exact = true };
+ for_each_set_bit(i, &consts, BPF_REG_FP)
+ xregs[i] = const_origin(aux->const_reg_vals[i]);
+ for (n = h; n >= 0 && (n == h || insn_is_in_loop(env, loop, n)) && !over_limit(env);
+ prev = n, n = bpf_flow_insn_after(env, n)) {
+ insn = &env->prog->insnsi[n];
+ scan->on_spine = top && spine[top - 1] == n;
+ top -= scan->on_spine;
+ scan->on_spine &= bpf_loop_at_index(env, n) == h;
+ if (n != h && (env->idoms[n] != prev ||
+ bpf_loop_at_index(env, n) != bpf_loop_at_index(env, prev)))
+ merge_origins(scan);
+ if (scan->on_spine && bpf_insn_leaves_loop(env, h, n)) {
+ steps->at[steps->cnt].insn_idx = n;
+ steps->at[steps->cnt].op[0] = xregs[insn->dst_reg];
+ steps->at[steps->cnt++].op[1] = BPF_SRC(insn->code) == BPF_X ?
+ xregs[insn->src_reg] : const_origin(insn->imm);
+ }
+ scan_insn(scan, n);
+ }
+ merge_origins(scan);
+ for_each_set_bit(i, live, MAX_BPF_XREG)
+ __assign_bit(i, steps->changed_xregs, !is_invariant(xregs, i));
+}
+
+static int find_steps(struct bpf_verifier_env *env, struct bpf_loop *loop)
+{
+ const struct bpf_insn_aux_data *aux = &env->insn_aux_data[loop->header];
+ DECLARE_BITMAP(live, MAX_BPF_XREG) = {};
+ struct bpf_loop_steps *steps;
+ struct origin *xregs, op;
+ struct scan *scan;
+ int *spine, n, h = loop->header, err = -ENOMEM;
+ u32 i, top = 0, jumps = 0;
+
+ live[0] = aux->live_regs_before & ~aux->const_reg_mask & GENMASK(BPF_REG_9, BPF_REG_0);
+ for_each_set_bit(i, aux->live_stack_before, MAX_BPF_STACK_SLOTS)
+ __set_bit(bpf_spi_to_xreg(i), live);
+ scan = kvzalloc_obj(*scan, BPF_VERIFIER_GFP);
+ spine = kvmalloc_objs(*spine, loop->nr_insns, BPF_VERIFIER_GFP);
+ for (n = bpf_loop_spine_end(env, loop); spine && n >= 0 && top < loop->nr_insns;
+ n = n == h ? -1 : env->idoms[n]) {
+ spine[top++] = n;
+ jumps += bpf_insn_leaves_loop(env, h, n);
+ }
+ steps = kvzalloc_flex(*steps, at, jumps + bitmap_weight(live, MAX_BPF_XREG),
+ BPF_VERIFIER_GFP);
+ if (!scan || !spine || !steps)
+ goto out;
+ scan->env = env;
+ scan->steps = steps;
+ xregs = scan->xregs;
+ /* the second scan knows which xregs change */
+ for (i = 0; i < 2; i++)
+ scan_loop(scan, loop, live, spine, top);
+ err = bpf_check_limits(env);
+ if (err)
+ goto out;
+ /* keep the jumps that compare an iv with a bound */
+ for (i = 0; i < steps->cnt; i++)
+ if (find_step(xregs, &steps->at[i].op[0]) &&
+ find_step(xregs, &steps->at[i].op[1]) &&
+ !steps->at[i].op[0].step != !steps->at[i].op[1].step)
+ steps->at[steps->jump_cnt++] = steps->at[i];
+ steps->cnt = steps->jump_cnt;
+ for_each_set_bit(i, live, MAX_BPF_XREG) {
+ op = xregs[i];
+ if (op.xregno != i || !op.exact)
+ __set_bit(i, steps->nonlinear_xregs);
+ if (op.xregno >= ORIGIN_CONST)
+ continue;
+ if (op.changed ? !is_invariant(xregs, op.xregno) :
+ op.xregno != i && find_step(xregs, &op) && op.step)
+ __set_bit(i, steps->sum_xregs);
+ else if (op.xregno == i && find_step(xregs, &op) && op.step)
+ steps->at[steps->cnt++].op[0] = op;
+ }
+ loop->steps = kvmemdup(steps, struct_size(steps, at, steps->cnt), BPF_VERIFIER_GFP);
+ env->mem += struct_size(steps, at, steps->cnt);
+ err = loop->steps ? 0 : -ENOMEM;
+out:
+ kvfree(steps);
+ kvfree(scan);
+ kvfree(spine);
+ return err;
+}
+
+static struct scalar *origin_bounds(struct bpf_verifier_env *env,
+ const struct bpf_verifier_state *st, const struct origin *op,
+ bool *is_ptr)
+{
+ return bpf_find_var_bounds(env, st, bpf_xreg(st->curframe, op->xregno), is_ptr);
+}
+
+/* In how many trips does the jump at @idx stay in the loop? */
+static u64 trips_by_steps(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ const struct bpf_loop *loop, int idx, const struct origin *op)
+{
+ const struct bpf_insn *insn = &env->prog->insnsi[idx];
+ bool is_ptr[2] = {}, i = !op[0].step;
+ u8 cond = loop_cond_opcode(insn, insn_is_in_loop(env, loop, idx + insn->off + 1), i);
+ struct scalar bound, off, *iv, *s;
+ u64 x;
+
+ iv = origin_bounds(env, st, &op[i], &is_ptr[0]);
+ if (!iv)
+ return U64_MAX;
+ scalar_set_const(&bound, op[!i].off);
+ if (op[!i].xregno != ORIGIN_CONST) {
+ s = origin_bounds(env, st, &op[!i], &is_ptr[1]);
+ if (!s)
+ return U64_MAX;
+ off = bound;
+ scalar_add(&bound, s, &off);
+ scalar_bounds_sync(&bound);
+ }
+ /* it needs the first trip, see add_distance_equation() */
+ if (cond == BPF_JNE && (iv->r64.size || bound.r64.size))
+ return 0;
+ if (iv->r64.size || is_ptr[0] != is_ptr[1])
+ return U64_MAX;
+ x = iv->r64.base + op[i].off;
+ x = op[i].low32 ? extend_low32(x, op[i].sext) : x;
+ return compute_trip_count(op[i].step, &x, cond, BPF_CLASS(insn->code) == BPF_JMP32, bound);
+}
+
+/* Make @st the state that the steps give for the next trip */
+static bool move_by_steps(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct bpf_loop_steps *steps)
+{
+ struct bpf_xreg_equation *xreg_eq;
+ const struct origin *op;
+ struct scalar *s;
+ bool is_ptr;
+ u64 last;
+ u32 i;
+
+ for (i = steps->jump_cnt; i < steps->cnt; i++) {
+ op = steps->at[i].op;
+ s = origin_bounds(env, st, op, &is_ptr);
+ if (!s || (is_ptr && op->low32) ||
+ type_is_pkt_pointer(container_of(&s->var_off, struct bpf_reg_state,
+ var_off)->type))
+ return false;
+ move_bounds(env, s, is_ptr, op->step);
+ last = s->r64.base + s->r64.size;
+ if (op->low32 && !s->r64.size)
+ scalar_set_const(s, extend_low32(last, op->sext));
+ else if (op->low32 &&
+ (s->r64.size > U32_MAX || extend_low32(last, op->sext) != last ||
+ extend_low32(s->r64.base, op->sext) != s->r64.base))
+ return false;
+ xreg_eq = bpf_find_xreg_equation(&st->equations,
+ bpf_xreg(st->curframe, op->xregno));
+ if (xreg_eq)
+ xreg_eq->off += op->step;
+ }
+ bpf_top_loop(st->frame[st->curframe])->trip++;
+ bpf_equations_next_trip(env, st, st->frame[st->curframe]->loop_stack_cnt - 1);
+ return true;
+}
+
+/*
+ * @st starts the leftover of @loop. Two trips show the steps and the thresholds. If the code
+ * shows them, join @st with what these trips possibly give, and check the body once:
+ *
+ * i = 0 -> i = 0, 150 -> i = 0..450, i = 150 * k
+ *
+ * A wrong guess can cause an error or may_goto. Then it is not used again.
+ * Returns 1 if @st grew.
+ */
+int bpf_guess_first_trips(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ struct bpf_loop *loop)
+{
+ struct loop_stack_entry *e = bpf_top_loop(st->frame[st->curframe]);
+ struct bpf_verifier_state next = {};
+ struct bpf_loop_steps *steps;
+ u64 trips = U64_MAX, xreg;
+ struct scalar *s;
+ bool is_ptr;
+ int err;
+ u32 i;
+
+ if (!env->eq_env || !env->bypass_spec_v1 || bpf_prog_is_offloaded(env->prog->aux) ||
+ st->speculative || st->equations.infeasible || e->loop != loop ||
+ loop->no_first_trips_guess || loop->irreducible || loop->backedges_overflow ||
+ e->trip_index.r64.size)
+ return 0;
+ err = loop->steps ? 0 : find_steps(env, loop);
+ if (err)
+ return err;
+ steps = loop->steps;
+ if (bpf_xregs_are_partitioned(env, st, st->curframe, steps->changed_xregs))
+ return 0;
+ /* only two trips show how a sum goes with the ivs, if its start is known */
+ for_each_set_bit(i, steps->sum_xregs, MAX_BPF_XREG) {
+ xreg = bpf_xreg(st->curframe, i);
+ s = bpf_find_var_bounds(env, st, xreg, &is_ptr);
+ if (s && (s->r32.size != U32_MAX || bpf_find_xreg_equation(&st->equations, xreg)))
+ return 0;
+ }
+ for (i = 0; i < steps->jump_cnt; i++)
+ trips = min(trips, trips_by_steps(env, st, loop, steps->at[i].insn_idx,
+ steps->at[i].op));
+ if (!trips || trips > BPF_MAX_TRIPS)
+ return 0;
+ e->exit_threshold = e->trip_index.r64.base + trips;
+ add_iv_threshold(env, loop, e->exit_threshold, 1, THRESHOLD_EXIT_JUMP, U32_MAX);
+ for (i = 0; i < min(trips, 2) && !err; i++) {
+ err = bpf_copy_verifier_state(env, &next, st);
+ if (err || !move_by_steps(env, &next, steps))
+ break;
+ err = bpf_states_join(env, st, &next, loop, true);
+ env->extra_values = true;
+ bitmap_or(loop->nonlinear_xregs, loop->nonlinear_xregs, steps->nonlinear_xregs,
+ MAX_BPF_XREG);
+ }
+ bpf_free_verifier_state(env, &next, false);
+ return err ?: i > 0;
+}
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 3bec52051a9..eed9d13f6a6 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -755,6 +755,7 @@ static void free_loop(struct bpf_loop *loop)
return;
kvfree(loop->thresholds);
kvfree(loop->iv_thresholds);
+ kvfree(loop->steps);
kvfree(loop);
}
diff --git a/kernel/bpf/flow.c b/kernel/bpf/flow.c
index ffae6cf95f9..c31c05058b8 100644
--- a/kernel/bpf/flow.c
+++ b/kernel/bpf/flow.c
@@ -391,6 +391,7 @@ int bpf_flow_needs_guard(struct bpf_verifier_env *env, struct bpf_loop *loop)
{
if (loop->rt_bounded)
return 0;
+ loop->no_first_trips_guess = true;
if (!loop->has_guards) {
verbose(env, "loop at insn %d never exits\n", loop->header);
return -EINVAL;
@@ -1259,6 +1260,11 @@ static int flow_prune_or_join(struct bpf_verifier_env *env, struct flow_state *s
joined = false;
if (log && joined)
verbose(env, "from %d to %d: %s\n", prev, insn_idx, widened ? "widened" : "joined");
+ err = enters && !widened ? bpf_guess_first_trips(env, &s->st, loop) : 0;
+ if (err < 0)
+ goto free;
+ if (err)
+ flow_find(env, s);
flow_link(env, s);
return 0;
free:
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index cafa7805295..e23df42359c 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -84,6 +84,21 @@ bool bpf_call_needs_frames(struct bpf_verifier_env *env, const struct bpf_verifi
return false;
}
+/* Did a partition ask for the value or the type of one of @xregs in frame @fr? */
+bool bpf_xregs_are_partitioned(struct bpf_verifier_env *env, const struct bpf_verifier_state *st,
+ u32 fr, const unsigned long *xregs)
+{
+ unsigned long types[XREG_WORDS], control_deps[XREG_WORDS];
+ u32 bits;
+
+ if (!env->precision->data_deps)
+ return false;
+ bits = bpf_frame_mask(env, env->precision->type_apart, st, fr, types);
+ bpf_frame_mask(env, env->precision->control_deps, st, fr, control_deps);
+ return bitmap_intersects(xregs, types, bits) ||
+ bitmap_intersects(xregs, control_deps, bits);
+}
+
/* The value in a slot. NULL if the slot has bytes of several types. */
static const struct bpf_reg_state *slot_value(struct bpf_verifier_env *env,
const struct bpf_func_state *f, u32 spi,
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 41/49] bpf: Refine the copies of a number that AND or OR splits
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (39 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 40/49] bpf: Check the body of a counted loop once Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 42/49] bpf: Allow a range of offsets for a constant string Alexei Starovoitov
` (7 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
maybe_fork_scalars() splits the state when AND or OR reads a number
that is 0 or -1. The copies of the number are not split. The two states
forget that the copies go with the result:
w8 = 0 or -1
w2 = w8; w2 &= 255 state A: w2 = 255 state B: w2 = 0
if w2 > 9 goto out only B continues, w8 is still 0 or -1
r2 = r8; r2 &= 255 split again, a use of r2 = 255 fails
A split is a jump that is not in the program. Refine the related
numbers as after a jump: w8 is -1 in A and 0 in B.
Without bypass_spec_v1 a join does not relate two constants again. So
the copies are not refined there.
A program with this code in a loop goes from 11 passes to 1.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 2 ++
kernel/bpf/equation.c | 18 ++++++++++++++++++
kernel/bpf/verifier.c | 2 ++
3 files changed, 22 insertions(+)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 16eac538f43..5ceb141e4ae 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -2532,6 +2532,8 @@ void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_
void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
bool taken);
u16 bpf_args_with_equation(const struct bpf_verifier_state *st);
+void bpf_equations_fork(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct scalar *part);
void bpf_equations_next_trip(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
u32 depth);
bool bpf_equations_subsume(struct bpf_verifier_env *env, const struct bpf_verifier_state *old,
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index 4216fd00879..65f101c5e6e 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -1040,6 +1040,24 @@ void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state
refine_diff(env, st, &untruncated[0], &untruncated[1], op, true);
}
+/*
+ * The insn is checked in parts. In @st the operand that it overwrites is in @part.
+ * Without bypass_spec_v1 a join does not relate the constants again, see join_expr().
+ */
+void bpf_equations_fork(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ const struct scalar *part)
+{
+ struct bpf_equation_env *eq_env = env->eq_env;
+ struct scalar bounds = *part;
+
+ if (!eq_env || !env->bypass_spec_v1 || !eq_env->has_expr[0] ||
+ !eq_env->operand_expr[0].coef)
+ return;
+ scalar_bounds_sync(&bounds);
+ refine_sym(env, &st->equations, eq_env->operand_expr[0], bounds, 1);
+ propagate_bounds(env, st, false);
+}
+
/* The largest sum of two arguments of a call: numbers, or offsets of pointers. Modulo 2^64. */
s64 bpf_equations_max_sum(struct bpf_verifier_env *env, int regno1, int regno2)
{
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e29a72eba02..9aa6b331768 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -17619,6 +17619,8 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins
__mark_reg_known(reg, 0);
__mark_reg_known(dst_reg, -1ull);
}
+ bpf_equations_fork(env, branch, scalar_of(reg));
+ bpf_equations_fork(env, env->cur_state, scalar_of(dst_reg));
/*
* The insn is done with in the branch too, and what gets to the next insn is a state
* like any other. At this insn the branch is a part of the state that is looked at:
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 42/49] bpf: Allow a range of offsets for a constant string
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (40 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 41/49] bpf: Refine the copies of a number that AND or OR splits Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 43/49] bpf: Keep spilled pointers at a range of stack offsets Alexei Starovoitov
` (6 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
clang merges two calls that differ only in a string. The join of the
two pointers has a range. "is not a constant address" costs one more
pass.
The map is read-only. If a zero is at the largest offset or after it,
it ends all strings that start before it:
.rodata a % d 0 b c d 0
^ ^ ^
umin umax ends both
Allow the range. There are two exceptions:
- bpf_snprintf(): the verifier checks the format,
- no bypass_spec_v1: the cpu can speculate past the test of the offset.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
kernel/bpf/verifier.c | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9aa6b331768..c19eed8a6e0 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -9573,7 +9573,8 @@ static int check_arg_const_str(struct bpf_verifier_env *env,
return -EACCES;
}
- if (!tnum_is_const(reg->var_off)) {
+ /* the cpu can speculate past the test of a variable offset */
+ if (!tnum_is_const(reg->var_off) && !env->bypass_spec_v1) {
verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno));
return -EACCES;
}
@@ -9583,13 +9584,12 @@ static int check_arg_const_str(struct bpf_verifier_env *env,
return -EACCES;
}
- err = check_map_access(env, reg, argno, 0,
- map->value_size - reg->var_off.value, false,
- ACCESS_HELPER);
+ err = check_map_access(env, reg, argno, 0, 1, false, ACCESS_HELPER);
if (err)
return err;
- map_off = reg->var_off.value;
+ /* the zero of the last string ends all strings that start before it */
+ map_off = reg_umax(reg);
err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
if (err) {
verbose(env, "direct value access on string failed\n");
@@ -9602,7 +9602,8 @@ static int check_arg_const_str(struct bpf_verifier_env *env,
return -EINVAL;
}
/* the bytes of a pointer to a function are not known until the program is jitted */
- if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) {
+ map_off += strlen(str_ptr + map_off) + 1;
+ if (bpf_map_range_func_ptrs(env, map, reg_umin(reg), map_off - reg_umin(reg), &cnt)) {
verbose(env, "string overlaps with a pointer to a function\n");
return -EACCES;
}
@@ -12761,9 +12762,12 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
return -EINVAL;
num_args = data_len_reg->var_off.value / 8;
- /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
- * and map_direct_value_addr is set.
- */
+ if (!tnum_is_const(fmt_reg->var_off)) {
+ verbose(env, "R3 is not a constant address\n");
+ return -EACCES;
+ }
+
+ /* fmt being ARG_PTR_TO_CONST_STR guarantees that map_direct_value_addr is set */
fmt_map_off = fmt_reg->var_off.value;
err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
fmt_map_off);
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 43/49] bpf: Keep spilled pointers at a range of stack offsets
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (41 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 42/49] bpf: Allow a range of offsets for a constant string Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 44/49] selftests/bpf: Adjust tests to the new log text Alexei Starovoitov
` (5 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
An access to the stack with a range of offsets forgets the pointers
that are spilled in the range. The access is proven, and a later use of
the pointer is unproven. Two cases need no such loss.
1. A call writes at one of several offsets:
*(u64 *)(r10 - 16) = ptr
r1 = fp-24 or fp-8 the join is fp-24..fp-8, step 16
r2 = 8
call bpf_probe_read_kernel it does not touch fp-16
Skip the bytes between two offsets. The size has to be a constant,
see size_at_max_offset().
2. A load reads one element of an array of pointers:
char *p[2] = { a, b };
bpf_for(i, 0, 2)
sum += p[i][63];
i is 0..1 in each trip. The range comes from a load and a test, not
from a join. So it does not help to keep states apart.
fork_stack_load() does the load once for each slot:
r1 = *(u64 *)(r2 + 0)
r2 = fp-16 or fp-8 --+--> r2 = fp-16, r1 = a
+--> r2 = fp-8, r1 = b
The equations refine the related numbers: i is 0 in the first state
and 1 in the second one.
A program can also use such a word as a number. 'sum += word[i]'
fails if the word is a pointer. So a load forks only if its result
flows into an insn that needed a pointer and got a number. ADD_DEPS
finds these loads with one more backward slice. The slice follows
both operands of an add. So if a program adds such a word to such a
pointer, both loads fork, and the add can fail.
The cpu can speculate with the address of another state, so the fork
needs bypass_spec_v1.
insns processed
program bpf-next without with
wprof wprof_task_switch 32379 123446 79727
tetragon generic_lsm_process_event 114211 64030 31347
tracee trace_ret_vfs_write 5418 9413 3495
bpf_flow flow_dissector_0 715 2235 701
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
include/linux/bpf_verifier.h | 5 ++
kernel/bpf/equation.c | 10 ++++
kernel/bpf/precision.c | 60 ++++++++++++++++++++++++
kernel/bpf/precision.h | 4 ++
kernel/bpf/verifier.c | 90 +++++++++++++++++++++++++++++++++---
5 files changed, 163 insertions(+), 6 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 5ceb141e4ae..1b6643beed6 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1038,6 +1038,7 @@ struct bpf_insn_aux_data {
* are not in @use_regs. The bits above r10 are not valid.
*/
u16 use_regs, def_regs;
+ u16 needs_ptr_regs; /* see bpf_note_needs_ptr() */
/*
* Bitmask of R0-R9 that hold known values at this instruction.
* const_reg_mask: scalar constants that fit in 32 bits.
@@ -1761,6 +1762,8 @@ void bpf_precision_new_pass(struct bpf_verifier_env *env);
u64 bpf_pass_max_cost(struct bpf_verifier_env *env);
u64 bpf_precision_budget(struct bpf_verifier_env *env);
u64 bpf_precision_first_budget(struct bpf_verifier_env *env);
+void bpf_note_needs_ptr(struct bpf_verifier_env *env, const struct bpf_reg_state *reg);
+bool bpf_insn_makes_ptr(struct bpf_verifier_env *env);
bool bpf_failing_path_is_due(struct bpf_verifier_env *env);
int bpf_failing_path_prepare(struct bpf_verifier_env *env, bool repeat);
bool bpf_failing_path_may_end_at(struct bpf_verifier_env *env, u32 idx);
@@ -2529,6 +2532,8 @@ void bpf_equations_spill(struct bpf_verifier_env *env, struct bpf_func_state *fr
void bpf_equations_fill(struct bpf_verifier_env *env, struct bpf_func_state *frame, int spi,
int size);
void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
+void bpf_equations_refine_by_reg(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ u32 regno);
void bpf_equations_jump(struct bpf_verifier_env *env, struct bpf_verifier_state *st, int idx,
bool taken);
u16 bpf_args_with_equation(const struct bpf_verifier_state *st);
diff --git a/kernel/bpf/equation.c b/kernel/bpf/equation.c
index 65f101c5e6e..5d644e9c3eb 100644
--- a/kernel/bpf/equation.c
+++ b/kernel/bpf/equation.c
@@ -875,6 +875,16 @@ static void refine_from_reg(struct bpf_verifier_env *env, struct bpf_verifier_st
eq_env->jump_expr.coef = 0;
}
+/* @regno has fewer values now */
+void bpf_equations_refine_by_reg(struct bpf_verifier_env *env, struct bpf_verifier_state *st,
+ u32 regno)
+{
+ if (!env->eq_env)
+ return;
+ refine_from_reg(env, st, regno);
+ propagate_bounds(env, st, false);
+}
+
/* Apply all equations */
void bpf_equations_refine_all(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
{
diff --git a/kernel/bpf/precision.c b/kernel/bpf/precision.c
index e23df42359c..a130b30bf43 100644
--- a/kernel/bpf/precision.c
+++ b/kernel/bpf/precision.c
@@ -749,6 +749,14 @@ static u16 compute_call_deps(struct bpf_verifier_env *env, unsigned long *masks,
(GENMASK(BPF_REG_5, BPF_REG_1) | BIT(XREG_STACK_ARGS));
}
+static bool may_keep_ptr(const struct bpf_insn *insn)
+{
+ u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
+
+ return class != BPF_ALU &&
+ (class != BPF_ALU64 || op == BPF_MOV || op == BPF_ADD || op == BPF_SUB);
+}
+
/* What the insn at @idx of function @s needs before it. @in has what is needed after it. */
static bool compute_insn_deps(struct bpf_verifier_env *env, unsigned long *masks, u8 *ret, int s,
int idx, bool is_seed, unsigned long *in, bool held)
@@ -760,6 +768,7 @@ static bool compute_insn_deps(struct bpf_verifier_env *env, unsigned long *masks
bool changed = false, stores, reads, loads, call, held_after;
struct taken_funcs *taken = &env->precision->taken;
u8 class = BPF_CLASS(insn->code), needs;
+ u16 ptrs = 0;
int t, callee;
call = class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL;
@@ -785,6 +794,11 @@ static bool compute_insn_deps(struct bpf_verifier_env *env, unsigned long *masks
test_bit(idx, env->precision->last_pass.bad_read_insns));
if (!loads && class == BPF_STX)
use = BIT(insn->dst_reg);
+ if (env->precision->ptr_deps_only) {
+ reads &= may_keep_ptr(insn);
+ ptrs = is_seed ? aux->needs_ptr_regs : 0;
+ is_seed = false;
+ }
reads |= is_seed;
in[0] &= ~(unsigned long)(def & XREG_REGS);
for (t = 0; t < BITS_TO_LONGS(bits); t++)
@@ -829,6 +843,7 @@ static bool compute_insn_deps(struct bpf_verifier_env *env, unsigned long *masks
if (held)
changed |= !__test_and_set_bit(idx, env->precision->held_insns);
in[0] |= args | (reads ? use & XREG_REGS : 0);
+ in[0] |= ptrs;
if (reads && (call || is_stack_arg_ldx(insn)))
__set_bit(XREG_STACK_ARGS, in);
if ((reads && loads) || (needs & RET_LOADS)) {
@@ -1125,6 +1140,43 @@ static int compute_deps_of_unproven(struct bpf_verifier_env *env)
return add_control_deps(env);
}
+/* The current insn needs a pointer in @reg */
+void bpf_note_needs_ptr(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
+{
+ const struct bpf_reg_state *regs = cur_regs(env);
+
+ if (reg->type != SCALAR_VALUE || reg->from_unproven || env->path_sensitive ||
+ reg < regs || reg >= regs + BPF_REG_FP)
+ return;
+ __set_bit(env->insn_idx, env->precision->needs_ptr_insns);
+ env->insn_aux_data[env->insn_idx].needs_ptr_regs |= BIT(reg - regs);
+}
+
+/* The current insn can load a pointer. Does a later insn need it? */
+bool bpf_insn_makes_ptr(struct bpf_verifier_env *env)
+{
+ env->precision->this_pass.can_fork = true;
+ return test_bit(env->insn_idx, env->precision->makes_ptr_insns);
+}
+
+/* See fork_stack_load(). Returns 1 if there are more such insns. */
+static int add_makes_ptr_insns(struct bpf_verifier_env *env)
+{
+ struct bpf_precision *prec = env->precision;
+ u32 len = env->prog->len;
+ int err;
+
+ if (!prec->last_pass.can_fork || bitmap_empty(prec->needs_ptr_insns, len))
+ return 0;
+ prec->ptr_deps_only = true;
+ err = bpf_compute_deps(env, prec->new_data_deps, prec->needs_ptr_insns);
+ prec->ptr_deps_only = false;
+ if (err || bitmap_subset(prec->controlled_insns, prec->makes_ptr_insns, len))
+ return err;
+ bitmap_or(prec->makes_ptr_insns, prec->makes_ptr_insns, prec->controlled_insns, len);
+ return 1;
+}
+
/* An unreadable xreg needs one type and object, whatever the numbers are. Try that first. */
static int add_deps(struct bpf_verifier_env *env)
{
@@ -1143,6 +1195,10 @@ static int add_deps(struct bpf_verifier_env *env)
prec->last_pass.bad_read_insns, len);
bitmap_or(prec->bad_read_insns_done, prec->bad_read_insns_done,
prec->last_pass.bad_read_insns, len);
+ ret = add_makes_ptr_insns(env);
+ if (ret < 0)
+ return ret;
+ more |= ret;
ret = compute_deps_of_unproven(env);
for (i = 0; i < 5 * prec->words; i++)
prec->kept[i] |= prec->data_deps[i] & ~prec->undo[i];
@@ -1620,6 +1676,8 @@ void bpf_precision_new_pass(struct bpf_verifier_env *env)
bitmap_zero(prec->this_pass.unproven_funcs, env->subprog_cnt);
prec->this_pass.suspect_loops = prec->this_pass.first_suspect_loops =
prec->this_pass.disjoint_joined = prec->this_pass.arg_lost_equation = 0;
+ prec->this_pass.can_fork = false;
+ bitmap_zero(prec->needs_ptr_insns, env->prog->len);
if (prec->apart_xregno)
memset(prec->apart_xregno, 0, env->prog->len * sizeof(u16));
if (prec->data_deps && !env->replay && (prec->budget_grew || env->for_log)) {
@@ -1690,6 +1748,8 @@ int bpf_precision_init(struct bpf_verifier_env *env)
prec->bad_value_insns = bits + 15 * words;
prec->this_pass.unproven_funcs = bits + 16 * words;
prec->last_pass.unproven_funcs = bits + 17 * words;
+ prec->needs_ptr_insns = bits + 18 * words;
+ prec->makes_ptr_insns = bits + 19 * words;
prec->pass_cost = env->cost;
prec->live_types_cost = env->max_cost / LIVE_TYPES_PART;
prec->live_types_mem = env->max_mem / LIVE_TYPES_PART;
diff --git a/kernel/bpf/precision.h b/kernel/bpf/precision.h
index 3af6f10eed7..295c744cfd7 100644
--- a/kernel/bpf/precision.h
+++ b/kernel/bpf/precision.h
@@ -24,6 +24,7 @@ struct precision_per_pass {
bool disjoint_joined; /* disjoint values in data_deps were joined */
bool over_budget, calls_joined;
bool arg_lost_equation; /* see bpf_call_needs_frames() */
+ bool can_fork; /* see fork_stack_load() */
};
/* callx calls one of the functions whose address is taken. They count as one function. */
@@ -54,6 +55,8 @@ struct bpf_precision {
unsigned long *last_unproven_insns; /* in the last pass */
unsigned long *bad_read_insns_done;
unsigned long *earlier_unproven_insns; /* in an earlier pass */
+ unsigned long *needs_ptr_insns; /* got a number in place of a pointer */
+ unsigned long *makes_ptr_insns; /* the result flows into needs_ptr_insns */
unsigned long *control_seeds, *controlled_insns, *controlled_insns_done, *def_insns;
u32 first_unproven_insn; /* of the last pass */
u32 last_unproven_left; /* last_unproven_insns whose check did not fail again */
@@ -67,6 +70,7 @@ struct bpf_precision {
bool failing_path_due; /* see bpf_failing_path_is_due() */
bool budget_grew; /* see partition_next() */
bool new_nospec;
+ bool ptr_deps_only; /* see add_makes_ptr_insns() */
/* see bpf_hash_live_types() */
u64 live_types_cost, live_types_mem, start_mem;
bool live_types_over, join_live_types; /* in this pass, in all next passes */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c19eed8a6e0..b8a944984b7 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -4717,6 +4717,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
reg_type_str(env, reg->type), reg_name);
verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
+ bpf_note_needs_ptr(env, reg);
if (kptr_field->type == BPF_KPTR_UNREF)
verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
targ_name);
@@ -7343,8 +7344,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
return size;
err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
- if (err)
+ if (err) {
+ bpf_note_needs_ptr(env, reg);
return err;
+ }
if (reg->type == PTR_TO_MAP_KEY) {
if (t == BPF_WRITE) {
@@ -7557,6 +7560,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),
reg_type_str(env, reg->type));
+ bpf_note_needs_ptr(env, reg);
if (reg->type == SCALAR_VALUE)
kind = BPF_DIAG_DEREF_SCALAR;
else if (type_may_be_null(reg->type))
@@ -7876,8 +7880,9 @@ static int check_stack_range_initialized(
u32 arg_slot = arg_slot_from_argno(argno);
bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS &&
(meta->arg_raw_mem.mask & BIT(arg_slot));
- bool raw_mode = uninit && env->allow_uninit_stack &&
- !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot));
+ bool var_size = meta && arg_slot < MAX_BPF_FUNC_ARGS &&
+ (meta->arg_raw_mem.var_size_mask & BIT(arg_slot));
+ bool raw_mode = uninit && env->allow_uninit_stack && !var_size;
access_size = abs(access_size);
@@ -7926,6 +7931,10 @@ static int check_stack_range_initialized(
struct bpf_stack_state *ss;
u8 *stype;
+ /* The byte is between two accesses. See size_at_max_offset() for @var_size. */
+ if (reg->step > 1 && !var_size &&
+ imod(i - off - reg->base, reg->step) >= access_size)
+ continue;
slot = -i - 1;
spi = slot / BPF_REG_SIZE;
if (state->allocated_stack <= slot) {
@@ -8077,6 +8086,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_
verbose(env, "%s type=%s ", reg_arg_name(env, argno),
reg_type_str(env, reg->type));
verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
+ bpf_note_needs_ptr(env, reg);
return -EACCES;
}
}
@@ -9432,6 +9442,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re
for (j = 0; j + 1 < i; j++)
verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
+ bpf_note_needs_ptr(env, reg);
actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type));
accepted = bpf_diag_expected_reg_types(env, compatible->types, i);
bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name,
@@ -12064,6 +12075,7 @@ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn,
reg = reg_state(env, insn->dst_reg);
if (reg->type != PTR_TO_FUNC) {
+ bpf_note_needs_ptr(env, reg);
verbose(env, "R%d has type %s, expected func\n", insn->dst_reg,
reg_type_str(env, reg->type));
reason = bpf_diag_fmt(
@@ -21579,6 +21591,74 @@ static void free_pushed_states(struct bpf_verifier_env *env)
}
}
+/*
+ * A load with a range of offsets makes a number of the pointers that are spilled there. If an
+ * insn needed a pointer and got that number, load each slot in its own state:
+ * char *p[2] = { a, b };
+ * return p[i & 1][63];
+ * Other loads keep the number: 'sum += word[i]' fails for a pointer.
+ */
+static int fork_stack_load(struct bpf_verifier_env *env, const struct bpf_insn *insn,
+ unsigned long ranges)
+{
+ struct bpf_verifier_state *st = env->cur_state, *part;
+ DECLARE_BITMAP(slots, MAX_BPF_XREG) = {};
+ u32 regno = insn->src_reg, xregno, last;
+ struct bpf_stack_state *slot;
+ struct bpf_func_state *frame;
+ struct bpf_reg_state *reg;
+ struct scalar value = {};
+ bool has_ptr = false, needs_ptr;
+ s64 min_off, max_off;
+ int spi;
+
+ /* the cpu can speculate with the address of another state */
+ if (insn->code != (BPF_LDX | BPF_MEM | BPF_DW) || !(ranges & BIT(regno)) ||
+ cur_regs(env)[regno].type != PTR_TO_STACK || !env->bypass_spec_v1)
+ return 0;
+ /* without allow_ptr_leaks the load fails */
+ needs_ptr = !env->allow_ptr_leaks || bpf_insn_makes_ptr(env);
+ reg = &cur_regs(env)[regno];
+ frame = bpf_func(env, reg);
+ min_off = reg_smin(reg) + insn->off;
+ max_off = reg_smax(reg) + insn->off;
+ if (offset_is_far(reg) || min_off < -frame->allocated_stack || max_off >= 0 ||
+ !tnum_is_aligned(tnum_add(reg->var_off, tnum_const(insn->off)), BPF_REG_SIZE))
+ return 0;
+ bpf_add_cost(env, (max_off - min_off) / BPF_REG_SIZE, BPF_COST_XREG);
+ for (spi = bpf_get_spi(max_off); spi <= bpf_get_spi(min_off); spi++) {
+ scalar_set_const(&value, -(spi + 1) * BPF_REG_SIZE - insn->off);
+ if (!scalar_subsumes(scalar_of(reg), &value))
+ continue;
+ slot = bpf_stack_slot(frame, spi);
+ __set_bit(bpf_spi_to_xreg(spi), slots);
+ has_ptr |= bpf_is_spilled_reg(slot) && !bpf_is_spilled_scalar_reg(slot);
+ }
+ if (!has_ptr)
+ return 0;
+ env->imprecise |= !env->err;
+ if (!needs_ptr)
+ return 0;
+ /* no run has to load from all slots */
+ env->extra_values = true;
+ if (env->log.level & BPF_LOG_LEVEL)
+ verbose(env, "fork R%d\n", regno);
+ mark_reg_scratched(env, regno);
+ last = find_last_bit(slots, MAX_BPF_XREG);
+ for_each_set_bit(xregno, slots, MAX_BPF_XREG) {
+ part = xregno == last ? st : push_stack(env, env->insn_idx, env->insn_idx, false);
+ if (IS_ERR(part))
+ return PTR_ERR(part);
+ spi = bpf_xreg_to_spi(xregno);
+ scalar_set_const(scalar_of(&part->frame[part->curframe]->regs[regno]),
+ -(spi + 1) * BPF_REG_SIZE - insn->off);
+ bpf_equations_refine_by_reg(env, part, regno);
+ if (part != st)
+ env->head->again = true;
+ }
+ return 0;
+}
+
/* Process the current state until it gets to the start of a basic block. */
static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
{
@@ -21709,10 +21789,8 @@ static int do_check_block(struct bpf_verifier_env *env, int prev_insn_idx)
ranges = operands_with_range(env, insn);
stack_args = cur_func(env)->out_stack_arg_cnt;
- if (err)
- return err;
-
err = check_loop_guard(env) ?: check_recursion_guard(env);
+ err = err ?: fork_stack_load(env, insn, ranges);
if (err)
return err;
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 44/49] selftests/bpf: Adjust tests to the new log text
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (42 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 43/49] bpf: Keep spilled pointers at a range of stack offsets Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 45/49] selftests/bpf: Adjust tests that get another verdict or may_goto Alexei Starovoitov
` (4 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
verifier_scev.c tests trip counts and ranges in loops. Rename it to
verifier_trip_count.c. All programs stay. The verdicts do not change.
old text new text
id=2 id={{[0-9]+}}, ids are renumbered
from 3 to 7: within from 3 to 7: pruned
header_count is 5 loop at 3 has trip count 5
can't compute iterations count loop at 3 has no trip count
widening r7 to 0..99 the state at the header, see RANGE()
r6=(linear r6 2), can't widen r7 no such text
exit from 5 to 7 first exit to 7
processed 48 insns another number, or a pattern
math between ... is not allowed the access through the far pointer
fails, not the add
R1 has pointer with unsupported gone with REASON_*, unprivileged
alu operation loads get the error of root
first error in the order the error on the failing path
cannot return stack pointer R0 !read_ok in the caller
Some ranges are smaller, so an error has another offset or text. Some
results keep a step.
Some loops get may_goto, or lose it:
- NO_COUNT now: 'w6 += -n' wraps in each trip (post32_gt, _ge, _ne,
_sgt, _sge, backedges_end_then_wrap), the first step can wrap
(range_first_wraps_s32), or the counter starts with any value
(uninit_slot_counter, range_lt_dir_any_first,
no_widen_small_spill_counter).
- COUNT() now: irreducible loops, 17 back edges, 301 exits in a nested
loop, back edges that test different registers.
A jump on r9 gives a trip count now. LIMIT_ITERATIONS hides r9 behind
an atomic add, so the count comes from the tested jump. Three pre_lt
tests move to the new largest count.
A few tests change an insn to keep the error that they are about.
Two tests of a double release keep NULL away from the second release.
shift_with_64_bit_input needs an access through the far pointer, else
it loads. variable_subtraction loads a second number. Five tests in
verifier_spill_fill.c make STACK_MISC with other stores: a spill keeps
its low bytes now. dynptr_slice_var_len2 uses 29 sizes: the call is
safe for each of 15, and then a later store is the error.
aggregate_ret_static_ptr_fail reads through r2, else it loads.
var_off_insn_off_test1 and _test2 keep NULL away from the access.
more_back_edge was safe, it tests a new number in each trip now.
guard_at_head_dead_slot was safe and loads now: the slot is live at the
header. A larger shift makes it fail with another error.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
.../selftests/bpf/prog_tests/log_fixup.c | 6 +-
.../selftests/bpf/prog_tests/spin_lock.c | 12 +-
.../selftests/bpf/prog_tests/verifier.c | 4 +-
.../selftests/bpf/progs/aggregate_ret_func.c | 10 +-
.../bpf/progs/compute_live_registers.c | 10 +-
.../testing/selftests/bpf/progs/dynptr_fail.c | 6 +-
tools/testing/selftests/bpf/progs/iters.c | 4 +-
.../selftests/bpf/progs/iters_state_safety.c | 6 +-
.../selftests/bpf/progs/iters_testmod_seq.c | 2 +-
.../selftests/bpf/progs/map_kptr_fail.c | 2 +-
.../testing/selftests/bpf/progs/rbtree_fail.c | 2 +-
.../bpf/progs/refcounted_kptr_fail.c | 4 +-
.../struct_ops_refcounted_fail__ref_leak.c | 2 +-
tools/testing/selftests/bpf/progs/timer.c | 5 +-
.../selftests/bpf/progs/timer_failure.c | 6 +-
.../selftests/bpf/progs/verifier_align.c | 12 +-
.../selftests/bpf/progs/verifier_arena.c | 2 +-
.../bpf/progs/verifier_array_access.c | 2 +-
.../selftests/bpf/progs/verifier_bounds.c | 36 +-
.../bpf/progs/verifier_bounds_deduction.c | 14 +-
.../bpf/progs/verifier_bounds_step.c | 33 +-
.../selftests/bpf/progs/verifier_budget.c | 16 +-
.../bpf/progs/verifier_direct_packet_access.c | 6 +-
.../verifier_direct_stack_access_wraparound.c | 10 +-
.../bpf/progs/verifier_jumps_ahead.c | 2 +-
.../bpf/progs/verifier_linked_scalars.c | 2 +-
.../selftests/bpf/progs/verifier_live_stack.c | 10 +-
.../bpf/progs/verifier_liveness_exp.c | 4 +-
.../selftests/bpf/progs/verifier_loop_as_is.c | 23 +-
.../bpf/progs/verifier_loop_hierarchy.c | 13 +-
.../selftests/bpf/progs/verifier_loop_latch.c | 81 +-
.../selftests/bpf/progs/verifier_loop_need.c | 23 +-
.../selftests/bpf/progs/verifier_loop_walk.c | 2 +-
.../bpf/progs/verifier_meta_access.c | 2 +-
.../selftests/bpf/progs/verifier_precision.c | 2 +-
.../bpf/progs/verifier_ref_tracking.c | 7 +-
.../selftests/bpf/progs/verifier_scalar_ids.c | 12 +-
.../selftests/bpf/progs/verifier_sock.c | 4 +-
.../selftests/bpf/progs/verifier_spill_fill.c | 36 +-
.../selftests/bpf/progs/verifier_spin_lock.c | 2 +-
.../selftests/bpf/progs/verifier_stack_ptr.c | 14 +-
.../bpf/progs/verifier_subprog_callers.c | 2 +-
.../bpf/progs/verifier_subprog_precision.c | 8 +-
...{verifier_scev.c => verifier_trip_count.c} | 1216 ++++++-----------
.../selftests/bpf/progs/verifier_unpriv.c | 3 +-
tools/testing/selftests/bpf/verifier/calls.c | 2 +-
.../testing/selftests/bpf/verifier/map_kptr.c | 2 +-
47 files changed, 678 insertions(+), 1006 deletions(-)
rename tools/testing/selftests/bpf/progs/{verifier_scev.c => verifier_trip_count.c} (77%)
diff --git a/tools/testing/selftests/bpf/prog_tests/log_fixup.c b/tools/testing/selftests/bpf/prog_tests/log_fixup.c
index 89a7f594708..57c6751a0e4 100644
--- a/tools/testing/selftests/bpf/prog_tests/log_fixup.c
+++ b/tools/testing/selftests/bpf/prog_tests/log_fixup.c
@@ -41,7 +41,7 @@ static void bad_core_relo(size_t log_buf_size, enum trunc_type trunc_type)
"struct task_struct___bad.fake_field (0:1 @ offset 4)\n",
"log_buf_part2");
ASSERT_HAS_SUBSTR(log_buf,
- "max_states_per_insn 1 total_states 1 peak_states 1 mark_read 0\n",
+ "max_states_per_insn 1 total_states 2 peak_states 1 mark_read 0\n",
"log_buf_end");
break;
case TRUNC_PARTIAL:
@@ -50,7 +50,7 @@ static void bad_core_relo(size_t log_buf_size, enum trunc_type trunc_type)
"struct task_struct___bad.fake_field (0:1 @ offset 4)\n",
"log_buf_part2");
/* we shouldn't get full end of BPF verifier log */
- ASSERT_NULL(strstr(log_buf, "max_states_per_insn 1 total_states 1 peak_states 1 mark_read 0\n"),
+ ASSERT_NULL(strstr(log_buf, "max_states_per_insn 1 total_states 2 peak_states 1 mark_read 0\n"),
"log_buf_end");
break;
case TRUNC_FULL:
@@ -58,7 +58,7 @@ static void bad_core_relo(size_t log_buf_size, enum trunc_type trunc_type)
ASSERT_NULL(strstr(log_buf, "struct task_struct___bad.fake_field (0:1 @ offset 4)\n"),
"log_buf_part2");
/* we shouldn't get full end of BPF verifier log */
- ASSERT_NULL(strstr(log_buf, "max_states_per_insn 1 total_states 1 peak_states 1 mark_read 0\n"),
+ ASSERT_NULL(strstr(log_buf, "max_states_per_insn 1 total_states 2 peak_states 1 mark_read 0\n"),
"log_buf_end");
break;
}
diff --git a/tools/testing/selftests/bpf/prog_tests/spin_lock.c b/tools/testing/selftests/bpf/prog_tests/spin_lock.c
index e368370262c..99051ce543b 100644
--- a/tools/testing/selftests/bpf/prog_tests/spin_lock.c
+++ b/tools/testing/selftests/bpf/prog_tests/spin_lock.c
@@ -13,8 +13,8 @@ static struct {
const char *err_msg;
} spin_lock_fail_tests[] = {
{ "lock_id_kptr_preserve",
- "[0-9]\\+: (bf) r1 = r0 ; R0=ptr_foo(id=2)"
- " R1=ptr_foo(id=2) refs=2\n"
+ "[0-9]\\+: (bf) r1 = r0 ; R0=ptr_foo(id=\\([0-9]\\+\\))"
+ " R1=ptr_foo(id=\\1) refs=\\1\n"
"[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n"
"R1 type=ptr_ expected=percpu_ptr_" },
{ "lock_id_global_zero",
@@ -22,14 +22,14 @@ static struct {
"R1 type=map_value expected=percpu_ptr_" },
{ "lock_id_mapval_preserve",
"[0-9]\\+: (bf) r1 = r0 ;"
- " R0=map_value(id=1,map=array_map,ks=4,vs=8)"
- " R1=map_value(id=1,map=array_map,ks=4,vs=8)\n"
+ " R0=map_value(id=\\([0-9]\\+\\),map=array_map,ks=4,vs=8)"
+ " R1=map_value(id=\\1,map=array_map,ks=4,vs=8)\n"
"[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n"
"R1 type=map_value expected=percpu_ptr_" },
{ "lock_id_innermapval_preserve",
"[0-9]\\+: (bf) r1 = r0 ;"
- " R0=map_value(id=2,ks=4,vs=8)"
- " R1=map_value(id=2,ks=4,vs=8)\n"
+ " R0=map_value(id=\\([0-9]\\+\\),ks=4,vs=8)"
+ " R1=map_value(id=\\1,ks=4,vs=8)\n"
"[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n"
"R1 type=map_value expected=percpu_ptr_" },
{ "lock_id_mismatch_kptr_kptr", "bpf_spin_unlock of different lock" },
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 57ff2578a99..67151c55b03 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -127,7 +127,6 @@
#include "verifier_ringbuf.skel.h"
#include "verifier_runtime_jit.skel.h"
#include "verifier_scalar_ids.skel.h"
-#include "verifier_scev.skel.h"
#include "verifier_sdiv.skel.h"
#include "verifier_search_pruning.skel.h"
#include "verifier_sock.skel.h"
@@ -146,6 +145,7 @@
#include "verifier_subreg.skel.h"
#include "verifier_tailcall.skel.h"
#include "verifier_tailcall_jit.skel.h"
+#include "verifier_trip_count.skel.h"
#include "verifier_typedef.skel.h"
#include "verifier_uninit.skel.h"
#include "verifier_unpriv.skel.h"
@@ -350,7 +350,6 @@ void test_verifier_regalloc(void) { RUN(verifier_regalloc); }
void test_verifier_ringbuf(void) { RUN(verifier_ringbuf); }
void test_verifier_runtime_jit(void) { RUN(verifier_runtime_jit); }
void test_verifier_scalar_ids(void) { RUN(verifier_scalar_ids); }
-void test_verifier_scev(void) { RUN(verifier_scev); }
void test_verifier_sdiv(void) { RUN(verifier_sdiv); }
void test_verifier_search_pruning(void) { RUN(verifier_search_pruning); }
void test_verifier_sock(void) { RUN(verifier_sock); }
@@ -369,6 +368,7 @@ void test_verifier_subprog_topo(void) { RUN(verifier_subprog_topo); }
void test_verifier_subreg(void) { RUN(verifier_subreg); }
void test_verifier_tailcall(void) { RUN(verifier_tailcall); }
void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); }
+void test_verifier_trip_count(void) { RUN(verifier_trip_count); }
void test_verifier_typedef(void) { RUN(verifier_typedef); }
void test_verifier_uninit(void) { RUN(verifier_uninit); }
void test_verifier_unpriv(void) { RUN(verifier_unpriv); }
diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c
index 2671323255c..bb2f19f2b1a 100644
--- a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c
+++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c
@@ -70,18 +70,18 @@ static __naked __noinline u128 static_agg_bad_ptr(void)
}
/*
- * R2 is a return register once the subprogram returns a pair, so a stack
- * pointer left in it is rejected at the callee's exit exactly as one in R0
- * is: the callee frame is gone by the time the caller could use it.
+ * R2 is a return register once the subprogram returns a pair, so the caller
+ * cannot read a stack pointer left in it, exactly as one in R0: the callee
+ * frame is gone by the time the caller could use it.
*/
SEC("tc")
__load_if_JITed()
-__failure __msg("cannot return stack pointer to the caller")
+__failure __msg("R2 !read_ok")
__naked int aggregate_ret_static_ptr_fail(void)
{
asm volatile (
"call %[static_agg_bad_ptr];"
- "r0 = 0;"
+ "r0 = *(u8 *)(r2 - 1);"
"exit;"
:
: __imm(static_agg_bad_ptr)
diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c
index f5c505dfd46..f1a0c2187c6 100644
--- a/tools/testing/selftests/bpf/progs/compute_live_registers.c
+++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c
@@ -356,13 +356,13 @@ __naked void ldimm64(void)
:: __clobber_all);
}
-/* No rules specific for LD_ABS/LD_IND, default behaviour kicks in */
+/* LD_ABS/LD_IND read r6 and clobber r0-r5 */
SEC("socket")
__log_level(2)
-__msg("2: 0123456789 (30) r0 = *(u8 *)skb[42]")
-__msg("3: 012.456789 (0f) r7 += r0")
-__msg("4: 012.456789 (b7) r3 = 42")
-__msg("5: 0123456789 (50) r0 = *(u8 *)skb[r3 + 0]")
+__msg("2: ......67.. (30) r0 = *(u8 *)skb[42]")
+__msg("3: 0.....67.. (0f) r7 += r0")
+__msg("4: ......67.. (b7) r3 = 42")
+__msg("5: ...3..67.. (50) r0 = *(u8 *)skb[r3 + 0]")
__msg("6: 0......7.. (0f) r7 += r0")
__naked void ldabs(void)
{
diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c
index 2425e05a12a..232886dcfc5 100644
--- a/tools/testing/selftests/bpf/progs/dynptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c
@@ -91,7 +91,7 @@ int ringbuf_missing_release1(void *ctx)
}
SEC("?raw_tp")
-__failure __msg("Unreleased reference id=3")
+__failure __msg("Unreleased reference id=")
int ringbuf_missing_release2(void *ctx)
{
struct bpf_dynptr ptr1, ptr2;
@@ -782,7 +782,7 @@ int dynptr_pruning_overwrite(struct __sk_buff *ctx)
}
SEC("?tc")
-__success __msg("to 12: within") __log_level(2)
+__success __msg("to 12: pruned") __log_level(2)
int dynptr_pruning_stacksafe(struct __sk_buff *ctx)
{
asm volatile (
@@ -1879,7 +1879,7 @@ __failure __msg("must be a known constant")
__msg("requires this memory size to be a verifier-known constant")
int dynptr_slice_var_len2(struct __sk_buff *skb)
{
- char buffer[sizeof(struct ethhdr)] = {};
+ char buffer[2 * sizeof(struct ethhdr)] = {};
struct bpf_dynptr ptr;
struct ethhdr *hdr;
diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c
index b9677d30e69..ede1ee53839 100644
--- a/tools/testing/selftests/bpf/progs/iters.c
+++ b/tools/testing/selftests/bpf/progs/iters.c
@@ -47,7 +47,7 @@ int iter_err_unsafe_c_loop(const void *ctx)
}
SEC("?raw_tp")
-__failure __msg("math between map_value pointer and register with unbounded min value is not allowed")
+__failure __msg("R1 unbounded memory access")
int iter_err_unsafe_asm_loop(const void *ctx)
{
struct bpf_iter_num it;
@@ -1989,7 +1989,7 @@ static int loop1_wrapper(void)
*/
SEC("raw_tp")
/* -32 or -31 */
-__failure __msg("misaligned stack access off (0xffffffffffffffe0; 0x1)+0 size 8")
+__failure __msg("misaligned stack access off -31+0 size 8")
__naked void absent_mark_in_the_middle_state4(void)
{
/*
diff --git a/tools/testing/selftests/bpf/progs/iters_state_safety.c b/tools/testing/selftests/bpf/progs/iters_state_safety.c
index fa9604f0a17..52487631a52 100644
--- a/tools/testing/selftests/bpf/progs/iters_state_safety.c
+++ b/tools/testing/selftests/bpf/progs/iters_state_safety.c
@@ -30,7 +30,7 @@ int force_clang_to_emit_btf_for_externs(void *ctx)
SEC("?raw_tp")
__success __log_level(2)
-__msg("fp-8=iter_num(id=1,state=active)")
+__msg("fp-8=iter_num(id={{[0-9]+}},state=active)")
int create_and_destroy(void *ctx)
{
struct bpf_iter_num iter;
@@ -208,7 +208,7 @@ SEC("?raw_tp")
__failure
/* ensure there was a call to subprog, which might happen without __noinline */
__msg("0: (85) call pc+2")
-__msg("Unreleased reference id=2 alloc_insn=7")
+__msg("Unreleased reference id={{[0-9]+}} alloc_insn=7")
int leak_iter_from_subprog_fail(void *ctx)
{
subprog_with_iter();
@@ -218,7 +218,7 @@ int leak_iter_from_subprog_fail(void *ctx)
SEC("?raw_tp")
__success __log_level(2)
-__msg("fp-8=iter_num(id=1,state=active)")
+__msg("fp-8=iter_num(id={{[0-9]+}},state=active)")
int valid_stack_reuse(void *ctx)
{
struct bpf_iter_num iter;
diff --git a/tools/testing/selftests/bpf/progs/iters_testmod_seq.c b/tools/testing/selftests/bpf/progs/iters_testmod_seq.c
index 88ce4b7a9c3..1822aabdf0e 100644
--- a/tools/testing/selftests/bpf/progs/iters_testmod_seq.c
+++ b/tools/testing/selftests/bpf/progs/iters_testmod_seq.c
@@ -59,7 +59,7 @@ static volatile int zero = 0;
SEC("raw_tp/sys_enter")
__success __log_level(2)
__msg("fp-16=iter_testmod_seq(id=1,state=active)")
-__msg("fp-16=iter_testmod_seq(id=1,state=drained)")
+__msg("fp-16=iter_testmod_seq(id={{[0-9]+}},state=drained)")
__msg("call bpf_iter_testmod_seq_destroy")
int testmod_seq_truncated(const void *ctx)
{
diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c
index ac4003bfb8b..f3dfe08b365 100644
--- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c
@@ -345,7 +345,7 @@ int reject_indirect_global_func_access(struct __sk_buff *ctx)
}
SEC("?tc")
-__failure __msg("Unreleased reference id=4 alloc_insn=")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=")
int kptr_xchg_ref_state(struct __sk_buff *ctx)
{
struct prog_test_ref_kfunc *p;
diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c
index 08709f23ec0..a22d912d489 100644
--- a/tools/testing/selftests/bpf/progs/rbtree_fail.c
+++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c
@@ -107,7 +107,7 @@ long rbtree_api_remove_unadded_node(void *ctx)
}
SEC("?tc")
-__failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn={{[0-9]+}}")
long rbtree_api_remove_no_drop(void *ctx)
{
struct bpf_rb_node *res;
diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
index e80f78fae22..e2ff5db90e0 100644
--- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
@@ -76,7 +76,7 @@ static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
}
SEC("?tc")
-__failure __msg("Unreleased reference id=4 alloc_insn={{[0-9]+}}")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn={{[0-9]+}}")
long rbtree_refcounted_node_ref_escapes(void *ctx)
{
struct node_acquire *n, *m;
@@ -239,7 +239,7 @@ long graph_kptr_after_spin_unlock(void *ctx)
}
SEC("?tc")
-__failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn={{[0-9]+}}")
long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx)
{
struct node_acquire *n, *m;
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c b/tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c
index e945b1a0429..aae5e0dd984 100644
--- a/tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c
+++ b/tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c
@@ -9,7 +9,7 @@ char _license[] SEC("license") = "GPL";
* kptr through context without releasing the reference
*/
SEC("struct_ops/test_refcounted")
-__failure __msg("Unreleased reference id=1 alloc_insn=0")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=0")
int BPF_PROG(refcounted_fail__ref_leak, int dummy,
struct task_struct *task)
{
diff --git a/tools/testing/selftests/bpf/progs/timer.c b/tools/testing/selftests/bpf/progs/timer.c
index aa966dbed02..83a96aaf157 100644
--- a/tools/testing/selftests/bpf/progs/timer.c
+++ b/tools/testing/selftests/bpf/progs/timer.c
@@ -159,10 +159,7 @@ static __noinline int sum_to(__u64 n)
static int loop_cb(void *map, int *key, struct elem *val);
-/*
- * Re-arms from a frame below the callback's, where the caller frame carries
- * no async_entry_cnt of its own.
- */
+/* Re-arms from a function that the callback calls. */
static __noinline void rearm(struct elem *val)
{
if (loop_rearm > 0) {
diff --git a/tools/testing/selftests/bpf/progs/timer_failure.c b/tools/testing/selftests/bpf/progs/timer_failure.c
index 84ea861575f..d5fd85ccc1b 100644
--- a/tools/testing/selftests/bpf/progs/timer_failure.c
+++ b/tools/testing/selftests/bpf/progs/timer_failure.c
@@ -60,11 +60,7 @@ long BPF_PROG2(test_bad_ret, int, a)
return 0;
}
-/*
- * A real loop inside an async callback must still be rejected: both entries
- * into the callback have the same async_entry_cnt, so telling entries apart
- * does not apply here.
- */
+/* A loop with no exit is rejected in an async callback too. */
static int timer_cb_infinite_loop(void *map, int *key, struct elem *val)
{
for (;;) {}
diff --git a/tools/testing/selftests/bpf/progs/verifier_align.c b/tools/testing/selftests/bpf/progs/verifier_align.c
index aecc5279dba..9dbaec98a0d 100644
--- a/tools/testing/selftests/bpf/progs/verifier_align.c
+++ b/tools/testing/selftests/bpf/progs/verifier_align.c
@@ -247,7 +247,7 @@ __msg("7: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)")
/* Offset is added to packet pointer R5, resulting in
* known fixed offset, and variable offset from R6.
*/
-__msg("11: {{.*}} R5=pkt(id=1,{{[^)]*}},var_off=(0x2; 0x7fc)")
+__msg("11: {{.*}} R5=pkt(id={{[0-9]+}},{{[^)]*}},var_off=(0x2; 0x7fc)")
/* At the time the word size load is performed from R5,
* it's total offset is NET_IP_ALIGN + reg->off (0) +
* reg->aux_off (14) which is 16. Then the variable
@@ -266,7 +266,7 @@ __msg("18: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x3fc){{.*}} R5={{[^)]*}}var_off=(0x
/* Constant offset is added to R5, resulting in
* reg->off of 14.
*/
-__msg("19: {{.*}} R5=pkt(id=2,{{[^)]*}}var_off=(0x2; 0x7fc)")
+__msg("19: {{.*}} R5=pkt(id={{[0-9]+}},{{[^)]*}}var_off=(0x2; 0x7fc)")
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off
* (14) which is 16. Then the variable offset is 4-byte
@@ -284,7 +284,7 @@ __msg("26: {{.*}} R5=pkt(r=8,imm=14)")
*/
__msg("28: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc){{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)")
/* Constant is added to R5 again, setting reg->off to 18. */
-__msg("29: {{.*}} R5=pkt(id=3,{{[^)]*}}var_off=(0x2; 0xffc)")
+__msg("29: {{.*}} R5=pkt(id={{[0-9]+}},{{[^)]*}}var_off=(0x2; 0xffc)")
/* And once more we add a variable; resulting {{[^)]*}}var_off
* is still (4n), fixed offset is not changed.
* Also, we create a new reg->id.
@@ -432,8 +432,8 @@ __msg("9: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7ffffffffffffffc)")
/* packet pointer + nonnegative (4n+2) */
__msg("11: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7ffffffffffffffc){{.*}} R6={{[^)]*}}var_off=(0x2; 0x7ffffffffffffffc)")
__msg("12: (07) r4 += 4")
-/* packet smax bound overflow */
-__msg("pkt pointer offset -9223372036854775808 is not allowed")
+/* the offset can overflow, so the bounds check gives r6 no range */
+__msg("R6 min value is outside of the allowed memory range")
__naked void dubious_pointer_arithmetic(void)
{
asm volatile (" \
@@ -494,7 +494,7 @@ __naked void variable_subtraction(void)
asm volatile (" \
/* Create an unknown offset, (4n+2)-aligned */ \
" LOAD_UNKNOWN("r6") " \
- r7 = r6; \
+ r7 = *(u8*)(r2 + 1); \
r6 <<= 2; \
r6 += 14; \
/* Create another unknown, (4n)-aligned, and subtract\
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c
index 4741cd7d20c..99856ee1079 100644
--- a/tools/testing/selftests/bpf/progs/verifier_arena.c
+++ b/tools/testing/selftests/bpf/progs/verifier_arena.c
@@ -618,7 +618,7 @@ __naked void name(void) \
:: __imm_addr(arena) : __clobber_all); \
}
-#define NOT_UNROLLED100 __msg("loop header at 5, widening r7 to 0..99 step 1")
+#define NOT_UNROLLED100 __msg("to 5: {{.*}}R7=scalar(smin=smin32=0,smax=umax=smax32=umax32=99,")
/* A loop that moves the pointer is not unrolled */
ARENA_TEST(arena_pointer_moves, "r6 += 8;", __success NOT_UNROLLED100)
/* The loop makes a number of it: the state that returns to the loop header tells */
diff --git a/tools/testing/selftests/bpf/progs/verifier_array_access.c b/tools/testing/selftests/bpf/progs/verifier_array_access.c
index 29bed1e496a..d81c886a8bb 100644
--- a/tools/testing/selftests/bpf/progs/verifier_array_access.c
+++ b/tools/testing/selftests/bpf/progs/verifier_array_access.c
@@ -738,7 +738,7 @@ unsigned int a_pcpu_array_with_a_key_in_range_no_nullness(void)
return val->index;
}
-/* Scalar evolution gives r6 the range [0, 1] for all iterations */
+/* The trip count gives r6 the range [0, 1] for all iterations */
SEC("socket")
__description("valid map access into an array using the counter of a loop without nullness")
__success __retval(0)
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c
index 9dc23611332..8382ea850e2 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c
@@ -48,7 +48,7 @@ SEC("socket")
__description("subtraction bounds (map value) variant 2")
__failure
__msg("R0 min value is negative, either use unsigned index or do a if (index >=0) check.")
-__msg_unpriv("R1 has unknown scalar with mixed signed bounds")
+__failure_unpriv
__naked void bounds_map_value_variant_2(void)
{
asm volatile (" \
@@ -140,7 +140,7 @@ l0_%=: /* exit */ \
SEC("socket")
__description("bounds check based on sign-extended MOV. test1")
-__failure __msg("map_value pointer and 4294967295")
+__failure __msg("R0 min value is outside of the allowed memory range")
__failure_unpriv
__naked void on_sign_extended_mov_test1(void)
{
@@ -202,7 +202,7 @@ l0_%=: /* exit */ \
SEC("tc")
__description("bounds check based on reg_off + var_off + insn_off. test1")
-__failure __msg("map_value pointer offset 1073741822 is not allowed")
+__failure __msg("R0 min value is outside of the allowed memory range")
__naked void var_off_insn_off_test1(void)
{
asm volatile (" \
@@ -218,8 +218,8 @@ __naked void var_off_insn_off_test1(void)
r6 += %[__imm_0]; \
r0 += r6; \
r0 += %[__imm_0]; \
-l0_%=: r0 = *(u8*)(r0 + 3); \
- r0 = 0; \
+ r0 = *(u8*)(r0 + 3); \
+l0_%=: r0 = 0; \
exit; \
" :
: __imm(bpf_map_lookup_elem),
@@ -231,7 +231,7 @@ l0_%=: r0 = *(u8*)(r0 + 3); \
SEC("tc")
__description("bounds check based on reg_off + var_off + insn_off. test2")
-__failure __msg("value 1073741823")
+__failure __msg("R0 min value is outside of the allowed memory range")
__naked void var_off_insn_off_test2(void)
{
asm volatile (" \
@@ -247,8 +247,8 @@ __naked void var_off_insn_off_test2(void)
r6 += %[__imm_0]; \
r0 += r6; \
r0 += %[__imm_1]; \
-l0_%=: r0 = *(u8*)(r0 + 3); \
- r0 = 0; \
+ r0 = *(u8*)(r0 + 3); \
+l0_%=: r0 = 0; \
exit; \
" :
: __imm(bpf_map_lookup_elem),
@@ -484,7 +484,7 @@ l0_%=: /* exit */ \
SEC("socket")
__description("bounds check after 32-bit right shift with 64-bit input")
-__failure __msg("math between map_value pointer and 4294967294 is not allowed")
+__failure __msg("R0 min value is outside of the allowed memory range")
__failure_unpriv
__naked void shift_with_64_bit_input(void)
{
@@ -503,10 +503,10 @@ __naked void shift_with_64_bit_input(void)
w1 >>= 31; \
/* r1 = 0xffff'fffe (NOT 0!) */ \
w1 -= 2; \
- /* error on computing OOB pointer */ \
+ /* computes OOB pointer */ \
r0 += r1; \
- /* exit */ \
- r0 = 0; \
+ /* OOB access */ \
+ r0 = *(u8*)(r0 + 0); \
l0_%=: exit; \
" :
: __imm(bpf_map_lookup_elem),
@@ -516,7 +516,7 @@ l0_%=: exit; \
SEC("socket")
__description("bounds check map access with off+size signed 32bit overflow. test1")
-__failure __msg("map_value pointer and 2147483646")
+__failure __msg("R0 min value is outside of the allowed memory range")
__failure_unpriv
__naked void size_signed_32bit_overflow_test1(void)
{
@@ -541,7 +541,7 @@ l1_%=: exit; \
SEC("socket")
__description("bounds check map access with off+size signed 32bit overflow. test2")
-__failure __msg("pointer offset 1073741822")
+__failure __msg("R0 min value is outside of the allowed memory range")
__msg_unpriv("R0 pointer arithmetic of map value goes out of range")
__naked void size_signed_32bit_overflow_test2(void)
{
@@ -568,7 +568,7 @@ l1_%=: exit; \
SEC("socket")
__description("bounds check map access with off+size signed 32bit overflow. test3")
-__failure __msg("pointer offset -1073741822")
+__failure __msg("R0 min value is negative")
__msg_unpriv("R0 pointer arithmetic of map value goes out of range")
__naked void size_signed_32bit_overflow_test3(void)
{
@@ -594,7 +594,7 @@ l1_%=: exit; \
SEC("socket")
__description("bounds check map access with off+size signed 32bit overflow. test4")
-__failure __msg("map_value pointer and 1000000000000")
+__failure __msg("R0 min value is outside of the allowed memory range")
__failure_unpriv
__naked void size_signed_32bit_overflow_test4(void)
{
@@ -1572,7 +1572,7 @@ l0_%=: r0 = 0; \
SEC("socket")
__description("bounds deduction cross sign boundary, negative overlap")
__success __log_level(2)
-__msg("7: (1f) r0 -= r6 {{.*}} R0=scalar(smin=smin32=-527,smax=smax32=-146,umin=0xfffffffffffffdf1,umax=0xffffffffffffff6e,umin32=0xfffffdf1,umax32=0xffffff6e,var_off=(0xfffffffffffffc00; 0x3ff))")
+__msg("7: (1f) r0 -= r6 {{.*}} R0=scalar(smin=smin32=-400,smax=smax32=-273,umin=0xfffffffffffffe70,umax=0xfffffffffffffeef,umin32=0xfffffe70,umax32=0xfffffeef,var_off=(0xfffffffffffffe00; 0xff))")
__retval(0)
__naked void bounds_deduct_negative_overlap(void)
{
@@ -2286,7 +2286,7 @@ l0_%=: r0 = 0; \
}
/*
- * The insn that forks is the head of a loop that scalar evolution counted. The state with
+ * The insn that forks is the head of a loop with a trip count. The state with
* [-1, 0] is kept there, and the state with 0 that is split off is within it.
*/
SEC("socket")
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c b/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c
index a474c9978a3..14e10760606 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c
@@ -8,7 +8,7 @@
SEC("socket")
__description("check deducing bounds from const, 1")
__failure __msg("R0 tried to subtract pointer from scalar")
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__failure_unpriv
__naked void deducing_bounds_from_const_1(void)
{
asm volatile (" \
@@ -40,7 +40,7 @@ l1_%=: r1 -= r0; \
SEC("socket")
__description("check deducing bounds from const, 3")
__failure __msg("R0 tried to subtract pointer from scalar")
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__failure_unpriv
__naked void deducing_bounds_from_const_3(void)
{
asm volatile (" \
@@ -73,7 +73,7 @@ l1_%=: r6 -= r0; \
SEC("socket")
__description("check deducing bounds from const, 5")
__failure __msg("R0 tried to subtract pointer from scalar")
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__failure_unpriv
__naked void deducing_bounds_from_const_5(void)
{
asm volatile (" \
@@ -87,7 +87,7 @@ l0_%=: exit; \
SEC("socket")
__description("check deducing bounds from const, 6")
__failure __msg("R0 tried to subtract pointer from scalar")
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__failure_unpriv
__naked void deducing_bounds_from_const_6(void)
{
asm volatile (" \
@@ -102,7 +102,7 @@ l0_%=: r0 -= r1; \
SEC("socket")
__description("check deducing bounds from const, 7")
__failure __msg("dereference of modified ctx ptr")
-__msg_unpriv("dereference of modified ctx ptr")
+__failure_unpriv
__flag(BPF_F_ANY_ALIGNMENT)
__naked void deducing_bounds_from_const_7(void)
{
@@ -121,7 +121,7 @@ l0_%=: r1 -= r0; \
SEC("socket")
__description("check deducing bounds from const, 8")
__failure __msg("negative offset ctx ptr R1 off=-1 disallowed")
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__failure_unpriv
__flag(BPF_F_ANY_ALIGNMENT)
__naked void deducing_bounds_from_const_8(void)
{
@@ -140,7 +140,7 @@ l0_%=: r0 = *(u32*)(r1 + %[__sk_buff_mark]); \
SEC("socket")
__description("check deducing bounds from const, 9")
__failure __msg("R0 tried to subtract pointer from scalar")
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__failure_unpriv
__naked void deducing_bounds_from_const_9(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
index 2b8ff0f44c3..28bdc63f48d 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
@@ -38,7 +38,7 @@ __naked void step_arith64_no_overflow(void)
/*
* Safe ALU32 multiplication and addition preserve the stride and scale its base:
* step=0+3 -> step=1+3 -> step=2+6. MOV32 preserves the result.
- * LSH32 currently resets the stride, but must preserve the correct upper bound.
+ * LSH32 resets the stride, the equation of the result tells it again.
*/
SEC("socket")
__success __log_level(2)
@@ -46,7 +46,7 @@ __msg("w0 *= 3 {{.*}}step=0+3)")
__msg("w0 += 1 {{.*}}step=1+3)")
__msg("w0 *= 2 {{.*}}step=2+6)")
__msg("w1 = w0 {{.*}}R0=scalar({{.*}}step=2+6) R1=scalar({{.*}}step=2+6)")
-__msg("w0 <<= 1 {{.*}}smax=umax=smax32=umax32=3064,") __no_step
+__msg("w0 <<= 1 {{.*}}smax=umax=smax32=umax32=3064,{{.*}}step=4+12)")
__naked void step_arith32_no_overflow(void)
{
asm volatile (" \
@@ -93,13 +93,14 @@ __naked void step_neg_value_range(void)
* incorrectly narrows the result to 1.
* Shifting the same input left by 1 gives {0xfffffff8, 0xfffffffe};
* inferring step=0+6 incorrectly narrows the result to 0xfffffffc.
+ * The equation of a result tells the right base.
*/
SEC("socket")
__success __log_level(2)
__msg("w0 *= 7 {{.*}}umax=0xffffffff,") __no_step
__msg("r2 = r0 {{.*}}R0=scalar({{.*}}step=0+3) R2=scalar({{.*}}step=0+3)")
-__msg("w0 += 4 {{.*}}smax=umax=smax32=umax32=3,") __no_step
-__msg("w2 <<= 1 {{.*}}smax=umax=umax32=0xfffffffe,") __no_step
+__msg("w0 += 4 {{.*}}smax=umax=smax32=umax32=3,{{.*}}step=0+3)")
+__msg("w2 <<= 1 {{.*}}smax=umax=umax32=0xfffffffe,{{.*}}step=2+6)")
__naked void step_arith32_overflow(void)
{
asm volatile (" \
@@ -132,16 +133,18 @@ __naked void step_arith32_overflow(void)
*
* Shifting {0x4000000000000008, 0x400000000000000b} left by 1 gives
* {S64_MIN + 16, S64_MIN + 22}. Keeping step=0+6 excludes both.
+ *
+ * The equation of a result tells the right base.
*/
SEC("socket")
__success __log_level(2)
-__msg("r0 *= 3 {{.*}}smax=0x8000000000000004,") __no_step
+__msg("r0 *= 3 {{.*}}smax=0x8000000000000004,{{.*}}step=2+3)")
__msg("r0 += r1 {{.*}}step=0+3)")
-__msg("r0 += 5 {{.*}}smax=0x8000000000000003,") __no_step
+__msg("r0 += 5 {{.*}}smax=0x8000000000000003,{{.*}}step=1+3)")
__msg("r0 += r1 {{.*}}step=2+3)")
-__msg("r0 += -5 {{.*}}umax=0x7fffffffffffffff,") __no_step
+__msg("r0 += -5 {{.*}}umax=0x7fffffffffffffff,{{.*}}step=1+3)")
__msg("r0 += r1 {{.*}}step=0+3)")
-__msg("r0 <<= 1 {{.*}}smax=0x8000000000000016,") __no_step
+__msg("r0 <<= 1 {{.*}}smax=0x8000000000000016,{{.*}}step=2+6)")
__naked void step_arith64_overflow(void)
{
asm volatile (" \
@@ -231,11 +234,11 @@ __naked void step_mov32_negative(void)
SEC("socket")
__success __log_level(2)
__msg("r0 += 128 {{.*}}step=2+3)")
-__msg("r0 = (s8)r0 {{.*}}smax=smax32=-125,") __no_step
+__msg("r0 = (s8)r0 {{.*}}smax=smax32=-125,{{.*}}step=1+3)")
__msg("r0 += 32768 {{.*}}step=2+3)")
-__msg("r0 = (s16)r0 {{.*}}smax=smax32=-32765,") __no_step
+__msg("r0 = (s16)r0 {{.*}}smax=smax32=-32765,{{.*}}step=1+3)")
__msg("r0 += r1 {{.*}}step=2+3)")
-__msg("r0 = (s32)r0 {{.*}}smax=0xffffffff80000003,") __no_step
+__msg("r0 = (s32)r0 {{.*}}smax=0xffffffff80000003,{{.*}}step=1+3)")
__naked void step_movsx(void)
{
asm volatile (" \
@@ -329,18 +332,18 @@ l_out_%=: \
* r1 = (s16)r0 -> [0, 168] step=0+24
* r0 += -24 -> [-24, 144] step=0+24
* r2 = (s16)r0 -> [-24, 144] step=0+24
- * r3 = (s8)r0 -> [-128, 127] no step
+ * r3 = (s8)r0 -> [-128, 120] no step
* w4 = (s16)w1 -> [0, 168] step=0+24
- * w5 = (s8)w1 -> [-128, 127] no step
+ * w5 = (s8)w1 -> [-128, 120] no step
* Bounds and step stay in sync, the step is dropped when the value does not fit.
*/
SEC("socket")
__success __log_level(2)
__msg("r1 = (s16)r0 {{.*}}R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)")
__msg("r2 = (s16)r0 {{.*}}R2=scalar(smin=smin32=-24,smax=smax32=144,{{.*}}step=0+24)")
-__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)")
+__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=120,var_off=(0x0; 0xfffffffffffffff8))")
__msg("w4 = (s16)w1 {{.*}}R4=scalar(smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)")
-__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,var_off=(0x0; 0xffffffff))")
+__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=120,var_off=(0x0; 0xfffffff8))")
__naked void step_movsx_bounds_on_line(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_budget.c b/tools/testing/selftests/bpf/progs/verifier_budget.c
index 61c22ac58c1..890b316166c 100644
--- a/tools/testing/selftests/bpf/progs/verifier_budget.c
+++ b/tools/testing/selftests/bpf/progs/verifier_budget.c
@@ -1112,7 +1112,7 @@ __naked void guard_at_head_unset_reg_caller(void)
);
}
-/* fp-8 is 0 on every edge of the program that gets behind the loop. It's dead at the head. */
+/* fp-8 is 0 on every edge of the program that leaves the loop. It's 1 at the header. */
__naked __noinline __used
static unsigned long spins_and_makes_slot(void)
{
@@ -1127,7 +1127,7 @@ static unsigned long spins_and_makes_slot(void)
"*(u64 *)(r10 - 8) = r7;"
"if r1 != 2 goto 1b;"
"r0 = *(u64 *)(r10 - 8);"
- "r0 <<= 3;"
+ "r0 <<= 12;"
"r8 += r0;"
"r0 = *(u64 *)(r8 + 0);"
"exit;"
@@ -1138,8 +1138,8 @@ static unsigned long spins_and_makes_slot(void)
}
SEC("socket")
-__description("budget: may_goto at the head of a loop, a stack slot that is dead there")
-__failure __msg("slot poisoned by dead code elimination")
+__description("budget: may_goto at the header of a loop, a stack slot that the body writes")
+__failure __msg("invalid access to map value")
__naked void guard_at_head_dead_slot(void)
{
asm volatile (
@@ -1245,7 +1245,7 @@ SEC("socket")
__description("budget: may_goto at the head of a loop, another bound for the next loop")
__arch_x86_64
__arch_arm64
-__failure __msg("math between map_value pointer and register with unbounded min value")
+__failure __msg("invalid access to map value, value_size=5")
__naked void guard_at_head_bound_of_next_loop(void)
{
asm volatile (
@@ -1729,7 +1729,7 @@ __description("budget: every second trip leaves everything as it was")
__arch_x86_64
__arch_arm64
__success __retval(0) __log_level(2)
-__msg("processed 25 insns")
+__msg("processed {{[0-9][0-9]}} insns")
__naked void trips_of_period_two(void)
{
asm volatile (
@@ -1758,7 +1758,7 @@ __description("budget: a trip is dropped in the middle of the loop")
__arch_x86_64
__arch_arm64
__success __retval(0) __log_level(2)
-__msg("processed 33 insns")
+__msg("processed {{[0-9][0-9]}} insns")
__naked void trip_dropped_in_the_middle(void)
{
asm volatile (
@@ -1811,7 +1811,7 @@ __description("budget: a trip is dropped at a call that waits")
__arch_x86_64
__arch_arm64
__success __retval(0) __log_level(2)
-__msg("processed 27 insns")
+__msg("processed {{[0-9][0-9]}} insns")
__naked void trip_dropped_at_a_call(void)
{
asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
index 212c08a2705..7c3dc9643eb 100644
--- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
+++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
@@ -569,7 +569,7 @@ l0_%=: r0 = 0; \
SEC("tc")
__description("direct packet access: test23 (x += pkt_ptr, 4)")
-__failure __msg("invalid access to packet, off=31 size=8, R5(id=1,off=31,r=0)")
+__failure __msg("invalid access to packet, off=31 size=8, R5(id={{[0-9]+}},off=31,r=0)")
__flag(BPF_F_ANY_ALIGNMENT)
__naked void test23_x_pkt_ptr_4(void)
{
@@ -950,7 +950,7 @@ l0_%=: r0 = 0; \
SEC("tc")
__description("direct packet access: 8-aligned offset, check p, load past it via p + 8")
-__failure __msg("invalid access to packet, off=51 size=1")
+__failure __msg("invalid access to packet, off=49 size=1")
__naked void pkt_same_id_check_base_load_via_copy(void)
{
asm volatile (" \
@@ -978,7 +978,7 @@ l0_%=: r0 = 0; \
SEC("tc")
__description("direct packet access: 8-aligned offset, check p + 4, 4-byte load at p + 2")
-__failure __msg("invalid access to packet, off=52 size=4")
+__failure __msg("invalid access to packet, off=50 size=4")
__naked void pkt_same_id_check_base_4_load_via_copy(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_stack_access_wraparound.c b/tools/testing/selftests/bpf/progs/verifier_direct_stack_access_wraparound.c
index c538c689355..4385ff6225f 100644
--- a/tools/testing/selftests/bpf/progs/verifier_direct_stack_access_wraparound.c
+++ b/tools/testing/selftests/bpf/progs/verifier_direct_stack_access_wraparound.c
@@ -7,8 +7,8 @@
SEC("socket")
__description("direct stack access with 32-bit wraparound. test1")
-__failure __msg("fp pointer and 2147483647")
-__failure_unpriv
+__failure __msg("invalid unbounded variable-offset write to stack R1")
+__failure_unpriv __msg_unpriv("R1 variable stack access prohibited for !root")
__naked void with_32_bit_wraparound_test1(void)
{
asm volatile (" \
@@ -23,8 +23,8 @@ __naked void with_32_bit_wraparound_test1(void)
SEC("socket")
__description("direct stack access with 32-bit wraparound. test2")
-__failure __msg("fp pointer and 1073741823")
-__failure_unpriv
+__failure __msg("invalid unbounded variable-offset write to stack R1")
+__failure_unpriv __msg_unpriv("R1 variable stack access prohibited for !root")
__naked void with_32_bit_wraparound_test2(void)
{
asm volatile (" \
@@ -39,7 +39,7 @@ __naked void with_32_bit_wraparound_test2(void)
SEC("socket")
__description("direct stack access with 32-bit wraparound. test3")
-__failure __msg("fp pointer offset 1073741822")
+__failure __msg("invalid unbounded variable-offset write to stack R1")
__msg_unpriv("R1 stack pointer arithmetic goes out of range")
__naked void with_32_bit_wraparound_test3(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_jumps_ahead.c b/tools/testing/selftests/bpf/progs/verifier_jumps_ahead.c
index de3252b5997..7fc4a3c2acd 100644
--- a/tools/testing/selftests/bpf/progs/verifier_jumps_ahead.c
+++ b/tools/testing/selftests/bpf/progs/verifier_jumps_ahead.c
@@ -715,7 +715,7 @@ struct {
SEC("socket")
__description("shape: nothing in a register stands for a pointer")
__success __log_level(2)
-__msg("processed 135 insns")
+__msg("processed 76 insns")
__naked void shape_unset(void)
{
SHAPE_MANY("",
diff --git a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c
index c8b3690dd81..dcb8f981f47 100644
--- a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c
+++ b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c
@@ -8,7 +8,7 @@
SEC("socket")
__description("scalars: find linked scalars")
__failure
-__msg("math between fp pointer and 2147483647 is not allowed")
+__msg("invalid unbounded variable-offset write to stack R3")
__naked void scalars(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c
index 4bc4091b17a..b32463b73c2 100644
--- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c
+++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c
@@ -381,7 +381,7 @@ __msg("12: (95) exit")
/*
* fp0-8 is dead after goto +0, so one state stands for both.
*/
-__msg("from 11 to 12: within")
+__msg("from 11 to 12: pruned")
__naked void caller_stack_pruning(void)
{
asm volatile (
@@ -877,7 +877,7 @@ static __used __naked void maybe_clobber_key(void)
SEC("socket")
__log_level(2)
__success
-__msg("from 16 to 6: within")
+__msg("from 16 to 6: pruned")
__naked void spill_reload_inflates_stack_liveness(void)
{
asm volatile (
@@ -1374,7 +1374,7 @@ __naked void half_spi_clean_preserves_stack_zero(void)
SEC("socket")
__success
__log_level(2)
-__msg("from 3 to 7: within")
+__msg("from 3 to 7: pruned")
__naked void dead_spill_at_merge_enables_pruning(void)
{
asm volatile (
@@ -1416,7 +1416,7 @@ __naked void dead_spill_at_merge_enables_pruning(void)
SEC("socket")
__success
__log_level(2)
-__msg("from 9 to 10: within")
+__msg("from 9 to 10: pruned")
__naked void byte_load_in_subprog_keeps_no_spilled_ptr(void)
{
asm volatile (
@@ -1467,7 +1467,7 @@ __naked void byte_of_replaced_ptr_is_unknown(void)
SEC("socket")
__success
__log_level(2)
-__msg("from 9 to 10: within")
+__msg("from 9 to 10: pruned")
__failure_unpriv
__msg_unpriv("invalid read from stack R3 off -8+0 size 8")
__naked void helper_keeps_no_spilled_ptr(void)
diff --git a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
index 3f2a60f1931..bb343bad924 100644
--- a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
+++ b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
@@ -132,8 +132,8 @@ static unsigned long exp_sub1(void)
* defeating any memoization keyed on entry args.
*/
SEC("?raw_tp")
-__failure __log_level(2)
-__msg("liveness analysis exceeded complexity limit")
+__failure
+__msg("BPF program is too complex")
__naked int liveness_exponential_complexity(void)
{
asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_as_is.c b/tools/testing/selftests/bpf/progs/verifier_loop_as_is.c
index fffc53e17b5..08641994185 100644
--- a/tools/testing/selftests/bpf/progs/verifier_loop_as_is.c
+++ b/tools/testing/selftests/bpf/progs/verifier_loop_as_is.c
@@ -49,13 +49,10 @@ struct {
/*
* do { n += 3; } while (v[0]);
- * Two iterations are looked at as they are. The second one ends the walk: its jump goes both
- * ways. Its state is joined with the one before, nothing stops n, so n is any number.
- * What comes back next is within that. The number of insns tells that there is no fourth pass.
- * The loop gets may_goto when the walk ends, and all is done again: two times 24 insns.
+ * Nothing stops n, so n is any number. The loop has no trip count and gets may_goto.
*/
OK("the state that ends a walk is joined with the one before")
-__log_level(1) __msg("processed 48 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
__naked void two_passes(void)
{
asm volatile (
@@ -78,7 +75,7 @@ __naked void two_passes(void)
* the known bits are what the range says after the first one.
*/
OK("known bits are not lost one by one")
-__log_level(1) __msg("processed 74 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
__naked void bits_one_by_one(void)
{
asm volatile (
@@ -107,7 +104,7 @@ __naked void bits_one_by_one(void)
* time: no signed bound moves, and the range does, a million times if nothing is done about it.
*/
OK("a range moves and its signed bounds do not")
-__log_level(1) __msg("processed 68 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
__naked void unsigned_bound_moves(void)
{
asm volatile (
@@ -184,22 +181,22 @@ __naked void name(void) \
/* No jump of the loop depends on the sum, no stop holds it: two passes, with 0 and any number */
OK("a sum is any number at once")
-__log_level(1) __msg("processed 25 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
SUM(sum_two_passes, "", "r6 += r1;", 3, "r6 &= 63;")
BAD("a sum is any number at once, and no index")
-__msg("register with unbounded min value is not allowed")
+__msg("R7 max value is outside of the allowed memory range")
SUM(sum_two_passes_bad, "", "r6 += r1;", 3, "")
/* sum = x + sum + x; if (x != 3) n -= x; */
OK("a sum has many forms")
-__log_level(1) __msg("processed 36 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
SUM(sum_forms, "",
"r2 = r1; r2 += r6; r6 = r2; r6 += r1; if r1 == 3 goto +1; r9 -= r1;", 8, "r6 += r9; r6 &= 63;")
/* A test of the low half is sufficient behind the loop */
OK("a sum in 32 bits is any number of 32 bits at once")
-__log_level(1) __msg("processed 25 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
SUM(sum32_two_passes, "", "w6 += w1;", 3, "if w6 > 63 goto 9f;")
/* 0..S32_MAX at the head, 0..63 behind the jump. Of any number the jump leaves S64_MIN..63. */
@@ -215,12 +212,12 @@ SUM(sum_below_max_nested,
/* 0, 0..S32_MAX, 0..U32_MAX, any number. 0..S64_MAX would be one more pass, and wraps. */
OK("a sum that gets past U32_MAX is any number")
-__log_level(1) __msg("processed 37 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
SUM(sum_past_u32_max, "if r6 == 77 goto +0;", "r6 += r1;", 4, "r6 &= 63;")
/* 0, S32_MIN..0, any number */
OK("a sum that gets past S32_MIN is any number")
-__log_level(1) __msg("processed 32 insns")
+__log_level(1) __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
SUM(sum_past_s32_min, "if r6 == 77 goto +0;", "r6 -= r1;", 4, "r6 &= 63;")
/*
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c
index 4f4e04d50bc..2d032e9d969 100644
--- a/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c
+++ b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c
@@ -285,16 +285,16 @@ __success
__log_level(2)
__msg("loop at 1{{$}}")
__msg_next(" backedge from 10, latch at 10")
-__msg_next(" exit from 10 to 11")
+__msg_next(" first exit to 11")
__msg_next("loop at 3, nested in 1")
__msg_next(" backedge from 9, latch at 9")
-__msg_next(" exit from 9 to 10")
+__msg_next(" first exit to 10")
__msg_next("loop at 5, nested in 3")
__msg_next(" backedge from 6, latch at 6")
-__msg_next(" exit from 6 to 7")
+__msg_next(" first exit to 7")
__msg_next("loop at 7, nested in 3")
__msg_next(" backedge from 8, latch at 8")
-__msg_next(" exit from 8 to 9")
+__msg_next(" first exit to 9")
__naked void loop_nested_siblings_common_ancestors(void)
{
asm volatile (" \
@@ -322,11 +322,10 @@ __success
__log_level(2)
__msg("loop at 2{{$}}")
__msg_next(" backedge from 4{{$}}")
-__msg_next(" exit from 5 to 7")
+__msg_next(" first exit to 7")
__msg_next("loop at 3, nested in 2")
__msg_next(" backedge from 6, latch at 5")
-__msg_next(" exit from 4 to 2")
-__msg_next(" exit from 5 to 7")
+__msg_next(" first exit to 2")
__naked void loop_outer_backedge_from_inner(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_latch.c b/tools/testing/selftests/bpf/progs/verifier_loop_latch.c
index 5ebd3dfadac..e1281d00b43 100644
--- a/tools/testing/selftests/bpf/progs/verifier_loop_latch.c
+++ b/tools/testing/selftests/bpf/progs/verifier_loop_latch.c
@@ -31,9 +31,10 @@ void __kfunc_btf_root(void)
bpf_rcu_read_unlock();
}
-#define OK(text, count) \
+#define ONCE(text) \
SEC("socket") __description("latch: " text) __success __log_level(2) \
- __msg("header_count is " count)
+ __not_msg("has no trip count")
+#define OK(text, count) ONCE(text) __msg("has trip count " count "{{$}}") __not_msg("has no trip count")
#define BAD(text) \
SEC("socket") __description("latch: " text) __failure __msg("invalid access to map value")
/* the pointer goes on and on */
@@ -74,13 +75,13 @@ __naked void name(void) \
*/
#define UP_TO_8 "if r8 >= r7 goto 9f; r8 += 1; if r8 < 8 goto 1b;"
-OK("false at once, and another exit", "[0..1]")
+ONCE("false at once, and another exit")
LOOP(false_at_once, "r8 = 7; r6 += 2047;", UP_TO_8)
BAD("two iterations, and another exit")
LOOP(false_in_second, "r8 = 6; r6 += 2047;", UP_TO_8)
-OK("false at once for all of a range", "[0..1]")
+ONCE("false at once for all of a range")
LOOP(false_at_once_range, "r8 = r7; r8 &= 3; r8 += 7; r6 += 2047;", UP_TO_8)
BAD("false at once for all of a range but one")
@@ -157,7 +158,7 @@ LOOP(behind_two_bad, "w8 = 0; r9 = r6; r9 += 2044;", AT_TOP(5) "if r7 == 7 goto
"4: if w2 == 0 goto 1b; goto 9f;" \
"2: if w8 < 4 goto 1b;"
-OK("a store behind a jump that leaves on both edges", "[0..4]")
+OK("a store behind a jump that leaves on both edges", "4")
LOOP(behind_both, "w8 = 0; r9 = r6; r9 += 2043;", BOTH_LEAVE)
BAD("a store behind a jump that leaves on both edges, one too far")
@@ -168,7 +169,7 @@ LOOP(behind_both_bad, "w8 = 0; r9 = r6; r9 += 2044;", BOTH_LEAVE)
#define FAR_UP(n) "if w8 == " #n " goto 9f; w8 += 1;" \
STATUS STATUS STATUS STATUS STATUS "goto 1b;"
-OK("behind five jumps that don't count", "[0..2048]")
+OK("behind five jumps that don't count", "2048")
LOOP(far_up, "w8 = 0;", FAR_UP(2047))
BAD("behind five jumps that don't count, one too many")
@@ -205,7 +206,8 @@ int name(void *ctx) \
return idx; \
}
-OK("frames of a stack walker", "[0..9]") __retval(256)
+/* trips prove it */
+SEC("socket") __description("latch: frames of a stack walker") __success __retval(256)
#ifndef __clang__
__skip("GCC unrolls the inner loop")
#endif
@@ -240,7 +242,7 @@ LOOP(down_unsigned_bad, "w8 = 2048;", "w8 += -1; if w8 < 2049 goto 1b;")
OK("down, one unsigned test, 64 bits", "2048")
LOOP(down_unsigned64, "r8 = 2047;", "r8 += -1; if r8 <= 2047 goto 1b;")
-OK("down from a range, one unsigned test", "[0..2048]")
+OK("down from a range, one unsigned test", "2048")
LOOP(down_unsigned_range, "if r7 > 2047 goto 9f;", "w7 += -1; if w7 < 2048 goto 1b;")
BAD("down from a range, one unsigned test, one too many")
@@ -262,7 +264,7 @@ LOOP(down_two_tests, "w8 = 4; r6 += 2043;",
/* one more store, at the index that the counter is behind the loop */
#define AT_R8 "r5 += r8; *(u8 *)(r5 + 0) = r1;"
-OK("a copy of the counter", "[0..2048]")
+OK("a copy of the counter", "2048")
LOOP(copy, "r8 = 0; r9 = r8;", COPY(2046))
BAD("a copy of the counter, one too many")
@@ -278,14 +280,14 @@ LOOP(copy_ahead, "r8 = 0; r9 = 1;", COPY(2047))
/* r5 is a copy of the pointer r6 from before the increment: an induction variable too */
#define PTR_COPY(off) "*(u8 *)(r5 + " #off ") = r1; r5 = r6; r8 += 1; if r8 < 2048 goto 1b;"
-OK("a copy of the pointer", "2048") __msg("widening r5 to 0..2047 step 1")
+OK("a copy of the pointer", "2048")
LOOP(copy_of_pointer, "r5 = r6; r8 = 0;", PTR_COPY(0))
BAD("a copy of the pointer, one too many")
LOOP(copy_of_pointer_bad, "r5 = r6; r8 = 0;", PTR_COPY(1))
/* r8 and r9 are computed in the same way from r7: equal by value number, 'r9 = r8' is not needed */
-OK("a copy of the counter, equal by value number", "[0..2]")
+OK("a copy of the counter, equal by value number", "2")
LOOP(copy_same_value_number,
"r5 = r6; r8 = r7; r8 &= 1; r8 += 2045; r9 = r7; r9 &= 1; r9 += 2045;", COPY(2045) AT_R8)
@@ -297,7 +299,7 @@ LOOP(copy_same_value_number,
"r8 += -1; r2 = r5; r2 += r9; *(u8 *)(r2 " off ") = r1; r9 = r8; if r8 > 0 goto 1b;"
#define FROM_RANGE "if r7 > 2048 goto 9f; if r7 == 0 goto 9f; r5 = r6; r8 = r7; r9 = r7;"
-OK("a copy of the counter, down from a range", "[0..2048]")
+OK("a copy of the counter, down from a range", "2048")
LOOP(copy_down_from_range, FROM_RANGE, DOWN("- 1"))
BAD("a copy of the counter, down from a range, one too many")
@@ -307,11 +309,11 @@ LOOP(copy_down_from_range_bad, FROM_RANGE, DOWN("+ 0"))
* r9 is zero extended. [1, 2048], its range at the header, is the range of the zero extension too.
* [-1000, 1001] is not: the first store is 4G behind the value.
*/
-OK("a copy of the counter, zero extended", "[0..2048]")
+OK("a copy of the counter, zero extended", "2048")
LOOP(copy_zext, FROM_RANGE, "w9 = w9;" DOWN("- 1"))
SEC("socket") __description("latch: a copy of the counter, zero extended, from below 0")
-__failure __msg("R4 unbounded memory access")
+__failure __msg("value is outside of the allowed memory range")
LOOP(copy_zext_bad, "r5 = r6; r8 = r7; r8 &= 255; r8 += -1000; r9 = r8;",
"w9 = w9; if r8 s> 1000 goto 9f; r4 = r5; r4 += r9; *(u8 *)(r4 + 1000) = r1;"
"r8 += 1; r9 = r8; if r7 != 0 goto 1b;")
@@ -328,7 +330,7 @@ LOOP(copy_zext_bad, "r5 = r6; r8 = r7; r8 &= 255; r8 += -1000; r9 = r8;",
#endif
#define PTR_FROM_RANGE "r5 = r6; r8 = r6; r8 += 2047; r7 &= 1023; r6 += r7; r9 = r6;"
-OK("a copy of the pointer that the latch tests", "[0..2047]")
+OK("a copy of the pointer that the latch tests", "2047")
LOOP(copy_of_tested_pointer, PTR_FROM_RANGE, "*(u8 *)(r9 + 1) = r1; r9 = r6; if r6 < r8 goto 1b;")
SEC("socket") __description("latch: a copy of the pointer that the latch tests, as a scalar")
@@ -336,7 +338,7 @@ __failure __msg("math between map_value pointer and register with unbounded min
LOOP(copy_of_tested_pointer_scalar, PTR_FROM_RANGE NOP64 "if r9 == 0 goto 9f;",
"r4 = r5; r4 += r9; *(u8 *)(r4 + 0) = r1; r9 = r6;" NOP64 "if r6 < r8 goto 1b;")
-OK("a copy of the counter, a range", "[0..2]")
+OK("a copy of the counter, a range", "2")
LOOP(copy_range, "r5 = r6; r8 = r7; r8 &= 1; r8 += 2045; r9 = r8;", COPY(2045) AT_R8)
BAD("a copy of the counter, another number of the same range at the entry")
@@ -349,7 +351,7 @@ LOOP(copy_behind, "r8 = 0; r9 = r8;",
"r8 += 1; if r9 > 2046 goto 9f; r9 = r8; r9 += -1; if r7 != 0 goto 1b;")
/* what the jump says about the copy holds for the counter */
-OK("a copy of the counter, the jump bounds both", "[0..2048]")
+OK("a copy of the counter, the jump bounds both", "2047")
LOOP(copy_bounds_both, "r5 = r6; if r7 > 2047 goto 9f; r8 = r7; r9 = r8;",
"if r9 > 2046 goto 8f; r8 += 1; r9 = r8; if r7 != 0 goto 1b; 8:" AT_R8)
@@ -361,13 +363,13 @@ LOOP(copy_bounds_both, "r5 = r6; if r7 > 2047 goto 9f; r8 = r7; r9 = r8;",
"r8 += " #step "; if r9 " leave " goto 9f; if r8 " skip " goto 2f;" \
"r4 = r5; r4 += r9; *(u8 *)(r4 " off ") = r1; 2: r9 = r8; if r7 != 0 goto 1b;"
-OK("a copy of the counter, one less behind the jump", "[0..2048]")
+OK("a copy of the counter, one less behind the jump", "2048")
LOOP(copy_less, "r5 = r6; r8 = 0; r9 = r8;", NOT_EQUAL(1, "> 2046", "< 2", "+ 0"))
BAD("a copy of the counter, one less behind the jump, not equal")
LOOP(copy_less_bad, "r5 = r6; r8 = 0; r9 = r8;", NOT_EQUAL(1, "> 2046", "< 2", "- 2"))
-OK("a copy of the counter, one more behind the jump", "[0..2047]")
+OK("a copy of the counter, one more behind the jump", "2047")
LOOP(copy_more, "r5 = r6; r8 = 2047; r9 = r8;", NOT_EQUAL(-1, "s< 2", "> 2045", "+ 0"))
BAD("a copy of the counter, one more behind the jump, not equal")
@@ -382,7 +384,7 @@ LOOP(copy_more_bad, "r5 = r6; r8 = 2047; r9 = r8;", NOT_EQUAL(-1, "s< 2", "> 204
"r3 = r4; r3 &= 4095; r3 += r5; *(u8 *)(r3 + 0) = r1;" \
X4(X4(X4("r3 = 0;"))) "r9 = r8; if r7 != 0 goto 1b;"
-OK("a copy of a copy of the counter", "[0..2048]")
+OK("a copy of a copy of the counter", "2048")
LOOP(copy_of_copy, "r5 = r6; r8 = 0; r9 = 0; r4 = 0;", OF_COPY)
BAD("a copy of what is no copy of the counter")
@@ -391,7 +393,7 @@ LOOP(copy_of_no_copy, "r5 = r6; r8 = 0; r9 = r7; r9 &= 4095; r4 = 0;", OF_COPY)
/* 'while (n--)': the jump that counts sets a flag, the jump that leaves the loop tests it */
#define FLAG(mid) "w8 += 1; w2 = 1; if w8 == 0 goto 2f; w2 = 0; 2: " mid " if w2 != 1 goto 1b;"
-OK("a flag", "[0..2048]")
+OK("a flag", "2048")
LOOP(flag, "w8 = -2048;", FLAG("r3 = r6;"))
BAD("a flag, one too many")
@@ -400,21 +402,21 @@ LOOP(flag_bad, "w8 = -2049;", FLAG("r3 = r6;"))
/* A flag leaves the loop later: what is behind the jump runs in the last iteration too */
#define STORE "*(u8 *)(r9 + 0) = r1; r9 += 1;"
-OK("a flag, a store behind the jump that counts", "[0..2048]")
+OK("a flag, a store behind the jump that counts", "2048")
LOOP(flag_behind, "w8 = -2048; r9 = r6;", FLAG(STORE))
BAD("a flag, a store behind the jump that counts, one too far")
LOOP(flag_behind_bad, "w8 = -2048; r9 = r6; r9 += 1;", FLAG(STORE))
-OK("a flag, a store behind it", "[0..2048]")
+OK("a flag, a store behind it", "2048")
LOOP(flag_store, "w8 = -2048;",
"w8 += 1; w2 = 1; *(u32 *)(r10 - 4) = r8; if w8 == 0 goto 2f; w2 = 0; 2: if w2 != 1 goto 1b;")
-OK("a flag, a call behind it", "[0..2048]")
+OK("a flag, a call behind it", "2048")
LOOP(flag_call, "w8 = -2048;",
"w8 += 1; w9 = 1; if w8 == 0 goto 2f; w9 = 0; 2: call %[bpf_get_prandom_u32]; if w9 != 1 goto 1b;")
-OK("a flag that is set in front of another jump", "[0..2048]")
+OK("a flag that is set in front of another jump", "2048")
LOOP(flag_far, "w8 = -2048;",
"w8 += 1; w2 = 1; if r7 != 0 goto 3f; r3 = r6;"
"3: if w8 == 0 goto 2f; w2 = 0; 2: if w2 != 1 goto 1b;")
@@ -430,26 +432,26 @@ LOOP(flag_far, "w8 = -2048;",
#define WORK20 WORK4 WORK4 WORK4 WORK4 WORK4
#define RND "call %[bpf_get_prandom_u32];"
-OK("more: work", "[0..2048]")
+OK("more: work", "2048")
LOOP(more_work, "w8 = 0;", MORE(UP_TO(2047), WORK20))
BAD("more: work, one too many")
LOOP(more_work_bad, "w8 = 0;", MORE(UP_TO(2048), WORK20))
-OK("more: work with a jump in it", "[0..2048]")
+OK("more: work with a jump in it", "2048")
LOOP(more_jump, "w8 = 0;", MORE(UP_TO(2047), WORK4 "if r7 > 5 goto 4f;" WORK20 "4:"))
-OK("more: work with calls in it", "[0..2048]")
+OK("more: work with calls in it", "2048")
LOOP(more_calls, "w8 = 0;", MORE(UP_TO(2047), RND WORK20 RND))
-OK("more: the counter is spilled", "[0..2048]")
+OK("more: the counter is spilled", "2048")
LOOP(more_spill, "r8 = 0; *(u64 *)(r10 - 16) = r8;",
MORE("r8 = *(u64 *)(r10 - 16); w8 += 1; w9 = 0; *(u64 *)(r10 - 16) = r8;"
"if w8 > 2047 goto 2f;", RND "w8 = w0;" WORK20 RND))
#define NOT_TAKEN(n) "w8 += 1; if w8 < " #n " goto 4f; w9 = 0; goto 2f; 4:"
-OK("more: the edge that is not taken leaves", "[0..2048]")
+OK("more: the edge that is not taken leaves", "2048")
LOOP(more_not_taken, "w8 = 0;", MORE(NOT_TAKEN(2048), WORK20))
BAD("more: the edge that is not taken leaves, one too many")
@@ -458,19 +460,20 @@ LOOP(more_not_taken_bad, "w8 = 0;", MORE(NOT_TAKEN(2049), WORK20))
#define JUMP "if r7 > 5 goto 4f;" WORK4 "4:"
#define JUMP8 JUMP JUMP JUMP JUMP JUMP JUMP JUMP JUMP
-OK("more: work with 40 jumps in it", "[0..2048]")
+OK("more: work with 40 jumps in it", "2048")
LOOP(more_jumps, "w8 = 0;", MORE(UP_TO(2047), JUMP8 JUMP8 JUMP8 JUMP8 JUMP8))
-OK("more: behind five jumps that don't count", "[0..2048]")
+OK("more: behind five jumps that don't count", "2048")
LOOP(more_far, "w8 = 0;", MORE(UP_TO(2047), STATUS STATUS STATUS STATUS STATUS))
BAD("more: behind five jumps that don't count, one too many")
LOOP(more_far_bad, "w8 = 0;", MORE(UP_TO(2048), STATUS STATUS STATUS STATUS STATUS))
-OK("more: the flag is 0 or 2 where two paths meet", "[0..2048]")
+/* no state comes back from the last trip: the log has the trip before it */
+OK("more: the flag is 0 or 2 where two paths meet", "2047")
LOOP(more_even, "w8 = 0;", MORE(UP_TO(2047), "if r7 > 5 goto 4f; w9 += 2; 4:"))
-OK("a flag, a jump on the way out knows a constant", "[0..2048]")
+OK("a flag, a jump on the way out knows a constant", "2048")
LOOP(flag_const, "w8 = -2048;",
"w8 += 1; w2 = 1; if w8 == 0 goto 2f; w2 = 0;"
"2: w3 = 0; if w2 != 0 goto 3f; goto 1b; 3: if w3 == 2 goto 1b;")
@@ -479,16 +482,16 @@ LOOP(flag_const, "w8 = -2048;",
#define ERR(last, other, test) \
"w8 += 1; w9 = " #last "; if w8 > 2047 goto 2f; w9 = " #other "; 2: " WORK20 test
-OK("an error code", "[0..2048]")
+OK("an error code", "2048")
LOOP(err, "w8 = 0;", ERR(-7, 0, "if w9 == 0 goto 1b;"))
-OK("an error code, a signed test", "[0..2048]")
+OK("an error code, a signed test", "2048")
LOOP(err_signed, "w8 = 0;", ERR(-7, 0, "if w9 s>= 0 goto 1b;"))
-OK("a size", "[0..2048]")
+OK("a size", "2048")
LOOP(size, "w8 = 0;", ERR(128, 0, "if w9 <= 63 goto 1b;"))
-OK("a bit of a size", "[0..2048]")
+OK("a bit of a size", "2048")
LOOP(size_bit, "w8 = 0;", ERR(130, 0, "if w9 & 128 goto 9f; goto 1b;"))
/* in these the loop goes on when the counter gets to its last value */
@@ -529,7 +532,7 @@ LOOP(err_wide, "w8 = 0;",
NO_COUNT("more: a back edge that doesn't test the flag")
LOOP(more_back_edge, "w8 = 0;",
- MORE(UP_TO(2047), "if w7 == 3 goto 5f;") "goto 9f; 5: if w8 == 2048 goto 1b;")
+ MORE(UP_TO(2047), RND "if w0 == 3 goto 5f;") "goto 9f; 5: if w8 == 2048 goto 1b;")
/* A nested loop can get may_goto. It goes where a jump of that loop goes, whatever the flag is. */
NO_COUNT("more: the way out leads through a nested loop")
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_need.c b/tools/testing/selftests/bpf/progs/verifier_loop_need.c
index cfc98aeb155..b871dc8fd51 100644
--- a/tools/testing/selftests/bpf/progs/verifier_loop_need.c
+++ b/tools/testing/selftests/bpf/progs/verifier_loop_need.c
@@ -30,8 +30,7 @@ struct {
#define OK(text) SEC("socket") __description("need: " text) __success
#define BAD(text) SEC("socket") __description("need: " text) __failure
-#define NOT_WALKED __log_level(2) __msg("nothing looks at it")
-#define WALKED __log_level(2) __not_msg("nothing looks at it")
+#define NOT_WALKED __log_level(2) __not_msg(")=2 loop_stack=")
/* The read is not allowed */
#define UNLESS(cond) "if " cond " goto +1; r0 = *(u64 *)(r10 + 8);"
@@ -54,14 +53,13 @@ struct {
#define TIMES_3(n, use) TIMES_3_IN("", n, use "r0 = 0;")
OK("a number that is only stored is not followed")
-NOT_WALKED __msg("processed 17 insns")
+NOT_WALKED __msg("processed {{[0-9]?[0-9]?[0-9]}} insns")
__naked void stored_only(void)
{
TIMES_3(1000, "*(u64 *)(r10 - 8) = r6;");
}
OK("a jump looks at the number")
-WALKED
__naked void jump(void)
{
TIMES_3(5, UNLESS("r6 <= 486"));
@@ -75,7 +73,6 @@ __naked void jump_too_large(void)
}
OK("the number is an offset in a map value that holds numbers")
-WALKED
__naked void value_offset(void)
{
/* 27 or 54 */
@@ -91,7 +88,6 @@ static unsigned long value_at_plus_9(void)
}
OK("the number is an offset in a map value in a function")
-WALKED
__naked void value_offset_argument(void)
{
TIMES_3(3, "r1 = r6; call value_at_plus_9;");
@@ -104,7 +100,6 @@ static unsigned long times_27(void)
}
OK("what a function returns is an offset in a map value")
-WALKED
__naked void value_offset_returned(void)
{
asm volatile ("call times_27;" VALUE_AT(r0, 9) "exit;"
@@ -124,7 +119,7 @@ static struct pair zero_and_times_27(void)
}
OK("the second half of what a function returns is an offset in a map value")
-__load_if_JITed() WALKED
+__load_if_JITed()
__naked void value_offset_returned_in_r2(void)
{
asm volatile ("call zero_and_times_27;" VALUE_AT(r2, 9) "exit;"
@@ -141,7 +136,6 @@ static unsigned long times_27_there(void)
#define TIMES_27_AT_FP_8 "r1 = r10; r1 += -8; call times_27_there; r2 = *(u64 *)(r10 - 8);"
OK("a function stores the number in the stack of its caller")
-WALKED
__naked void stored_for_the_caller(void)
{
asm volatile (TIMES_27_AT_FP_8 VALUE_AT(r2, 9) "exit;"
@@ -149,7 +143,6 @@ __naked void stored_for_the_caller(void)
}
OK("the number is an offset in the stack")
-WALKED
__naked void stack_offset(void)
{
/* fp-9 or fp-18 */
@@ -169,7 +162,6 @@ static unsigned long times_9(void)
"r0 = -r0; r1 = r10; r1 += r0; r0 = *(u8 *)(r1 + " #off ");"
OK("what a function returns is an offset in the stack")
-WALKED
__naked void stack_offset_returned(void)
{
asm volatile ("call times_9;" STACK_AT_MINUS_R0(0) "exit;" ::: __clobber_all);
@@ -190,7 +182,6 @@ __naked void stack_offset_returned(void)
__imm_addr(ringbuf) : __clobber_all)
OK("the number is part of a size that has to be a constant")
-WALKED
__naked void constant_argument(void)
{
NINE("r1 = %[ringbuf] ll; r2 = 7; r2 += r6; r3 = 0; call %[bpf_ringbuf_reserve];"
@@ -223,7 +214,6 @@ __naked void arena_address(void)
* No jump and no add looks at p.
*/
OK("the base of a load is 0 in the first iteration")
-WALKED
__naked void base_of_a_load(void)
{
asm volatile (
@@ -244,7 +234,6 @@ __naked void base_of_a_load(void)
/* r8 goes on by 0 or 8 bytes. Nothing tells how far but the iterations. */
OK("a pointer that moves is followed")
-WALKED
__naked void pointer(void)
{
asm volatile (
@@ -265,7 +254,6 @@ __naked void pointer(void)
/* The sets have 64 slots of the stack. What is deeper is followed. */
OK("a jump looks at a slot below fp-512")
__load_if_large_stack()
-WALKED
__naked void deep_slot(void)
{
asm volatile (
@@ -286,7 +274,6 @@ __naked void deep_slot(void)
/* The iterator is not a number. It is at its end after 3 of 5 iterations. */
OK("a slot that holds an iterator is followed")
-WALKED
__naked void iterator_slot(void)
{
asm volatile (
@@ -333,7 +320,7 @@ __naked void iterator_slot(void)
:: __imm(bpf_get_prandom_u32), __imm_addr(vals) : __clobber_all)
OK("what is not set at the entry is set from the second iteration on")
-NOT_WALKED __msg("processed 22 insns")
+NOT_WALKED __msg("processed {{[0-9][0-9]}} insns")
__naked void set_in_the_first_iteration(void)
{
SUM_OF_PREV(0, 40, "*(u64 *)(r10 - 8) = r9;");
@@ -422,7 +409,7 @@ __naked void index_from_rodata_too_large(void)
/* A slot holds a leftover pointer. The loop fills it with what it loads. It's only copied later. */
OK("no walk for what the loop loads if nothing looks at it")
-__log_level(2) __msg("processed 17 insns")
+NOT_WALKED __msg("processed {{[0-9][0-9]}} insns")
__naked void leftover_in_a_slot(void)
{
asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_walk.c b/tools/testing/selftests/bpf/progs/verifier_loop_walk.c
index 1a6180aceac..4baf4717591 100644
--- a/tools/testing/selftests/bpf/progs/verifier_loop_walk.c
+++ b/tools/testing/selftests/bpf/progs/verifier_loop_walk.c
@@ -285,7 +285,7 @@ static __naked __noinline __used int long_loop(void)
*/
SEC("socket")
__description("walk: no other try for a loop that is too long")
-__success __log_level(4) __msg("processed {{....}} insns")
+__success __log_level(4) __msg("processed {{[0-9][0-9]}} insns")
__naked void walk_fails_once(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_meta_access.c b/tools/testing/selftests/bpf/progs/verifier_meta_access.c
index c87e0be4b2f..484b9c003e1 100644
--- a/tools/testing/selftests/bpf/progs/verifier_meta_access.c
+++ b/tools/testing/selftests/bpf/progs/verifier_meta_access.c
@@ -283,7 +283,7 @@ l0_%=: r0 = 0; \
SEC("xdp")
__description("meta access, 8-aligned offset, check p, load a byte at p + 1 via p + 8")
-__failure __msg("invalid access to packet, off=51 size=1")
+__failure __msg("invalid access to packet, off=49 size=1")
__naked void meta_access_check_base_load_via_copy(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_precision.c b/tools/testing/selftests/bpf/progs/verifier_precision.c
index f0f9b62494c..d48daefd7b7 100644
--- a/tools/testing/selftests/bpf/progs/verifier_precision.c
+++ b/tools/testing/selftests/bpf/progs/verifier_precision.c
@@ -141,7 +141,7 @@ __naked int state_loop_first_last_equal(void)
"r0 = 0;"
"l0_%=:"
"r0 += 2;"
- /* scalar evolution has no formula for it: the loop is not counted */
+ /* no equation holds for it: the loop is not counted */
"r0 |= 0;"
/* every few iterations we'll have a checkpoint here with
* first_idx == last_idx, potentially confusing precision
diff --git a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c
index 799db6f5713..3fbb6f9c43e 100644
--- a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c
+++ b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c
@@ -446,9 +446,10 @@ __naked void reference_tracking_release_reference_twice(void)
r6 = r0; \
if r0 == 0 goto l0_%=; \
call %[bpf_sk_release]; \
-l0_%=: r1 = r6; \
+l0_%=: if r6 == 0 goto l1_%=; \
+ r1 = r6; \
call %[bpf_sk_release]; \
- exit; \
+l1_%=: exit; \
" :
: __imm(bpf_sk_lookup_tcp),
__imm(bpf_sk_release),
@@ -585,8 +586,10 @@ __naked void reference_in_subprog_and_outside(void)
" r1 = r0; /* unchecked reference */ \
r6 = r0; \
call reference_in_subprog_and_outside__1; \
+ if r6 == 0 goto l0_%=; \
r1 = r6; \
call %[bpf_sk_release]; \
+l0_%=: r0 = 0; \
exit; \
" :
: __imm(bpf_sk_lookup_tcp),
diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
index 43578b93dfb..a2e081d515a 100644
--- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
+++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
@@ -481,7 +481,8 @@ __naked void check_ids_in_regsafe_2(void)
*/
SEC("socket")
__success __log_level(2)
-__msg("from 14 to 15: within")
+__msg("from 14 to 15: pruned")
+__msg("from 8 to 15: R3=0 R4=0")
__msg("15: (1d) if r3 == r4 goto pc+0 ; R3=0 R4=0")
__msg("processed 18 insns")
__naked void no_scalar_id_for_const(void)
@@ -520,7 +521,8 @@ __naked void no_scalar_id_for_const(void)
/* Same as no_scalar_id_for_const() but for 32-bit values */
SEC("socket")
__success __log_level(2)
-__msg("from 14 to 15: within")
+__msg("from 14 to 15: pruned")
+__msg("from 8 to 15: R3=0 R4=0")
__msg("15: (1e) if w3 == w4 goto pc+0 ; R3=0 R4=0")
__msg("processed 18 insns")
__naked void no_scalar_id_for_const32(void)
@@ -566,7 +568,7 @@ SEC("socket")
__success __log_level(2)
__msg("6: (25) if r6 > 0x7 goto pc+1")
__msg("7: (57) r1 &= 255")
-__msg("from 7 to 8: within")
+__msg("from 7 to 8: pruned")
__msg("8: R0=0 R1=" ALONE(255, 0xff))
__msg("processed 11 insns")
__naked void ignore_unique_scalar_ids_old(void)
@@ -606,7 +608,7 @@ SEC("socket")
__success __log_level(2)
__msg("6: (25) if r6 > 0x7 goto pc+1")
__msg("7: (05) goto pc+1")
-__msg("from 7 to 9: within")
+__msg("from 7 to 9: pruned")
__msg("9: R0=0 R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=255,var_off=(0x0; 0xff)) R10=fp0")
__msg("processed 12 insns")
__naked void ignore_unique_scalar_ids_cur(void)
@@ -644,7 +646,7 @@ __naked void ignore_unique_scalar_ids_cur(void)
*/
SEC("socket")
__success __log_level(2)
-__msg("from 10 to 11: within")
+__msg("from 10 to 11: pruned")
__msg("processed 15 insns")
__naked void two_nil_old_ids_one_cur_id(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c
index d8e65938c35..cd65555c13d 100644
--- a/tools/testing/selftests/bpf/progs/verifier_sock.c
+++ b/tools/testing/selftests/bpf/progs/verifier_sock.c
@@ -888,7 +888,7 @@ l0_%=: r0 = *(u32*)(r0 + %[bpf_xdp_sock_queue_id]); \
SEC("sk_skb")
__description("bpf_map_lookup_elem(sockmap, &key)")
-__failure __msg("Unreleased reference id=2 alloc_insn=6")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=6")
__naked void map_lookup_elem_sockmap_key(void)
{
asm volatile (" \
@@ -908,7 +908,7 @@ __naked void map_lookup_elem_sockmap_key(void)
SEC("sk_skb")
__description("bpf_map_lookup_elem(sockhash, &key)")
-__failure __msg("Unreleased reference id=2 alloc_insn=6")
+__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=6")
__naked void map_lookup_elem_sockhash_key(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
index 623ed5ef363..958ee0d6419 100644
--- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
+++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
@@ -296,7 +296,11 @@ l0_%=: r0 = 0; \
SEC("tc")
__description("Spill u32 const scalars. Refill as u64. Offset to skb->data")
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+__failure __msg("makes pkt pointer be out of bounds")
+#else
__failure __msg("math between pkt pointer and register with unbounded min value is not allowed")
+#endif
__naked void u64_offset_to_skb_data(void)
{
asm volatile (" \
@@ -431,7 +435,7 @@ __naked void and_then_at_fp_8(void)
SEC("xdp")
__description("32-bit spill of 64-bit reg should clear ID")
-__failure __msg("math between ctx pointer and 4294967295 is not allowed")
+__failure __msg("variable ctx access var_off=(0x0; 0xffffffff) disallowed")
__naked void spill_32bit_of_64bit_fail(void)
{
asm volatile (" \
@@ -1053,7 +1057,7 @@ l0_%=: r0 = 0; \
SEC("xdp")
__description("32-bit fill after 64-bit spill should clear ID")
-__failure __msg("math between ctx pointer and 4294967295 is not allowed")
+__failure __msg("variable ctx access var_off=(0x0; 0xffffffff) disallowed")
__naked void fill_32bit_after_spill_64bit_clear_id(void)
{
asm volatile (" \
@@ -1102,13 +1106,13 @@ l0_%=: r1 >>= 32; \
*/
SEC("socket")
__success __log_level(2)
-__msg("8: R0=scalar() R10=fp0 fp-8=??mm??mm")
+__msg("8: R0=scalar() R10=fp0 fp-8=mm??mm??")
__msg("8: (79) r1 = *(u64 *)(r10 -8)")
__msg("processed 10 insns")
/* STACK_INVALID is an error in unpriv mode */
__failure_unpriv
__msg_unpriv("8: (79) r1 = *(u64 *)(r10 -8)")
-__msg_unpriv("invalid read from stack off -8+2 size 8")
+__msg_unpriv("invalid read from stack off -8+0 size 8")
__naked void old_imprecise_scalar_vs_cur_stack_misc(void)
{
asm volatile(
@@ -1117,8 +1121,8 @@ __naked void old_imprecise_scalar_vs_cur_stack_misc(void)
"if r0 == 0 goto 1f;"
/* conjure STACK_{MISC,INVALID} at fp-8 */
"call %[bpf_ktime_get_ns];"
- "*(u16*)(r10 - 8) = r0;"
- "*(u16*)(r10 - 4) = r0;"
+ "*(u16*)(r10 - 6) = r0;"
+ "*(u16*)(r10 - 2) = r0;"
"goto 2f;"
"1:"
/* conjure scalar at fp-8 */
@@ -1152,7 +1156,7 @@ __naked void old_precise_scalar_vs_cur_stack_misc(void)
/* conjure STACK_MISC at fp-8 */
"call %[bpf_ktime_get_ns];"
"*(u64*)(r10 - 8) = r0;"
- "*(u32*)(r10 - 4) = r0;"
+ "lock *(u64*)(r10 - 8) += r0;"
"goto 2f;"
"1:"
/* conjure scalar at fp-8 */
@@ -1176,7 +1180,7 @@ __naked void old_precise_scalar_vs_cur_stack_misc(void)
*/
SEC("socket")
__success __log_level(2)
-__msg("from 4 to 8: within")
+__msg("from 4 to 8: pruned")
__msg("8: (79) r0 = *(u64 *)(r10 -8)")
__msg("processed 10 insns")
__naked void old_stack_misc_vs_cur_scalar(void)
@@ -1192,8 +1196,8 @@ __naked void old_stack_misc_vs_cur_scalar(void)
"1:"
/* conjure STACK_{MISC,INVALID} at fp-8 */
"call %[bpf_ktime_get_ns];"
- "*(u16*)(r10 - 8) = r0;"
- "*(u16*)(r10 - 4) = r0;"
+ "*(u16*)(r10 - 6) = r0;"
+ "*(u16*)(r10 - 2) = r0;"
"2:"
/* read fp-8, it's looked at once */
"r0 = *(u64*)(r10 - 8);"
@@ -1210,7 +1214,6 @@ __naked void old_stack_misc_vs_cur_scalar(void)
*/
SEC("socket")
__success __log_level(2)
-__msg("from 4 to 8: within")
__msg("8: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar()")
__msg("processed 10 insns")
__naked void old_stack_misc_vs_cur_ctx_ptr(void)
@@ -1228,7 +1231,7 @@ __naked void old_stack_misc_vs_cur_ctx_ptr(void)
/* conjure STACK_MISC at fp-8 */
"call %[bpf_ktime_get_ns];"
"*(u64*)(r10 - 8) = r0;"
- "*(u32*)(r10 - 4) = r0;"
+ "lock *(u64*)(r10 - 8) += r0;"
"2:"
/* read fp-8, it's looked at once */
"r1 = *(u64*)(r10 - 8);"
@@ -1284,9 +1287,9 @@ __naked void stack_noperfmon_spill_32bit_onto_64bit_slot(void)
*/
SEC("socket")
__success __log_level(2)
-__msg("from 5 to 8: within")
+__msg("from 5 to 8: pruned")
__msg("8: (79) r1 = *(u64 *)(r10 -8)")
-__msg("processed 10 insns")
+__msg("processed 11 insns")
__naked void old_imprecise_scalar32_vs_cur_stack_misc(void)
{
asm volatile(
@@ -1295,8 +1298,8 @@ __naked void old_imprecise_scalar32_vs_cur_stack_misc(void)
"if r0 == 0 goto 1f;"
/* conjure STACK_MISC at fp-8 */
"call %[bpf_ktime_get_ns];"
- "*(u16*)(r10 - 8) = r0;"
- "*(u16*)(r10 - 6) = r0;"
+ "*(u32*)(r10 - 8) = r0;"
+ "lock *(u32*)(r10 - 8) += w0;"
"goto 2f;"
"1:"
/* conjure 32-bit scalar spill at fp-8 */
@@ -1305,6 +1308,7 @@ __naked void old_imprecise_scalar32_vs_cur_stack_misc(void)
"2:"
/* read fp-8, it's looked at once */
"r1 = *(u64*)(r10 - 8);"
+ "r0 = 0;"
"exit;"
:
: __imm(bpf_ktime_get_ns)
diff --git a/tools/testing/selftests/bpf/progs/verifier_spin_lock.c b/tools/testing/selftests/bpf/progs/verifier_spin_lock.c
index f4e7f47e4cd..6853b0de0c8 100644
--- a/tools/testing/selftests/bpf/progs/verifier_spin_lock.c
+++ b/tools/testing/selftests/bpf/progs/verifier_spin_lock.c
@@ -469,7 +469,7 @@ l3_%=: r1 = r7; \
SEC("cgroup/skb")
__description("spin_lock: regsafe() check_ids() similar id mappings")
-__success __msg("to 29: within")
+__success __msg("to 29: pruned")
__failure_unpriv __msg_unpriv("")
__log_level(2) __retval(0)
__naked void check_ids_similar_id_mappings(void)
diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c
index e9194a560fb..d030a8c52da 100644
--- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c
+++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c
@@ -200,7 +200,7 @@ __naked void to_stack_check_high_6(void)
SEC("socket")
__description("PTR_TO_STACK check high 7")
-__failure __msg("fp pointer offset")
+__failure __msg("invalid unbounded variable-offset write to stack R1")
__msg_unpriv("R1 stack pointer arithmetic goes out of range")
__naked void to_stack_check_high_7(void)
{
@@ -360,7 +360,7 @@ __naked void to_stack_check_low_6(void)
SEC("socket")
__description("PTR_TO_STACK check low 7")
-__failure __msg("fp pointer offset")
+__failure __msg("invalid unbounded variable-offset write to stack R1")
__msg_unpriv("R1 stack pointer arithmetic goes out of range")
__naked void to_stack_check_low_7(void)
{
@@ -595,17 +595,15 @@ SEC("socket")
__success
__log_level(2)
/*
- * r7 is widened to [-72, -16] at the loop header (insn 5),
+ * r7 is joined to [-72, -16] at the loop header (insn 5),
* the loop body sees [-64, -8] after 'r7 += 8'.
*/
-__msg("loop header at 5, widening r7 to -72..-16 step 1")
__msg("R7=fp(smin=smin32=-64,smax=smax32=-8")
-/* 'r7 = r8' sets r7 to fp-16; its ANY SCEV is not clamped. */
+/* 'r7 = r8' sets r7 to fp-16 */
__msg("R7=fp-16 R8=fp-16 loop_stack=5")
-__not_msg("clamping r7")
/* back-edge state goes no further because {-16} is within [-72, -16]. */
-__msg("from 8 to 5: within")
-__msg("processed 11 insns")
+__msg("from 8 to 5: pruned")
+__msg("processed 16 insns")
__naked void stack_ptr_subrange_in_loop(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_callers.c b/tools/testing/selftests/bpf/progs/verifier_subprog_callers.c
index 48338c0ae46..5c3690803f5 100644
--- a/tools/testing/selftests/bpf/progs/verifier_subprog_callers.c
+++ b/tools/testing/selftests/bpf/progs/verifier_subprog_callers.c
@@ -461,7 +461,7 @@ static __naked __noinline __used void saves_irq_flags_many_times(void)
SEC("tc")
__description("subprog comes back with more ids than there is room for")
-__failure __msg("too many ids at the call at insn")
+__failure __msg("too many ids")
__log_level(4)
int too_many_ids_at_the_return(void *ctx)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
index 041b7a71fb3..df9687514cc 100644
--- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
+++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
@@ -214,7 +214,7 @@ __msg("18: (95) exit")
__msg("returning from callee:")
__msg("to caller at 9:")
/* the callback changes nothing for the caller */
-__msg("from 18 to 9: within")
+__msg("from 18 to 9: pruned")
__msg("14: (0f) r1 += r6 ; R1=map_value(map=.data.vals,ks=4,vs=16,smin=smin32=0,smax=umax=smax32=umax32=12,var_off=(0x0; 0xc),step=0+4) R6=scalar(smin=smin32=0,smax=umax=smax32=umax32=12,var_off=(0x0; 0xc),step=0+4)")
__naked int callback_result_precise(void)
{
@@ -308,7 +308,7 @@ __msg("16: (95) exit")
__msg("returning from callee:")
__msg("to caller at 9:")
/* r6 is what it was before the callback */
-__msg("from 16 to 9: within")
+__msg("from 16 to 9: pruned")
__naked int parent_callee_saved_reg_precise_with_callback(void)
{
asm volatile (
@@ -414,7 +414,7 @@ __msg("18: (95) exit")
__msg("returning from callee:")
__msg("to caller at 10:")
/* fp-8 is what it was before the callback */
-__msg("from 18 to 10: within")
+__msg("from 18 to 10: pruned")
__naked int parent_stack_slot_precise_with_callback(void)
{
asm volatile (
@@ -648,7 +648,7 @@ SEC("socket")
__success
__log_level(2)
/* the early exit and the exit of the subprogram give the same state */
-__msg("from 10 to 2: within")
+__msg("from 10 to 2: pruned")
__msg("3: (0f) r1 += r7 ; R1=fp-8 R7=-8")
__naked int ld_abs_backtrack_both_paths(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_trip_count.c
similarity index 77%
rename from tools/testing/selftests/bpf/progs/verifier_scev.c
rename to tools/testing/selftests/bpf/progs/verifier_trip_count.c
index 8039a5d6769..c8e3ddbc530 100644
--- a/tools/testing/selftests/bpf/progs/verifier_scev.c
+++ b/tools/testing/selftests/bpf/progs/verifier_trip_count.c
@@ -10,12 +10,25 @@
#include "bpf_kfuncs.h"
#include "bpf_compiler.h"
+/* The state at insn @at has @min..@max in @xreg */
+#define RANGE(at, xreg, min, max) "to " at ": " RANGE_TOO(xreg, min, max)
+#define RANGE_TOO(xreg, min, max) \
+ "{{.*}}" xreg "={{[a-z_?0]+[(]([^)]*,)?}}smin={{([a-z0-9]+=)*}}" #min \
+ ",smax={{([a-z0-9]+=)*}}" #max "{{[,)]}}"
+
+/* No may_goto. __not_msg() searches only between the __msg() before it and the next one. */
+#define NO_GUARD __not_msg("has no trip count")
+#define COUNT(n) NO_GUARD __msg("has trip count " #n "{{$}}") NO_GUARD
+#define ANY_COUNT NO_GUARD __msg("has trip count") NO_GUARD
+#define ONCE NO_GUARD
+#define NO_COUNT __msg("has no trip count")
+
#define COND_TEST(name, _initial, _step, _bound, msg, body) \
SEC("xdp") \
__success \
__log_level(2) \
__flag(BPF_F_TEST_STATE_FREQ) \
-__msg("loop header at {{[0-9]+}}, " msg) \
+msg \
__naked void name(void) \
{ \
asm volatile ( \
@@ -47,11 +60,12 @@ __naked void name(void)
"if r6 " op " r7 goto 1b;")
/*
- * Bound fallback verification of otherwise infinite or extremely long loops.
- * JSET is not a linear condition, SCEV ignores it and the tested condition
- * alone decides the iteration count.
+ * Leave the loop in trip 8. r9 moves with the trip index, so a jump on r9 gives a trip count.
+ * r3 is r9 at run time. An atomic add hides that, so the tested condition alone gives the count.
*/
-#define LIMIT_ITERATIONS "r9 += 1; if r9 & 8 goto 2f;"
+#define LIMIT_ITERATIONS \
+ "r9 += 1; *(u64 *)(r10 - 24) = 0; lock *(u64 *)(r10 - 24) += r9;" \
+ "r3 = *(u64 *)(r10 - 24); if r3 & 8 goto 2f;"
#define PRE__COND_TEST2(name, op, initial, step, bound, msg) \
PRE__COND_TEST(name, op, initial, step, bound, msg, LIMIT_ITERATIONS)
@@ -116,8 +130,7 @@ struct bpf_iter__bpf_map_elem {
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
+__msg("loop at 1 has trip count 11{{$}}")
__naked void simple_loop1(void)
{
asm volatile (" \
@@ -135,8 +148,8 @@ exit_%=: \
SEC("xdp")
__success
__log_level(2)
-__msg("8: (7b) *(u64 *)(r1 +0) = r0 ; *fp-8 (+ *fp-8 1) -> %fp-8.8?")
-__naked void indirect_write_invalidates_scev(void)
+__msg("loop at 2 has trip count 11{{$}}")
+__naked void indirect_write_deletes_equation(void)
{
asm volatile (" \
r0 = 0; \
@@ -165,8 +178,8 @@ __noinline int global_noop_counter(__u64 *p)
SEC("xdp")
__success
__log_level(2)
-__msg("8: (85) call pc+{{[0-9]+}} {{.*}}; *fp-8 (+ *fp-8 1) -> %fp-8.8?")
-__naked void global_call_invalidates_scev(void)
+__msg("loop at 3 has trip count 3{{$}}")
+__naked void global_call_deletes_equation(void)
{
asm volatile (
"r0 = 0;"
@@ -189,11 +202,7 @@ __naked void global_call_invalidates_scev(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 2:")
-__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)")
-__msg_next(" scev at latch 3:")
-__msg_next(" r0=*fp-8 / (linear *fp-8 1)")
-__msg_next(" *fp-8=*fp-8 / (linear *fp-8 1)")
+__msg("loop at 2 has trip count 11{{$}}")
__naked void simple_loop2(void)
{
asm volatile (" \
@@ -215,8 +224,7 @@ exit_%=: \
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 2:")
-__msg_next(" r1=(+ r1 1) / (linear r1 1)")
+__msg("loop at 2 has trip count 3{{$}}")
__naked void meet_agrees(void)
{
asm volatile (" \
@@ -241,8 +249,7 @@ __naked void meet_agrees(void)
SEC("xdp")
__success __retval(0)
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r6=%r6.1=phi(r6, (+ r6 1)) / ?")
+__msg("loop at 1 has no trip count")
__naked void meet_disagrees(void)
{
asm volatile (" \
@@ -265,8 +272,7 @@ __naked void meet_disagrees(void)
SEC("socket")
__success
__log_level(2)
-__msg("scev at header 3:")
-__msg_next(" r6=%r6.9=phi((+ r6 _)) / ?")
+__msg("loop at 3 has trip count 10{{$}}")
__naked void phi_operands_have_no_defs(void)
{
asm volatile (" \
@@ -297,14 +303,7 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev at header 2:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg_next(" r1=42 / (any r1 42)")
-__msg_next(" scev at latch 2:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg_next(" r1=42 / (any r1 42)")
-__msg("loop header at 2, widening r0 to 0..10 step 1")
-__msg_next("loop header at 2, widening r1 to 7..42 step 1")
+__msg(RANGE("2", "R0", 0, 10) RANGE_TOO("R1", 7, 42))
__naked void entry_state_join(void)
{
asm volatile (" \
@@ -321,8 +320,7 @@ __naked void entry_state_join(void)
SEC("xdp")
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r6=(bswap32 (bswap32 (zext32 (- (- (zext32 (+ (>>...) 1))))))) / ?")
+__msg("loop at 1 has trip count 2{{$}}")
__naked void expr_chain(void)
{
asm volatile (" \
@@ -351,88 +349,7 @@ __naked void expr_chain(void)
SEC("xdp")
__success
__log_level(2)
-__msg("r1 += 2 {{.*}}; r1 r6 -> (+ r6 2)")
-__msg("r1 += r7 {{.*}}; r1 r6 -> (+ r6 r7)")
-__msg("w1 += 2 {{.*}}; r1 r6 -> (zext32 (+ r6 2))")
-__msg("w1 += w7 {{.*}}; r1 r6 -> (zext32 (+ r6 r7))")
-__msg("r1 -= 2 {{.*}}; r1 r6 -> (+ r6 -2)")
-__msg("r1 -= r7 {{.*}}; r1 r6 -> (- r6 r7)")
-__msg("w1 -= 2 {{.*}}; r1 r6 -> (zext32 (+ r6 -2))")
-__msg("w1 -= w7 {{.*}}; r1 r6 -> (zext32 (- r6 r7))")
-__msg("r1 -= 2 {{.*}}; r1 (+ r6 2) -> r6")
-__msg("r1 *= 2 {{.*}}; r1 r6 -> (* r6 2)")
-__msg("r1 *= r7 {{.*}}; r1 r6 -> (* r6 r7)")
-__msg("w1 *= 2 {{.*}}; r1 r6 -> (zext32 (* r6 2))")
-__msg("w1 *= w7 {{.*}}; r1 r6 -> (zext32 (* r6 r7))")
-__msg("r1 /= 2 {{.*}}; r1 r6 -> (/ r6 2)")
-__msg("r1 /= r7 {{.*}}; r1 r6 -> (/ r6 r7)")
-__msg("w1 /= 2 {{.*}}; r1 r6 -> (zext32 (/ (zext32 r6) 2))")
-__msg("w1 /= w7 {{.*}}; r1 r6 -> (zext32 (/ (zext32 r6) (zext32 r7)))")
-__msg("r1 s/= 2 {{.*}}; r1 r6 -> (s/ r6 2)")
-__msg("r1 s/= r7 {{.*}}; r1 r6 -> (s/ r6 r7)")
-__msg("w1 s/= 2 {{.*}}; r1 r6 -> (zext32 (s/ (sext32 r6) 2))")
-__msg("w1 s/= w7 {{.*}}; r1 r6 -> (zext32 (s/ (sext32 r6) (sext32 r7)))")
-__msg("r1 %= 2 {{.*}}; r1 r6 -> (% r6 2)")
-__msg("r1 %= r7 {{.*}}; r1 r6 -> (% r6 r7)")
-__msg("w1 %= 2 {{.*}}; r1 r6 -> (zext32 (% (zext32 r6) 2))")
-__msg("w1 %= w7 {{.*}}; r1 r6 -> (zext32 (% (zext32 r6) (zext32 r7)))")
-__msg("r1 s%= 2 {{.*}}; r1 r6 -> (s% r6 2)")
-__msg("r1 s%= r7 {{.*}}; r1 r6 -> (s% r6 r7)")
-__msg("w1 s%= 2 {{.*}}; r1 r6 -> (zext32 (s% (sext32 r6) 2))")
-__msg("w1 s%= w7 {{.*}}; r1 r6 -> (zext32 (s% (sext32 r6) (sext32 r7)))")
-__msg("r1 |= 2 {{.*}}; r1 r6 -> (| r6 2)")
-__msg("r1 |= r7 {{.*}}; r1 r6 -> (| r6 r7)")
-__msg("w1 |= 2 {{.*}}; r1 r6 -> (zext32 (| r6 2))")
-__msg("w1 |= w7 {{.*}}; r1 r6 -> (zext32 (| r6 r7))")
-__msg("r1 &= 2 {{.*}}; r1 r6 -> (& r6 2)")
-__msg("r1 &= r7 {{.*}}; r1 r6 -> (& r6 r7)")
-__msg("w1 &= 2 {{.*}}; r1 r6 -> (zext32 (& r6 2))")
-__msg("w1 &= w7 {{.*}}; r1 r6 -> (zext32 (& r6 r7))")
-__msg("r1 ^= 2 {{.*}}; r1 r6 -> (^ r6 2)")
-__msg("r1 ^= r7 {{.*}}; r1 r6 -> (^ r6 r7)")
-__msg("w1 ^= 2 {{.*}}; r1 r6 -> (zext32 (^ r6 2))")
-__msg("w1 ^= w7 {{.*}}; r1 r6 -> (zext32 (^ r6 r7))")
-__msg("r1 <<= 2 {{.*}}; r1 r6 -> (<< r6 2)")
-__msg("r1 <<= r7 {{.*}}; r1 r6 -> (<< r6 r7)")
-__msg("w1 <<= 2 {{.*}}; r1 r6 -> (zext32 (<< r6 2))")
-__msg("w1 <<= w7 {{.*}}; r1 r6 -> (zext32 (<< r6 (& r7 31)))")
-__msg("r1 >>= 2 {{.*}}; r1 r6 -> (>> r6 2)")
-__msg("r1 >>= r7 {{.*}}; r1 r6 -> (>> r6 r7)")
-__msg("w1 >>= 2 {{.*}}; r1 r6 -> (zext32 (>> (zext32 r6) 2))")
-__msg("w1 >>= w7 {{.*}}; r1 r6 -> (zext32 (>> (zext32 r6) (& r7 31)))")
-__msg("r1 s>>= 2 {{.*}}; r1 r6 -> (s>> r6 2)")
-__msg("r1 s>>= r7 {{.*}}; r1 r6 -> (s>> r6 r7)")
-__msg("w1 s>>= 2 {{.*}}; r1 r6 -> (zext32 (s>> (sext32 r6) 2))")
-__msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> (sext32 r6) (& r7 31)))")
-__msg("r1 = -r1 {{.*}}; r1 r6 -> (- r6)")
-__msg("w1 = -w1 {{.*}}; r1 r6 -> (zext32 (- r6))")
-__msg("r1 = 42 {{.*}}; r1 r6 -> 42")
-__msg("w1 = 42 {{.*}}; r1 r6 -> 42")
-__msg("r1 = r7 {{.*}}; r1 r6 -> r7")
-__msg("w1 = w7 {{.*}}; r1 r6 -> (zext32 r7)")
-__msg("r1 = (s8)r6 {{.*}}; r1 r6 -> (sext8 r6)")
-__msg("r1 = (s16)r6 {{.*}}; r1 r6 -> (sext16 r6)")
-__msg("r1 = (s32)r6 {{.*}}; r1 r6 -> (sext32 r6)")
-__msg("w1 = (s8)w6 {{.*}}; r1 r6 -> (zext32 (sext8 r6))")
-__msg("w1 = (s16)w6 {{.*}}; r1 r6 -> (zext32 (sext16 r6))")
-__msg("r1 = bswap16 r1 {{.*}}; r1 r6 -> (bswap16 r6)")
-__msg("r1 = bswap32 r1 {{.*}}; r1 r6 -> (bswap32 r6)")
-__msg("r1 = bswap64 r1 {{.*}}; r1 r6 -> (bswap64 r6)")
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (zext16 r6)")
-__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (zext32 r6)")
-__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)")
-__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (bswap16 r6)")
-__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (bswap32 r6)")
-__msg("r1 = be64 r1 {{.*}}; r1 r6 -> (bswap64 r6)")
-#else
-__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (bswap16 r6)")
-__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (bswap32 r6)")
-__msg("r1 = le64 r1 {{.*}}; r1 r6 -> (bswap64 r6)")
-__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (zext16 r6)")
-__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (zext32 r6)")
-__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)")
-#endif
+COUNT(2)
__naked void alu_ops_exprs(void)
{
asm volatile (
@@ -510,10 +427,6 @@ __naked void alu_ops_exprs(void)
ALU_OP("r1 = bswap64 r1")
ALU_OP("r1 = le16 r1")
ALU_OP("r1 = le32 r1")
- /*
- * Expressions unchanged by transfer() do not appear in the log.
- * Add r1 += 1 to touch the expression and make it appear in the log.
- */
ALU_OP("r1 = le64 r1; r1 += 1")
ALU_OP("r1 = be16 r1")
ALU_OP("r1 = be32 r1")
@@ -527,7 +440,7 @@ __naked void alu_ops_exprs(void)
#undef ALU_OP
-/* Reset operands and consume results to keep transfer changes live. */
+/* Reset the operands. Read the results, so they are live. */
#define ST_OP(stmt) \
"r0 = 0; r1 = 1; r2 = 2;" \
"*(u64 *)(r10 - 8) = -8;" \
@@ -538,49 +451,7 @@ __naked void alu_ops_exprs(void)
SEC("xdp")
__success
__log_level(2)
-__msg("*(u64 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> 1")
-__msg("*(u32 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill32 1)")
-__msg("*(u16 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill16 1)")
-__msg("*(u8 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill8 1)")
-__msg("r1 = *(u64 *)(r10 -8) {{.*}}; r1 1 -> -8")
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> 4294967288")
-__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (zext16 -8)")
-__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> (zext8 -8)")
-__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> (sext32 -8)")
-__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> (sext16 -8)")
-__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> (sext8 -8)")
-__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> (zext8 -8)")
-__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> (zext16 -8)")
-__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> 4294967288")
-#else
-__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> (>> -8 32)")
-__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (>> -8 48)")
-__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> (>> -8 56)")
-__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> (s>> -8 32)")
-__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> (s>> -8 48)")
-__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> (s>> -8 56)")
-__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> (>> -8 56)")
-__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> (>> -8 48)")
-__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> (>> -8 32)")
-#endif
-__msg("r2 = load_acquire((u64 *)(r10 -8)) {{.*}}; r2 2 -> -8")
-__msg("store_release((u8 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill8 1)")
-__msg("store_release((u16 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill16 1)")
-__msg("store_release((u32 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill32 1)")
-__msg("store_release((u64 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> 1")
-__msg("lock *(u64 *)(r10 -8) += r1 {{.*}}; *fp-8 -8 -> %fp-8.{{[0-9]+}}")
-__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> %r1.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}")
-__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1) {{.*}}; r1 1 -> %r1.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}")
-__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> %r1.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}")
-__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> %r0.{{[0-9]+}}, *fp-8 -8 -> %fp-8.{{[0-9]+}}")
-__msg("*(u64 *)(r10 -8) = 42 {{.*}}; *fp-8 -8 -> 42")
-__msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> %fp-8.{{[0-9]+}}")
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-/* a fill that is not wider than the spill reads what was spilled */
-__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 r7 -> (sext8 r7)")
-__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 r7 -> (zext16 (spill8 r7))")
-#endif
+COUNT(2)
__naked void store_load_exprs(void)
{
asm volatile (
@@ -646,16 +517,7 @@ __naked void store_load_exprs(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" r7=%r7.4?")
-__msg_next(" scev at latch 1:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" r7=%r7.4?")
-__msg_next("scev at header 4:")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
-__msg_next(" scev at latch 4:")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
+__msg("loop at 1 has trip count 2{{$}}")
__naked void nested_loop1(void)
{
asm volatile (" \
@@ -682,19 +544,11 @@ SEC("xdp")
__success
__msg("loop at 1")
__msg_next(" backedge from 4{{$}}")
-__msg_next(" exit from 1 to 7")
+__msg_next(" first exit to 7")
__msg_next("loop at 4, nested in 1")
__msg_next(" backedge from 6, latch at 4")
-__msg_next(" exit from 4 to 1")
-__msg("scev at header 1:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" r7=%r7.4?")
-__msg_next("scev at header 4:")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
-__msg_next(" scev at latch 4:")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
-__msg("loop header at 1, latch not identified")
-__msg("loop header at 4, widening r7 to 0..2 step 1")
+__msg_next(" first exit to 1")
+__msg(RANGE("4", "R7", 0, 2))
__log_level(2)
__naked void nested_loop_hdr_backedge1(void)
{
@@ -718,7 +572,7 @@ __naked void nested_loop_hdr_backedge1(void)
* Outer loop {2, 3, 4, 5} contains inner loop {3, 4, 5}.
* The outer loop's only backedge is 4 -> 2, and its latch is also at 4,
* inside the inner loop. Outer loop's latch index is not computed in
- * such a case, as there is no mechanism to build SCEVs for such latches
+ * such a case, as there is no trip count for such latches
* at the moment.
*/
SEC("xdp")
@@ -727,20 +581,11 @@ __log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
__msg("loop at 2{{$}}")
__msg_next(" backedge from 4{{$}}")
-__msg_next(" exit from 5 to 6")
+__msg_next(" first exit to 6")
__msg_next("loop at 3, nested in 2")
__msg_next(" backedge from 5, latch at 5")
-__msg_next(" exit from 4 to 2")
-__msg_next(" exit from 5 to 6")
-__msg("scev at header 2:")
-__msg_next(" r6=%r6.3?")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
-__msg_next("scev at header 3:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" scev at latch 5:")
-__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)")
-__msg("loop header at 2, latch not identified")
-__msg("loop header at 3, header_count is [0..2] ")
+__msg_next(" first exit to 2")
+__msg("loop at 3 has trip count 2{{$}}")
__naked void nested_loop_hdr_backedge2(void)
{
asm volatile (" \
@@ -759,19 +604,15 @@ __naked void nested_loop_hdr_backedge2(void)
* Logical nest: for (r6 = 0; r6 < 3; r6++)
* for (r7 = 0; r7 < 4; r7++) r8++;
* Both backedges target header 3, so the CFG has one loop. r6 advances only
- * on the outer backedge and r7 resets there; neither has a linear SCEV.
- * r8 advances on both backedges and retains its linear SCEV.
+ * on the outer backedge and r7 resets there; neither moves with the trip index.
+ * r8 advances on both backedges and moves with the trip index.
*/
SEC("socket")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
__msg("loop at 3{{$}}")
-__msg("scev at header 3:")
-__msg_next(" r6=%r6.3=phi(r6, (+ r6 1)) / ?")
-__msg_next(" r7=%r7.3=phi((+ r7 1), 0) / ?")
-__msg_next(" r8=(+ r8 1) / (linear r8 1)")
-__msg("loop header at 3, can't compute iterations count")
+__msg("loop at 3 has no trip count")
__naked void nested_loops_shared_header(void)
{
asm volatile (" \
@@ -792,8 +633,8 @@ __naked void nested_loops_shared_header(void)
SEC("socket")
__success
__log_level(2)
-__msg("loop header at 1, header_count is 10")
-__msg("loop header at 5, header_count is 5")
+__msg("loop at 5 has trip count 5{{$}}")
+__msg("loop at 1 has trip count 10{{$}}")
__flag(BPF_F_TEST_STATE_FREQ)
__naked void nested_loops_outer_latch_true_branch_to_inner(void)
{
@@ -822,26 +663,26 @@ __naked void nested_loops_outer_latch_true_branch_to_inner(void)
* }
*/
/* name op initial step bound expected message */
-PRE__COND_TEST (pre_lt, ">=", 1, 2, 7, "header_count is 4 ")
-POST_COND_TEST (post_lt, "<", 1, 2, 7, "header_count is 3 ")
-PRE__COND_TEST (pre_le, ">", 1, 2, 7, "header_count is 5 ")
-POST_COND_TEST (post_le, "<=", 1, 2, 7, "header_count is 4 ")
-PRE__COND_TEST (pre_gt, "<=", 9, -2, 3, "header_count is 4 ")
-POST_COND_TEST (post_gt, ">", 9, -2, 3, "header_count is 3 ")
-PRE__COND_TEST (pre_ge, "<", 9, -2, 3, "header_count is 5 ")
-POST_COND_TEST (post_ge, ">=", 9, -2, 3, "header_count is 4 ")
-PRE__COND_TEST (pre_slt, "s>=", -5, 2, 1, "header_count is 4 ")
-POST_COND_TEST (post_slt, "s<", -5, 2, 1, "header_count is 3 ")
-PRE__COND_TEST (pre_sle, "s>", -5, 2, 1, "header_count is 5 ")
-POST_COND_TEST (post_sle, "s<=", -5, 2, 1, "header_count is 4 ")
-PRE__COND_TEST (pre_sgt, "s<=", 5, -2, -1, "header_count is 4 ")
-POST_COND_TEST (post_sgt, "s>", 5, -2, -1, "header_count is 3 ")
-PRE__COND_TEST (pre_sge, "s<", 5, -2, -1, "header_count is 5 ")
-POST_COND_TEST (post_sge, "s>=", 5, -2, -1, "header_count is 4 ")
-PRE__COND_TEST (pre_ne, "==", 1, 2, 7, "header_count is 4 ")
-POST_COND_TEST (post_ne, "!=", 1, 2, 7, "header_count is 3 ")
-PRE__COND_TEST (pre_ne_down, "==", 9, -2, 3, "header_count is 4 ")
-POST_COND_TEST (post_ne_down, "!=", 9, -2, 3, "header_count is 3 ")
+PRE__COND_TEST (pre_lt, ">=", 1, 2, 7, COUNT(4))
+POST_COND_TEST (post_lt, "<", 1, 2, 7, COUNT(3))
+PRE__COND_TEST (pre_le, ">", 1, 2, 7, COUNT(5))
+POST_COND_TEST (post_le, "<=", 1, 2, 7, COUNT(4))
+PRE__COND_TEST (pre_gt, "<=", 9, -2, 3, COUNT(4))
+POST_COND_TEST (post_gt, ">", 9, -2, 3, COUNT(3))
+PRE__COND_TEST (pre_ge, "<", 9, -2, 3, COUNT(5))
+POST_COND_TEST (post_ge, ">=", 9, -2, 3, COUNT(4))
+PRE__COND_TEST (pre_slt, "s>=", -5, 2, 1, COUNT(4))
+POST_COND_TEST (post_slt, "s<", -5, 2, 1, COUNT(3))
+PRE__COND_TEST (pre_sle, "s>", -5, 2, 1, COUNT(5))
+POST_COND_TEST (post_sle, "s<=", -5, 2, 1, COUNT(4))
+PRE__COND_TEST (pre_sgt, "s<=", 5, -2, -1, COUNT(4))
+POST_COND_TEST (post_sgt, "s>", 5, -2, -1, COUNT(3))
+PRE__COND_TEST (pre_sge, "s<", 5, -2, -1, COUNT(5))
+POST_COND_TEST (post_sge, "s>=", 5, -2, -1, COUNT(4))
+PRE__COND_TEST (pre_ne, "==", 1, 2, 7, COUNT(4))
+POST_COND_TEST (post_ne, "!=", 1, 2, 7, COUNT(3))
+PRE__COND_TEST (pre_ne_down, "==", 9, -2, 3, COUNT(4))
+POST_COND_TEST (post_ne_down, "!=", 9, -2, 3, COUNT(3))
/*
* Bounds between two consecutive counter values.
@@ -853,14 +694,14 @@ POST_COND_TEST (post_ne_down, "!=", 9, -2,
* r6 += 2; // r6 ∈ [3,5,7,9]
* }
*/
-PRE__COND_TEST (pre_lt_round, ">=", 1, 2, 8, "header_count is 5 ")
-PRE__COND_TEST (pre_le_round, ">", 1, 2, 8, "header_count is 5 ")
-PRE__COND_TEST (pre_gt_round, "<=", 9, -2, 2, "header_count is 5 ")
-PRE__COND_TEST (pre_ge_round, "<", 9, -2, 2, "header_count is 5 ")
-PRE__COND_TEST (pre_slt_round, "s>=", -5, 2, 2, "header_count is 5 ")
-PRE__COND_TEST (pre_sle_round, "s>", -5, 2, 2, "header_count is 5 ")
-PRE__COND_TEST (pre_sgt_round, "s<=", 5, -2, -2, "header_count is 5 ")
-PRE__COND_TEST (pre_sge_round, "s<", 5, -2, -2, "header_count is 5 ")
+PRE__COND_TEST (pre_lt_round, ">=", 1, 2, 8, COUNT(5))
+PRE__COND_TEST (pre_le_round, ">", 1, 2, 8, COUNT(5))
+PRE__COND_TEST (pre_gt_round, "<=", 9, -2, 2, COUNT(5))
+PRE__COND_TEST (pre_ge_round, "<", 9, -2, 2, COUNT(5))
+PRE__COND_TEST (pre_slt_round, "s>=", -5, 2, 2, COUNT(5))
+PRE__COND_TEST (pre_sle_round, "s>", -5, 2, 2, COUNT(5))
+PRE__COND_TEST (pre_sgt_round, "s<=", 5, -2, -2, COUNT(5))
+PRE__COND_TEST (pre_sge_round, "s<", 5, -2, -2, COUNT(5))
/*
* Equality on the first non-strict comparison still takes a backedge.
@@ -872,10 +713,10 @@ PRE__COND_TEST (pre_sge_round, "s<", 5, -2,
* r6 += 2; // r6 ∈ [9]
* }
*/
-PRE__COND_TEST (pre_le_eq, ">", 7, 2, 7, "header_count is 2 ")
-PRE__COND_TEST (pre_ge_eq, "<", 3, -2, 3, "header_count is 2 ")
-PRE__COND_TEST (pre_sle_eq, "s>", 1, 2, 1, "header_count is 2 ")
-PRE__COND_TEST (pre_sge_eq, "s<", -1, -2, -1, "header_count is 2 ")
+PRE__COND_TEST (pre_le_eq, ">", 7, 2, 7, COUNT(2))
+PRE__COND_TEST (pre_ge_eq, "<", 3, -2, 3, COUNT(2))
+PRE__COND_TEST (pre_sle_eq, "s>", 1, 2, 1, COUNT(2))
+PRE__COND_TEST (pre_sge_eq, "s<", -1, -2, -1, COUNT(2))
/*
* Unsigned order across the sign bit.
@@ -887,8 +728,8 @@ PRE__COND_TEST (pre_sge_eq, "s<", -1, -2,
* r6 += 1; // r6 ∈ [H-1,H,H+1]
* }
*/
-PRE__COND_TEST (pre_lt_sign, ">=", ITER_S64_MAX - 1, 1, ITER_S64_MIN + 1, "header_count is 4 ")
-PRE__COND_TEST (pre_gt_sign, "<=", ITER_S64_MIN + 1, -1, ITER_S64_MAX - 1, "header_count is 4 ")
+PRE__COND_TEST (pre_lt_sign, ">=", ITER_S64_MAX - 1, 1, ITER_S64_MIN + 1, COUNT(4))
+PRE__COND_TEST (pre_gt_sign, "<=", ITER_S64_MIN + 1, -1, ITER_S64_MAX - 1, COUNT(4))
/*
* The first latch comparison is false.
@@ -900,15 +741,15 @@ PRE__COND_TEST (pre_gt_sign, "<=", ITER_S64_MIN + 1, -1, ITER_S
* r6 += 2; // unreachable
* }
*/
-PRE__COND_TEST (pre_lt_false, ">=", 7, 2, 7, "header_count is 1 ")
-PRE__COND_TEST (pre_le_false, ">", 8, 2, 7, "header_count is 1 ")
-PRE__COND_TEST (pre_gt_false, "<=", 3, -2, 3, "header_count is 1 ")
-PRE__COND_TEST (pre_ge_false, "<", 2, -2, 3, "header_count is 1 ")
-PRE__COND_TEST (pre_slt_false, "s>=", 1, 2, 1, "header_count is 1 ")
-PRE__COND_TEST (pre_sle_false, "s>", 2, 2, 1, "header_count is 1 ")
-PRE__COND_TEST (pre_sgt_false, "s<=", -1, -2, -1, "header_count is 1 ")
-PRE__COND_TEST (pre_sge_false, "s<", -2, -2, -1, "header_count is 1 ")
-PRE__COND_TEST (pre_ne_false, "==", 7, 2, 7, "header_count is 1 ")
+PRE__COND_TEST (pre_lt_false, ">=", 7, 2, 7, ONCE)
+PRE__COND_TEST (pre_le_false, ">", 8, 2, 7, ONCE)
+PRE__COND_TEST (pre_gt_false, "<=", 3, -2, 3, ONCE)
+PRE__COND_TEST (pre_ge_false, "<", 2, -2, 3, ONCE)
+PRE__COND_TEST (pre_slt_false, "s>=", 1, 2, 1, ONCE)
+PRE__COND_TEST (pre_sle_false, "s>", 2, 2, 1, ONCE)
+PRE__COND_TEST (pre_sgt_false, "s<=", -1, -2, -1, ONCE)
+PRE__COND_TEST (pre_sge_false, "s<", -2, -2, -1, ONCE)
+PRE__COND_TEST (pre_ne_false, "==", 7, 2, 7, ONCE)
/*
* An unsigned counter goes down and leaves the range below 0.
@@ -922,12 +763,12 @@ PRE__COND_TEST (pre_ne_false, "==", 7, 2,
*
* From above S64_MAX it's a long way down.
*/
-PRE__COND_TEST2(pre_lt_dir, ">=", 5, -2, 7, "header_count is [0..4] ")
-PRE__COND_TEST2(pre_le_dir, ">", 5, -2, 7, "header_count is [0..4] ")
-PRE__COND_TEST2(pre_lt_dir_high, ">=", ITER_S64_MIN + 5, -2, ITER_S64_MIN + 7, "can't compute iterations count")
+PRE__COND_TEST2(pre_lt_dir, ">=", 5, -2, 7, COUNT(4))
+PRE__COND_TEST2(pre_le_dir, ">", 5, -2, 7, COUNT(4))
+PRE__COND_TEST2(pre_lt_dir_high, ">=", ITER_S64_MIN + 5, -2, ITER_S64_MIN + 7, NO_COUNT)
/*
- * The counter moves away from the exit; bound fallback verification.
+ * The counter moves away from the exit.
*
* Example: pre_gt_dir
*
@@ -936,12 +777,12 @@ PRE__COND_TEST2(pre_lt_dir_high, ">=", ITER_S64_MIN + 5, -2, ITER_S
* r6 += 2; // r6 ∈ [7,9,...,19]
* }
*/
-PRE__COND_TEST2(pre_gt_dir, "<=", 5, 2, 3, "can't compute iterations count")
-PRE__COND_TEST2(pre_ge_dir, "<", 5, 2, 3, "can't compute iterations count")
-PRE__COND_TEST2(pre_slt_dir, "s>=", -1, -2, 1, "can't compute iterations count")
-PRE__COND_TEST2(pre_sle_dir, "s>", -1, -2, 1, "can't compute iterations count")
-PRE__COND_TEST2(pre_sgt_dir, "s<=", 1, 2, -1, "can't compute iterations count")
-PRE__COND_TEST2(pre_sge_dir, "s<", 1, 2, -1, "can't compute iterations count")
+PRE__COND_TEST2(pre_gt_dir, "<=", 5, 2, 3, NO_COUNT)
+PRE__COND_TEST2(pre_ge_dir, "<", 5, 2, 3, NO_COUNT)
+PRE__COND_TEST2(pre_slt_dir, "s>=", -1, -2, 1, NO_COUNT)
+PRE__COND_TEST2(pre_sle_dir, "s>", -1, -2, 1, NO_COUNT)
+PRE__COND_TEST2(pre_sgt_dir, "s<=", 1, 2, -1, NO_COUNT)
+PRE__COND_TEST2(pre_sge_dir, "s<", 1, 2, -1, NO_COUNT)
/*
* The predicted exit would wrap the counter.
@@ -953,14 +794,14 @@ PRE__COND_TEST2(pre_sge_dir, "s<", 1, 2,
* r6 += 2; // r6 ∈ [0,2,...,14]
* }
*/
-PRE__COND_TEST2(pre_lt_wrap, ">=", ITER_U64_MAX - 1, 2, ITER_U64_MAX, "can't compute iterations count")
-PRE__COND_TEST2(pre_le_wrap, ">", ITER_U64_MAX, 1, ITER_U64_MAX, "can't compute iterations count")
-PRE__COND_TEST2(pre_gt_wrap, "<=", 1, -2, 0, "can't compute iterations count")
-PRE__COND_TEST2(pre_ge_wrap, "<", 0, -1, 0, "can't compute iterations count")
-PRE__COND_TEST2(pre_slt_wrap, "s>=", ITER_S64_MAX - 1, 2, ITER_S64_MAX, "can't compute iterations count")
-PRE__COND_TEST2(pre_sle_wrap, "s>", ITER_S64_MAX, 1, ITER_S64_MAX, "can't compute iterations count")
-PRE__COND_TEST2(pre_sgt_wrap, "s<=", ITER_S64_MIN + 1, -2, ITER_S64_MIN, "can't compute iterations count")
-PRE__COND_TEST2(pre_sge_wrap, "s<", ITER_S64_MIN, -1, ITER_S64_MIN, "can't compute iterations count")
+PRE__COND_TEST2(pre_lt_wrap, ">=", ITER_U64_MAX - 1, 2, ITER_U64_MAX, NO_COUNT)
+PRE__COND_TEST2(pre_le_wrap, ">", ITER_U64_MAX, 1, ITER_U64_MAX, NO_COUNT)
+PRE__COND_TEST2(pre_gt_wrap, "<=", 1, -2, 0, NO_COUNT)
+PRE__COND_TEST2(pre_ge_wrap, "<", 0, -1, 0, NO_COUNT)
+PRE__COND_TEST2(pre_slt_wrap, "s>=", ITER_S64_MAX - 1, 2, ITER_S64_MAX, NO_COUNT)
+PRE__COND_TEST2(pre_sle_wrap, "s>", ITER_S64_MAX, 1, ITER_S64_MAX, NO_COUNT)
+PRE__COND_TEST2(pre_sgt_wrap, "s<=", ITER_S64_MIN + 1, -2, ITER_S64_MIN, NO_COUNT)
+PRE__COND_TEST2(pre_sge_wrap, "s<", ITER_S64_MIN, -1, ITER_S64_MIN, NO_COUNT)
/*
* The non-strict count itself would overflow u64.
@@ -972,10 +813,10 @@ PRE__COND_TEST2(pre_sge_wrap, "s<", ITER_S64_MIN, -1, IT
* r6 += 1; // r6 ∈ [1..8]
* }
*/
-PRE__COND_TEST2(pre_le_count_ovf, ">", 0, 1, ITER_U64_MAX, "can't compute iterations count")
-PRE__COND_TEST2(pre_ge_count_ovf, "<", ITER_U64_MAX, -1, 0, "can't compute iterations count")
-PRE__COND_TEST2(pre_sle_count_ovf, "s>", ITER_S64_MIN, 1, ITER_S64_MAX, "can't compute iterations count")
-PRE__COND_TEST2(pre_sge_count_ovf, "s<", ITER_S64_MAX, -1, ITER_S64_MIN, "can't compute iterations count")
+PRE__COND_TEST2(pre_le_count_ovf, ">", 0, 1, ITER_U64_MAX, NO_COUNT)
+PRE__COND_TEST2(pre_ge_count_ovf, "<", ITER_U64_MAX, -1, 0, NO_COUNT)
+PRE__COND_TEST2(pre_sle_count_ovf, "s>", ITER_S64_MIN, 1, ITER_S64_MAX, NO_COUNT)
+PRE__COND_TEST2(pre_sge_count_ovf, "s<", ITER_S64_MAX, -1, ITER_S64_MIN, NO_COUNT)
/*
* JNE divisibility and equality across unsigned wraparound.
@@ -987,24 +828,24 @@ PRE__COND_TEST2(pre_sge_count_ovf, "s<", ITER_S64_MAX, -1, IT
* r6 += 2; // r6 ∈ [3,5,...,17]
* }
*/
-PRE__COND_TEST2(pre_ne_rem, "==", 1, 2, 8, "can't compute iterations count")
-PRE__COND_TEST2(pre_ne_rem_down, "==", 9, -2, 2, "can't compute iterations count")
-PRE__COND_TEST (pre_ne_wrap_up, "==", ITER_U64_MAX - 1, 2, 0, "header_count is 2 ")
-PRE__COND_TEST (pre_ne_wrap_down, "==", 1, -2, ITER_U64_MAX, "header_count is 2 ")
+PRE__COND_TEST2(pre_ne_rem, "==", 1, 2, 8, NO_COUNT)
+PRE__COND_TEST2(pre_ne_rem_down, "==", 9, -2, 2, NO_COUNT)
+PRE__COND_TEST (pre_ne_wrap_up, "==", ITER_U64_MAX - 1, 2, 0, COUNT(2))
+PRE__COND_TEST (pre_ne_wrap_down, "==", 1, -2, ITER_U64_MAX, COUNT(2))
/*
- * Largest supported header count, sentinel, and u32 overflow.
+ * The largest trip index that gives a count is 2^32.
*
- * Example: pre_lt_max_hdrs
+ * Example: pre_lt_max_trips
*
* u64 r6 = 0, r9 = 0;
- * while (++r9 <= 8 && r6 < U32_MAX - 2) { // r6 ∈ [0..8]
+ * while (++r9 <= 8 && r6 < U32_MAX + 1) { // r6 ∈ [0..8]
* r6 += 1; // r6 ∈ [1..8]
* }
*/
-PRE__COND_TEST2(pre_lt_max_hdrs, ">=", 0, 1, ITER_U32_MAX - 2, "header_count is [0..4294967294] ")
-PRE__COND_TEST2(pre_lt_sentinel, ">=", 0, 1, ITER_U32_MAX - 1, "can't compute iterations count")
-PRE__COND_TEST2(pre_lt_hdr_ovf, ">=", 0, 1, ITER_U32_MAX, "can't compute iterations count")
+PRE__COND_TEST2(pre_lt_u32_max, ">=", 0, 1, ITER_U32_MAX, COUNT(4294967296))
+PRE__COND_TEST2(pre_lt_max_trips, ">=", 0, 1, ITER_U32_MAX + 1, COUNT(4294967297))
+PRE__COND_TEST2(pre_lt_too_long, ">=", 0, 1, ITER_U32_MAX + 2, NO_COUNT)
/*
* The extreme signed step values remain valid unsigned magnitudes.
@@ -1016,9 +857,9 @@ PRE__COND_TEST2(pre_lt_hdr_ovf, ">=", 0, 1, IT
* r6 += S64_MAX; // r6 ∈ [S64_MAX]
* }
*/
-PRE__COND_TEST (pre_lt_step_max, ">=", 0, ITER_S64_MAX, ITER_S64_MAX, "header_count is 2 ")
-PRE__COND_TEST (pre_gt_step_min, "<=", ITER_S64_MIN, ITER_S64_MIN, 0, "header_count is 2 ")
-PRE__COND_TEST (pre_ne_step_min, "==", 0, ITER_S64_MIN, ITER_S64_MIN, "header_count is 2 ")
+PRE__COND_TEST (pre_lt_step_max, ">=", 0, ITER_S64_MAX, ITER_S64_MAX, COUNT(2))
+PRE__COND_TEST (pre_gt_step_min, "<=", ITER_S64_MIN, ITER_S64_MIN, 0, COUNT(2))
+PRE__COND_TEST (pre_ne_step_min, "==", 0, ITER_S64_MIN, ITER_S64_MIN, COUNT(2))
/*
* A zero step cannot establish a finite iteration count.
@@ -1030,57 +871,57 @@ PRE__COND_TEST (pre_ne_step_min, "==", 0, ITER_S64_MIN, IT
* r6 += 0; // r6 ∈ [1]
* }
*/
-PRE__COND_TEST2(pre_lt_zero_step, ">=", 1, 0, 7, "can't compute iterations count")
+PRE__COND_TEST2(pre_lt_zero_step, ">=", 1, 0, 7, NO_COUNT)
/*
* Signed JMP32 comparisons sign-extend the initial value, bound and step.
* Encode negative operands as positive u32 constants so the conversion matters.
- * post32_sgt also checks that an exit exactly at S32_MIN is accepted.
*/
/* name op initial step bound expected message */
-POST_COND_TEST32(post32_slt, "s<", -7u, 2, -1u, "header_count is [0..3] ")
-POST_COND_TEST32(post32_sle, "s<=", -7u, 2, -1u, "header_count is [0..4] ")
-POST_COND_TEST32(post32_sgt, "s>", (__u32)INT32_MIN + 6, -2u, (__u32)INT32_MIN, "header_count is [0..3] ")
-POST_COND_TEST32(post32_sge, "s>=", -1u, -2u, -7u, "header_count is [0..4] ")
+POST_COND_TEST32(post32_slt, "s<", -7u, 2, -1u, ANY_COUNT)
+POST_COND_TEST32(post32_sle, "s<=", -7u, 2, -1u, ANY_COUNT)
+POST_COND_TEST32(post32_sgt, "s>", (__u32)INT32_MIN + 6, -2u, (__u32)INT32_MIN, NO_COUNT)
+POST_COND_TEST32(post32_sge, "s>=", -1u, -2u, -7u, NO_COUNT)
/*
- * For a JMP32 comparison scev.c:compute_max_iters() takes the low 32 bits of the
+ * For a JMP32 comparison the trip count comes from the low 32 bits of the
* initial value, the bound and the step.
* Hence pick negative 64-bit initial values and bounds, and encode negative
* steps as positive u32 constants, so that the upper 32 bits differ.
* post32_lt also checks that an exit exactly at U32_MAX is accepted.
*/
-POST_COND_TEST32(post32_lt, "<", -7, 2, -1, "header_count is [0..3] ")
-POST_COND_TEST32(post32_le, "<=", -9, 2, -3, "header_count is [0..4] ")
-POST_COND_TEST32(post32_gt, ">", -1, -2u, -7, "header_count is [0..3] ")
-POST_COND_TEST32(post32_ge, ">=", -1, -2u, -7, "header_count is [0..4] ")
-POST_COND_TEST32(post32_ne, "!=", -1, -2u, -7, "header_count is [0..3] ")
+POST_COND_TEST32(post32_lt, "<", -7, 2, -1, ANY_COUNT)
+POST_COND_TEST32(post32_le, "<=", -9, 2, -3, ANY_COUNT)
+POST_COND_TEST32(post32_gt, ">", -1, -2u, -7, NO_COUNT)
+POST_COND_TEST32(post32_ge, ">=", -1, -2u, -7, NO_COUNT)
+POST_COND_TEST32(post32_ne, "!=", -1, -2u, -7, NO_COUNT)
/* across 0: '!=' has no order. From 4 up to 1 goes all the way around. */
-POST_COND_TEST32(post32_ne_to_m1, "!=", 3, -1u, -1, "header_count is [0..4] ")
-POST_COND_TEST32(post32_ne_to_0, "!=", -4, 1, 0, "header_count is [0..4] ")
-POST_COND_TEST32(post32_ne_to_m1_2, "!=", 3, -2u, -1, "header_count is [0..2] ")
-POST_COND_TEST32(post32_ne_miss, "!=", 4, -2u, -1, "can't compute iterations count")
-POST_COND_TEST32(post32_ne_around, "!=", 4, 1, 1, "header_count is [0..4294967293] ")
+POST_COND_TEST32(post32_ne_to_m1, "!=", 3, -1u, -1, COUNT(4))
+POST_COND_TEST32(post32_ne_to_0, "!=", -4, 1, 0, ANY_COUNT)
+POST_COND_TEST32(post32_ne_to_m1_2, "!=", 3, -2u, -1, COUNT(2))
+POST_COND_TEST32(post32_ne_miss, "!=", 4, -2u, -1, NO_COUNT)
+POST_COND_TEST32(post32_ne_around, "!=", 4, 1, 1,
+ COUNT(4294967293))
/* The variable moves away from the bound and wraps: -6, -4, -2, 0. From -7 it steps over 0. */
-POST_COND_TEST32(post32_ge_away, ">=", -8, 2, 5, "header_count is [0..4] ")
-POST_COND_TEST32(post32_ge_away_miss, ">=", -7, 2, 1, "can't compute iterations count")
+POST_COND_TEST32(post32_ge_away, ">=", -8, 2, 5, ANY_COUNT)
+POST_COND_TEST32(post32_ge_away_miss, ">=", -7, 2, 1, NO_COUNT)
/* Only the jump has 32 bits. The upper half of r6 does not count. */
-COND_TEST(post32_jump_wide, 1ULL << 32, 1, 5, "header_count is [0..5] ",
+COND_TEST(post32_jump_wide, 1ULL << 32, 1, 5, COUNT(5),
"1: r8 = %[step] ll;" LIMIT_ITERATIONS "r6 += r8; if w6 < w7 goto 1b;")
/* Only the addition has 32 bits. No zero-extended number is above -42. */
-COND_TEST(post32_zext_gt_wide, 14, -7, -42, "header_count is [0..1] ",
+COND_TEST(post32_zext_gt_wide, 14, -7, -42, ONCE,
"1: r8 = %[step] ll;" LIMIT_ITERATIONS "w6 += w8; if r6 > r7 goto 1b;")
/* An exit exactly at S32_MAX is accepted as well. */
-POST_COND_TEST32(post32_slt_max, "s<", INT32_MAX - 6, 2, INT32_MAX, "header_count is [0..3] ")
+POST_COND_TEST32(post32_slt_max, "s<", INT32_MAX - 6, 2, INT32_MAX, COUNT(3))
/*
* The first iteration fits in 32 bits, but the predicted exit overshoots the bound
* by one and wraps.
*/
-POST_COND_TEST32(post32_slt_wrap, "s<", INT32_MAX - 3, 2, INT32_MAX, "can't compute iterations count")
-POST_COND_TEST32(post32_sgt_wrap, "s>", (__u32)INT32_MIN + 3, -2u, (__u32)INT32_MIN, "can't compute iterations count")
-POST_COND_TEST32(post32_lt_wrap, "<", -4, 2, -1, "can't compute iterations count")
+POST_COND_TEST32(post32_slt_wrap, "s<", INT32_MAX - 3, 2, INT32_MAX, NO_COUNT)
+POST_COND_TEST32(post32_sgt_wrap, "s>", (__u32)INT32_MIN + 3, -2u, (__u32)INT32_MIN, NO_COUNT)
+POST_COND_TEST32(post32_lt_wrap, "<", -4, 2, -1, NO_COUNT)
/*
* The first value of the counter or the bound of the loop is a range: the verifier sees only the
@@ -1125,14 +966,8 @@ __naked void name(void) \
#define POST(add, jump) LOOP_HEAD add "; if " jump " goto 1b;"
/* 'count' is what the verifier has and what r9 is at run time. 'range' is r6 at the header. */
-#define COUNTED(count, range) \
- __retval(count) \
- __msg("loop header at {{[0-9]+}}, header_count is [0.." #count "] ") \
- __msg("loop header at {{[0-9]+}}, widening r6 to " range " ") \
- __msg("loop header at {{[0-9]+}}, clamping r6 to " range " ")
-#define NO_COUNT \
- __msg("loop header at {{[0-9]+}}, can't compute iterations count") \
- __not_msg("header_count is")
+#define COUNTED(count, min, max) \
+ __retval(count) NO_GUARD __msg(RANGE("{{[0-9]+}}", "R6", min, max)) COUNT(count)
#define M64 ITER_U64_MAX
#define M32 ITER_U32_MAX
@@ -1144,43 +979,43 @@ __naked void name(void) \
* r6 += 2;
*/
/* name sign first step bound loop expected */
-RANGE_TEST(range_lt, "", 1, 4, 2, 8, 12, PRE(">="), COUNTED(7, "1..13"))
-RANGE_TEST(range_le, "", 0, 2, 2, 6, 9, PRE(">"), COUNTED(6, "0..11"))
-RANGE_TEST(range_gt, "", 10, 14, -2, 3, 5, PRE("<="), COUNTED(7, "2..14"))
-RANGE_TEST(range_ge, "", 10, 14, -2, 4, 5, PRE("<"), COUNTED(7, "2..14"))
-RANGE_TEST(range_slt, "s", -5, 5, 2, 5, 7, PRE("s>="), COUNTED(7, "-5..8"))
-RANGE_TEST(range_ne_up, "", 0, 3, 1, 3, 6, PRE("=="), COUNTED(7, "0..6"))
-RANGE_TEST(range_ne_down, "", 8, 10, -1, 4, 8, PRE("=="), COUNTED(7, "4..10"))
+RANGE_TEST(range_lt, "", 1, 4, 2, 8, 12, PRE(">="), COUNTED(7, 1, 13))
+RANGE_TEST(range_le, "", 0, 2, 2, 6, 9, PRE(">"), COUNTED(6, 0, 11))
+RANGE_TEST(range_gt, "", 10, 14, -2, 3, 5, PRE("<="), COUNTED(7, 2, 14))
+RANGE_TEST(range_ge, "", 10, 14, -2, 4, 5, PRE("<"), COUNTED(7, 2, 14))
+RANGE_TEST(range_slt, "s", -5, 5, 2, 5, 7, PRE("s>="), COUNTED(7, -5, 8))
+RANGE_TEST(range_ne_up, "", 0, 3, 1, 3, 6, PRE("=="), COUNTED(7, 0, 6))
+RANGE_TEST(range_ne_down, "", 8, 10, -1, 4, 8, PRE("=="), COUNTED(7, 4, 10))
/* the order of unsigned numbers says nothing about -1..3, the order of signed numbers does */
-RANGE_TEST(range_ne_signed, "s", -3, -1, 1, -1, 3, PRE("=="), COUNTED(7, "-3..3"))
+RANGE_TEST(range_ne_signed, "s", -3, -1, 1, -1, 3, PRE("=="), COUNTED(7, -3, 3))
/* The jump tests r6 after the addition. At the header r6 is a step behind. */
RANGE_TEST(range_post_lt, "", 3, 10, 2, 14, 15, POST("r6 += r8", "r6 < r7"),
- COUNTED(6, "3..14"))
+ COUNTED(6, 3, 14))
/* 32 bits. r6 is negative at the entry and zero extended after that. */
RANGE_TEST(range32_slt, "s", -5, 5, 2, 7, 9, POST("w6 += w8", "w6 s< w7"),
- COUNTED(7, "-5..0xffffffff"))
+ COUNTED(7, -5, 0xffffffff))
/* Only the jump has 32 bits. It bounds the low half of r6, 'first + step * count' the rest. */
RANGE_TEST(range32_jump, "", 3, 10, 2, 13, 15, POST("r6 += r8", "w6 < w7"),
- COUNTED(6, "3..14"))
+ COUNTED(6, 3, 14))
/* 'first + step' is -1..0 + 1: a range that crosses 0 is a range */
RANGE_TEST(range_post_lt_crosses_0, "s", -1, 0, 1, 5, 5, POST("r6 += r8", "r6 < r7"),
- COUNTED(6, "-1..4"))
+ COUNTED(6, -1, 4))
/* First values for which the latch is false at once are seen at the header too */
-RANGE_TEST(range_first_after_bound, "", 0, 100, 1, 5, 5, PRE(">="), COUNTED(6, "0..100"))
+RANGE_TEST(range_first_after_bound, "", 0, 100, 1, 5, 5, PRE(">="), COUNTED(6, 0, 100))
-#define COUNTED_2G(range) \
- __msg("loop header at {{[0-9]+}}, header_count is [0..2147483{{[0-9]+}}] ") \
- __msg("loop header at {{[0-9]+}}, widening r6 to " range " ")
+#define COUNTED_2G(min, max) \
+ NO_GUARD __msg(RANGE("{{[0-9]+}}", "R6", min, max)) NO_GUARD \
+ __msg("loop at {{[0-9]+}} has trip count 2147483{{[0-9]+}}{{$}}") NO_GUARD
/* Too long to unroll: only the count ends the loop for the verifier */
RANGE_TEST(range_long, "", 0, 0, 1, 0, 0x7fffffff,
- "1: r8 = %[step] ll; if r6 >= r7 goto 2f; r6 += r8; goto 1b;", COUNTED_2G("0..0x7fffffff"))
+ "1: r8 = %[step] ll; if r6 >= r7 goto 2f; r6 += r8; goto 1b;", COUNTED_2G(0, 0x7fffffff))
/* 'first + step' wraps around for a part of the range: the loop starts anywhere */
RANGE_TEST(range_first_wraps_s64, "s", ITER_S64_MAX - 5, ITER_S64_MAX - 1, 2,
ITER_S64_MAX - 1, ITER_S64_MAX - 1, POST("r6 += r8", "r6 s< r7"), NO_COUNT)
RANGE_TEST(range_first_wraps_u32, "", 0, 3, -2, 5, 5, POST("w6 += w8", "w6 > w7"),
- COUNTED_2G("0..0xffffffff"))
+ ANY_COUNT)
RANGE_TEST(range_first_wraps_s32, "", INT32_MAX - 5, INT32_MAX - 1, 2, INT32_MAX - 1, INT32_MAX - 1,
- POST("w6 += w8", "w6 s< w7"), COUNTED_2G("0..0xffffffff"))
+ POST("w6 += w8", "w6 s< w7"), NO_COUNT)
/*
* The value that ends the loop can wrap around.
@@ -1200,7 +1035,7 @@ RANGE_TEST(range_ge_wrap, "", 5, 9, -2, 1, 3, PRE("<"
* M-26, M-25, M-24 end at M-2, M-1, M. M-23 wraps.
*/
RANGE_TEST(range_lt_near_max, "", M64 - 26, M64 - 24, 8, M64 - 2, M64 - 2, PRE(">="),
- COUNTED(4, "-27..-1"))
+ COUNTED(4, -27, -1))
RANGE_TEST(range_lt_near_max_wrap, "", M64 - 26, M64 - 23, 8, M64 - 2, M64 - 2, PRE(">="), NO_COUNT)
/* a 32-bit addition wraps around at 2^32, also under a 64-bit jump */
RANGE_TEST(range32_add_wrap, "", M32 - 9, M32 - 5, 2, M32 - 3, M32,
@@ -1209,16 +1044,16 @@ RANGE_TEST(range32_ne_far, "", M32 - 5, M32 - 3, 1, M32 - 1, M32 + 3,
POST("w6 += w8", "r6 != r7"), NO_COUNT)
/* the first jump tests what enters the loop, and that is not zero extended */
RANGE_TEST(range32_first_is_wide, "", 0, M32 + 9, 2, 15, 20,
- LOOP_HEAD "if r6 >= r7 goto 2f; w6 += w8; goto 1b;", NO_COUNT)
+ LOOP_HEAD "if r6 >= r7 goto 2f; w6 += w8; goto 1b;", COUNT(11))
/* An unsigned counter goes down and leaves the range below 0. */
-RANGE_TEST(range_lt_dir, "", 3, 10, -2, 15, 20, PRE(">="), COUNTED(7, "-2..10"))
-RANGE_TEST(range_le_dir, "", 3, 10, -2, 15, 20, PRE(">"), COUNTED(7, "-2..10"))
+RANGE_TEST(range_lt_dir, "", 3, 10, -2, 15, 20, PRE(">="), COUNTED(7, -2, 10))
+RANGE_TEST(range_le_dir, "", 3, 10, -2, 15, 20, PRE(">"), COUNTED(7, -2, 10))
/* 'for (i = n; i < 6; i--)': whatever n is */
RANGE_TEST(range_lt_dir_any_first, "", 0, M64, -1, 6, 6, PRE(">="),
- __retval(1) __msg("header_count is [0..7] "))
+ __retval(1) NO_COUNT)
/* Up from 100 while it is '>= 100..103', until it wraps to 84: the smallest bound has the count */
RANGE_TEST(range32_ge_away, "", 100, 100, 0x0fffffff, 100, 103,
- LOOP_HEAD "if w6 < w7 goto 2f; w6 += w8; goto 1b;", __msg("header_count is [0..17] "))
+ LOOP_HEAD "if w6 < w7 goto 2f; w6 += w8; goto 1b;", COUNT(17))
/* '>= 0' is always true, and so is '<= -1': -4..3 crosses 0 */
RANGE_TEST(range32_ge_min, "", 5, 5, 1, 0, 3, POST("w6 += w8", "w6 >= w7"), NO_COUNT)
RANGE_TEST(range32_le_max, "s", 0, 0, 2, -4, 3, POST("w6 += w8", "w6 <= w7"), NO_COUNT)
@@ -1230,7 +1065,7 @@ RANGE_TEST(range32_zext_sgt_neg, "s", 100, 100, -1, -3, 4, POST("w6 += w8", "r6
RANGE_TEST(range32_zext_slt_wide, "", M32 - 15, M32 - 15, 1, M32 - 5, M32 + 2,
POST("w6 += w8", "r6 s< r7"), NO_COUNT)
RANGE_TEST(range32_zext_slt_bit31, "", M32 / 2 - 15, M32 / 2 - 15, 1, M32 / 2 + 17, M32 / 2 + 17,
- POST("w6 += w8", "r6 s< r7"), __msg("header_count is [0..32] "))
+ POST("w6 += w8", "r6 s< r7"), __msg("has trip count 32{{$}}"))
/* The counter moves away from the exit. */
RANGE_TEST(range_gt_dir, "", 10, 20, 2, 3, 5, PRE("<="), NO_COUNT)
RANGE_TEST(range_ge_dir, "", 10, 20, 2, 3, 5, PRE("<"), NO_COUNT)
@@ -1244,7 +1079,7 @@ RANGE_TEST(range_ne_behind, "", 0, 4, 1, 3, 8, PRE("=="), NO_COUNT)
* and is less than 12, but steps over 12.
*/
RANGE_TEST(range_ne_mul24, "", 0, 0, 24, 0, 5, "r7 *= 24;" PRE("=="),
- COUNTED(6, "0..120"))
+ COUNTED(6, 0, 120))
RANGE_TEST(range_ne_mul12, "", 0, 0, 8, 0, 4, "r7 *= 12;" PRE("=="), NO_COUNT)
RANGE_TEST(range_ne_first_mul12, "", 0, 4, 8, 96, 96, "r6 *= 12;" PRE("=="), NO_COUNT)
/* The sum of a multiple of 3 and 0..1 is no multiple of 3 */
@@ -1257,26 +1092,25 @@ RANGE_TEST(range_ne_mul3_crosses_s64, "", 0, 3, 3, 0, 1,
NO_COUNT)
/* 'while (i-- > 0) sum += a[i];' of u16 a[]: the compiler counts 2 * i down to 0 */
RANGE_TEST(range_ne_shl1_down, "", 1, 5, -2, 0, 0, "r6 <<= 1;" POST("r6 += r8", "r6 != r7"),
- COUNTED(5, "2..10"))
+ COUNTED(5, 2, 10))
/* 20 modulo 24 plus multiples of 8: the induction variable is 4 modulo gcd(24, 8) */
RANGE_TEST(range_ne_first_mul24, "", 1, 2, 8, 84, 84, "r6 *= 24; r6 += 20;" PRE("=="),
- __retval(6) __msg("header_count is [0..6] ") __msg("widening r6 to 44..84 step 8")
- __msg("step=4+8"))
+ __retval(6) __msg(RANGE("{{[0-9]+}}", "R6", 44, 84)) __msg("has trip count 6{{$}}"))
/* 9, 17, 25 go to 41: each overshoots 40 by 1, not by up to 7. At the header it is 8 less. */
RANGE_TEST(range_lt_overshoot, "", 0, 2, 8, 40, 40,
- "r6 *= 8; r6 += 1;" POST("r6 += r8", "r6 < r7"), COUNTED(5, "1..33"))
+ "r6 *= 8; r6 += 1;" POST("r6 += r8", "r6 < r7"), COUNTED(5, 1, 33))
/* -5, -2, 1, 4 cross 0, where the order of '<' begins: 0 is none of them. 1 goes to 22. */
RANGE_TEST(range_lt_overshoot_crosses_0, "", 0, 3, 3, 20, 20, "r6 *= 3; r6 += -5;" PRE(">="),
- __retval(1) __msg("header_count is [0..8] ") __msg("widening r6 to -5..22 "))
+ __retval(1) COUNT(10))
/* Known low bits are a congruence too: 1..40 with three zero bits is 8..40 */
RANGE_TEST(range_ne_align8, "", 0, 0, 8, 1, 43,
- "r7 &= -8; if r7 == 0 goto 2f;" PRE("=="), COUNTED(6, "0..40"))
+ "r7 &= -8; if r7 == 0 goto 2f;" PRE("=="), __retval(6) COUNT(6))
/*
* The low 32 bits of a multiple of 24 are one as long as it fits in 32 bits.
* 24 * 0xaaaaaac is 0x100000020.
*/
RANGE_TEST(range32_ne_mul24, "", 0, 0, 24, 1, 5,
- "r7 *= 24;" POST("w6 += w8", "w6 != w7"), COUNTED(5, "0..96"))
+ "r7 *= 24;" POST("w6 += w8", "w6 != w7"), COUNTED(5, 0, 96))
RANGE_TEST(range32_ne_mul24_wide, "", 0, 0, 24, 1, 1ULL << 30,
"r7 *= 24; if w7 > 96 goto 2f; if w7 < 24 goto 2f;" POST("w6 += w8", "w6 != w7"),
NO_COUNT)
@@ -1308,28 +1142,26 @@ __naked void name(void) \
}
#define BUF(i) "r9 += 1; r2 = " i "; r2 -= r7; if r2 <= 7 goto +1; r0 = *(u64 *)(r5 + 0);"
#define OOB __failure __msg("R5 !read_ok")
-#define ALL_OF_S32 "0xffffffff80000000..0x7fffffff "
#define SEXT32_INC "w1 = w6; w1 += 1; r6 = (s32)r1;"
#define SEXT32_LOOP(bound) "1:" BUF("r6") SEXT32_INC "if w1 != " #bound " goto 1b;"
SEXT32_TEST(sext32_iv, -4, SEXT32_LOOP(4), __success __retval(8)
- __msg(" r6=(sext32 (+ r6 1)) / (sext32 (linear r6 1))")
- __msg("header_count is 8 ") __msg("widening r6 to -4..3 "))
+ COUNT(8))
SEXT32_TEST(sext32_iv_oob, -4, SEXT32_LOOP(5), OOB)
/* i wraps from INT32_MAX to INT32_MIN. The latch tests 0x7ffffffd .. 0x80000004. */
SEXT32_TEST(sext32_iv_wraps, INT32_MAX - 3, SEXT32_LOOP(0x80000004),
- __msg("header_count is 8 ") __msg("widening r6 to " ALL_OF_S32) OOB)
+ OOB)
/* 'i < n' implies the unsigned range 1 .. 0x80000000. Its last value is negative in r6. */
SEXT32_TEST(sext32_iv_unsigned_bound, 0,
"r8 &= 1; r6 += r8; r4 = 0x80000001 ll;"
"1:" SEXT32_INC "if w1 < w4 goto 1b; if r6 s>= 0 goto 2f; r0 = *(u64 *)(r5 + 0);",
- __msg("widening r6 to " ALL_OF_S32) OOB)
+ OOB)
/*
* for (long i = first; (u32)i < 128; i -= 1000)
*
- * r6 is a 64-bit induction variable, the latch tests a zero-extended copy of it. It has the SCEV
- * of a 32-bit induction variable, and it bounds the low 32 bits of r6 only.
+ * r6 is a 64-bit induction variable, the latch tests a zero-extended copy of it.
+ * That bounds the low 32 bits of r6 only.
*/
#define IV64_TEST(name, first, loop, expect) SEXT32_TEST(name, first, loop, expect)
#define HI32_IS_0 "r4 = r6; r4 >>= 32; if r4 == 0 goto +1; r0 = *(u64 *)(r5 + 0);"
@@ -1338,24 +1170,24 @@ SEXT32_TEST(sext32_iv_unsigned_bound, 0,
IV64_TEST(zext32_copy_below_0, 0,
"r8 &= 3; r6 += r8; 1: if r6 <= 3 goto +1; r0 = *(u64 *)(r5 + 0);"
ZEXT32_COPY "if r3 >= 128 goto 2f; r6 += -1000; goto 1b;",
- __msg("header_count is [0..2] ") __msg("widening r6 to -1000..3 ") OOB)
+ __msg(RANGE("{{[0-9]+}}", "R6", -1000, 3)) OOB)
/* A 32-bit induction variable is 0xfffffc18..0xfffffc1b then */
IV64_TEST(zext32_iv_below_0, 0,
"r8 &= 3; r6 += r8; 1:" HI32_IS_0
ZEXT32_COPY "if r3 >= 128 goto 2f; w6 += -1000; goto 1b;",
- __success __retval(0) __msg("header_count is [0..2] "))
+ __success __retval(0) __msg("has trip count 2{{$}}"))
/* r6 gets to 0x100000009 */
IV64_TEST(zext32_copy_above_u32_max, 0xfffffff0ULL,
"r8 &= 15; r6 += r8; 1:" HI32_IS_0 "r6 += 1;" ZEXT32_COPY "if r3 < 10 goto 1b;",
- __msg("header_count is [0..11] ") __msg("widening r6 to 0xfffffff0..0x100000009 ") OOB)
+ __msg(RANGE("{{[0-9]+}}", "R6", 0xfffffff0, 0x100000000)) OOB)
IV64_TEST(zext32_copy_above_u32_max_const, 0xfffffffeULL,
"1:" HI32_IS_0 "r6 += 1;" ZEXT32_COPY "if r3 != 3 goto 1b;",
- __msg("header_count is 5 ") __msg("widening r6 to 0xfffffffe..0x100000002 ") OOB)
+ OOB)
/* The latch sees the low 32 bits of the step */
IV64_TEST(zext32_copy_wide_step, 0,
"r8 &= 1; r6 += r8; 1: if r6 <= 10 goto +1; r0 = *(u64 *)(r5 + 0);"
"r1 = 0x100000001 ll; r6 += r1;" ZEXT32_COPY "if r3 < 10 goto 1b;",
- __msg("header_count is [0..10] ") OOB)
+ OOB)
/*
* Two back edges:
@@ -1374,53 +1206,51 @@ IV64_TEST(zext32_copy_wide_step, 0,
"1:" LIMIT_ITERATIONS add ";" \
"if " other " goto 3f; if " jump1 " goto 1b; goto 2f; 3: if " jump0 " goto 1b;"
/* The loop can go on through the other back edge and get to a latch that was false */
-#define AGAIN(iteration) \
- __msg("loop header at {{[0-9]+}}, a latch may be true again in iteration " #iteration) \
- __not_msg("header_count is")
+#define AGAIN NO_COUNT
/* The largest count. '!=' is false in one iteration: it has to be the last one. */
RANGE_TEST(backedges_ne_is_last, "", 0, 0, 1, 0, 1,
- BACKEDGES("r6 += 1", "r7 == 0", "r6 != 6", "r6 < 3"), COUNTED(6, "0..5"))
-/* Only the latch with the smallest count of a back edge is looked at: 'r6 != 3', not 'r6 < 100' */
+ BACKEDGES("r6 += 1", "r7 == 0", "r6 != 6", "r6 < 3"), COUNTED(6, 0, 5))
+/* The other back edge can step over 'r6 != 3'. Then 'r6 < 100' gives the count. */
#define AND_R8(add, count, jump0) \
"r8 = 0;" BACKEDGES(add "; r8 += 1", "r7 == 0", "r8 < " #count, jump0)
RANGE_TEST(backedges_ne_is_not_last, "", 0, 0, 1, 0, 1,
- AND_R8("r6 += 1", 5, "r6 >= 100 goto 2f; if r6 != 3"), AGAIN(4))
+ AND_R8("r6 += 1", 5, "r6 >= 100 goto 2f; if r6 != 3"), COUNT(100))
/*
* w6 is 31, 23, 15, 7 in iterations 0..3, and M32 in iteration 4. 'w6 > 15' is false in iterations
* 2 and 3.
*/
RANGE_TEST(backedges_end_then_wrap, "", 39, 39, 1, 0, 1,
- AND_R8("w5 = -8; w6 += w5", 4, "w6 > 15"), __retval(4) __msg("header_count is [0..4] "))
+ AND_R8("w5 = -8; w6 += w5", 4, "w6 > 15"), __retval(4) NO_COUNT)
RANGE_TEST(backedges_wrap_then_end, "", 39, 39, 1, 0, 1,
- AND_R8("w5 = -8; w6 += w5", 5, "w6 > 15"), AGAIN(4))
+ AND_R8("w5 = -8; w6 += w5", 5, "w6 > 15"), AGAIN)
/* The same upwards. The largest first value wraps around first. */
RANGE_TEST(backedges_wrap_up_then_end, "", M32 - 55, M32 - 39, 1, 0, 1,
- AND_R8("w6 += 8", 5, "w6 < -16"), AGAIN(4))
+ AND_R8("w6 += 8", 5, "w6 < -16"), AGAIN)
/*
* w6 moves away from the bound: 'w6 < 0x38000000' is false in iterations 3..14, and true again in
* iteration 15, in which '!=' is false.
*/
RANGE_TEST(backedges_away_then_again, "", 0x30000000, 0x30000000, 1, 0, 1,
BACKEDGES("w5 = -0x10000000; w6 += w5", "r7 == 0",
- "w6 < 0x38000000", "w6 != 0x30000000"), AGAIN(15))
-/* One latch bounds the low half of r6, the other r6: together they bound neither */
+ "w6 < 0x38000000", "w6 != 0x30000000"), AGAIN)
+/* One latch bounds the low half of r6, the other r6. r6 fits in 32 bits, so both bound it. */
RANGE_TEST(backedges_low32_and_64, "", 0, 3, 1, 0, 1,
BACKEDGES("r6 += 1", "r7 == 0", "w6 < 5", "r6 < 7"),
- __retval(5) __msg("header_count is [0..7] ") __msg("widening r6 to 0..9 "))
+ __retval(5) __msg(RANGE("{{[0-9]+}}", "R6", 0, 6)) __msg("has trip count 7{{$}}"))
/* It's base + offset that wraps around when the variable is a pointer */
RANGE_TEST(backedges_pointer_wraps, "", 0, 0, 1, 0, 1, "r4 = r10; r4 += -8; r5 = r10; r5 += -512;"
- AND_R8("r5 += 8", 100000000, "r5 < r4"), AGAIN(99999999))
+ AND_R8("r5 += 8", 100000000, "r5 < r4"), AGAIN)
/* The same test in front of each back edge: the first iteration in which it's false is the last */
#define TWO_TESTS(jump1, jump0) \
- "r5 = 0; r8 = 0;" BACKEDGES("r5 += 1; r8 += 1", "r9 & 1", jump1, jump0)
+ "r5 = 0; r8 = 0;" BACKEDGES("r5 += 1; r8 += 1", "r3 & 1", jump1, jump0)
RANGE_TEST(backedges_same_test, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r8 != r7"),
- __retval(6) __msg("header_count is [0..6] "))
-RANGE_TEST(backedges_other_bound, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r8 != r6"), AGAIN(5))
-RANGE_TEST(backedges_other_first, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r5 != r7"), AGAIN(5))
-RANGE_TEST(backedges_other_op, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r8 < r7"), AGAIN(5))
-RANGE_TEST(backedges_other_width, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "w8 != w7"), AGAIN(5))
+ __retval(6) __msg("has trip count 6{{$}}"))
+RANGE_TEST(backedges_other_bound, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r8 != r6"), AGAIN)
+RANGE_TEST(backedges_other_first, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r5 != r7"), COUNT(6))
+RANGE_TEST(backedges_other_op, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "r8 < r7"), COUNT(6))
+RANGE_TEST(backedges_other_width, "", 1, 6, 1, 1, 6, TWO_TESTS("r8 != r7", "w8 != w7"), COUNT(6))
/* A back edge has no count */
RANGE_TEST(backedges_no_count, "", 0, 0, 1, 0, 1,
@@ -1429,8 +1259,7 @@ RANGE_TEST(backedges_no_count, "", 0, 0, 1, 0, 1,
#define X4(s) s s s s
RANGE_TEST(backedges_17, "", 0, 0, 1, 0, 1,
"1:" LIMIT_ITERATIONS "if r6 >= 3 goto 2f; r6 += 1;" X4(X4("if r7 == 9 goto 1b;"))
- "goto 1b;",
- __msg("unsupported loop: too many backedges") __not_msg("header_count is"))
+ "goto 1b;", COUNT(4))
/*
* The first value is 0..3: 'first + step * count' is 0..8. At the header r6 is its first value or
@@ -1439,24 +1268,24 @@ RANGE_TEST(backedges_17, "", 0, 0, 1, 0, 1,
RANGE_TEST(backedges_first_is_a_range, "", 0, 3, 1, 0, 2,
"1:" LIMIT_ITERATIONS "r6 += 1; if r7 == 0 goto 3f; if r7 == 2 goto 4f;"
"if r6 < 3 goto 1b; goto 2f; 4: if r6 < 6 goto 1b; goto 2f; 3: if r6 < 4 goto 1b;",
- COUNTED(6, "0..5"))
+ COUNTED(6, 0, 5))
/* The latches test different variables: they tell nothing about r6 */
RANGE_TEST(backedges_two_variables, "", 0, 3, 1, 0, 1,
- BACKEDGES("r6 += 1; r7 += 1", "r9 & 1", "r6 < 3", "r7 < 6"),
- __msg("header_count is [0..6] ") __msg("widening r6 to 0..8 "))
+ BACKEDGES("r6 += 1; r7 += 1", "r3 & 1", "r6 < 3", "r7 < 6"),
+ COUNT(6))
/* A bound of 3 and a bound of 5 are joined in front of the loop */
#define PRECISE_BOUND(reg) \
"r4 = 2; r5 = 2; " reg " = 5; if r7 == 0 goto +1; " reg " = 3; r0 = 0;" \
- BACKEDGES("r6 += 1", "r9 & 1", "r6 < r4", "r6 < r5")
-#define EXPLORED_ONCE __msg("header_count is [0..5] ") __not_msg("header_count is [0..3] ")
+ BACKEDGES("r6 += 1", "r3 & 1", "r6 < r4", "r6 < r5")
+#define EXPLORED_ONCE __msg("has trip count 5{{$}}")
RANGE_TEST(backedges_bound1_is_precise, "", 0, 0, 1, 0, 1, PRECISE_BOUND("r4"), EXPLORED_ONCE)
RANGE_TEST(backedges_bound0_is_precise, "", 0, 0, 1, 0, 1, PRECISE_BOUND("r5"), EXPLORED_ONCE)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 1, header_count is 10 ")
-__msg("loop header at 1, widening r0 to 0..9 step 1")
+__msg(RANGE("1", "R0", 0, 9))
+__msg("loop at 1 has trip count 10{{$}}")
__msg("processed 4 insns")
__naked void post_cond_jlt(void)
{
@@ -1485,8 +1314,6 @@ __naked void post_cond_jlt(void)
SEC("xdp")
__failure
__log_level(2)
-__msg("loop header at 6, header_count is [0..4096] ")
-__msg("loop header at 6, widening r0 to 0..4095 step 1")
__msg("invalid variable-offset read from stack R2")
__naked void mark_bound_precise(void)
{
@@ -1526,8 +1353,6 @@ __naked void mark_bound_precise(void)
SEC("xdp")
__failure
__log_level(2)
-__msg("loop header at 5, header_count is [0..4096] ")
-__msg("loop header at 5, widening r0 to 0..4095 step 1")
__msg("invalid variable-offset read from stack R2")
__naked void mark_base_precise(void)
{
@@ -1551,7 +1376,7 @@ __naked void mark_base_precise(void)
}
/*
- * Spill size should be taken into account when evaluating bound scev.
+ * Spill size should be taken into account for the bound.
*
* r0 = 256;
* *(u8 *)(r10 - 8) = r0;
@@ -1559,16 +1384,14 @@ __naked void mark_base_precise(void)
* do {
* r0++;
* r1 = *(u64 *)(r10 - 8);
- * if (r0 & 0x200) break; // hidden from scev, limit main pass iterations.
+ * if (r0 & 0x200) break; // gives no trip count, limit main pass iterations.
* } while (r0 < r1);
* return r0;
*/
SEC("xdp")
__success
__log_level(2)
-__msg("scev at header 3:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg("loop header at 3, can't compute iterations count")
+__msg("loop at 3 has no trip count")
__naked void eval_expr_small_spill_bound(void)
{
asm volatile (" \
@@ -1587,9 +1410,7 @@ __naked void eval_expr_small_spill_bound(void)
SEC("xdp")
__success
__log_level(2)
-__msg("scev at header 3:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg("loop header at 3, can't compute iterations count")
+__msg("loop at 3 has no trip count")
__naked void eval_expr_small_spill_bound_fits(void)
{
asm volatile (" \
@@ -1617,11 +1438,8 @@ __naked void eval_expr_small_spill_bound_fits(void)
SEC("xdp")
__success
__log_level(2)
-__msg("scev at header 2:")
-__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)")
/* whatever is in the other bytes of the slot */
-__msg("loop header at 2, header_count is [0..266] ")
-__not_msg("widening")
+NO_COUNT
__naked void no_widen_small_spill_counter(void)
{
asm volatile (" \
@@ -1639,9 +1457,7 @@ __naked void no_widen_small_spill_counter(void)
SEC("xdp")
__success
__log_level(2)
-__msg("scev at header 2:")
-__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)")
-__msg("loop header at 2, widening *fp-8 to 5..14 step 1")
+__msg(RANGE("2", "fp-8", 5, 14))
__naked void widen_full_spill_counter(void)
{
asm volatile (" \
@@ -1658,9 +1474,7 @@ __naked void widen_full_spill_counter(void)
SEC("xdp")
__success
__log_level(2)
-__msg("scev at header 2:")
-__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)")
-__msg("loop header at 2, widening *fp-8 to 5..14 step 1")
+__msg(RANGE("2", "fp-8", 5, 14))
__naked void widen_acquire_release_counter(void)
{
asm volatile (
@@ -1679,9 +1493,9 @@ __naked void widen_acquire_release_counter(void)
}
/*
- * r6 scev is (any r6 *fp-8). The byte store leaves memory at zero,
- * but the spill metadata retains 256. Widen r6 to an unknown scalar
- * instead of [1,256], which would exclude the zero loaded from memory.
+ * r6 is r6 or *fp-8. The byte store leaves memory at zero,
+ * but the spill metadata retains 256. r6 must not be [1,256],
+ * which would exclude the zero loaded from memory.
*
* r0 = 0;
* *(u64 *)(r10 - 8) = r0;
@@ -1699,10 +1513,12 @@ __naked void widen_acquire_release_counter(void)
SEC("xdp")
__success __retval(0)
__log_level(2)
-__msg("scev at header 6:")
-__msg_next(" r6=*fp-8 / (any r6 *fp-8)")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
-__msg("loop header at 6, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1")
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+__msg("to 6: R6=scalar(smax={{[^)]*}}var_off=(0x0; 0xffffffffffffff01)) ")
+#else
+__msg("to 6: R6=scalar() ")
+#endif
+COUNT(2)
__naked void widen_any_small_spill_leaf(void)
{
asm volatile (" \
@@ -1721,7 +1537,7 @@ __naked void widen_any_small_spill_leaf(void)
}
/*
- * Widening (any *fp-8 7) must allocate the slot before writing its state.
+ * fp-8 is not set at the entry and is 7 later. The join must allocate the slot.
* The unallocated entry value makes the union an unknown scalar.
*
* r6 = 0;
@@ -1736,10 +1552,7 @@ __naked void widen_any_small_spill_leaf(void)
SEC("xdp")
__success __retval(7)
__log_level(2)
-__msg("scev at header 1:")
-__msg_next("r6=(+ r6 1) / (linear r6 1)")
-__msg_next("*fp-8=%fp-8.3=phi(*fp-8, 7) / (any *fp-8 7)")
-__msg("loop header at 1, widening r6 to 0..2 step 1")
+__msg(RANGE("1", "R6", 0, 2))
__msg("3: (79) r0 = *(u64 *)(r10 -8) ; R0=scalar() R10=fp0 loop_stack=1")
__naked void widen_any_unallocated_slot(void)
{
@@ -1758,7 +1571,7 @@ __naked void widen_any_unallocated_slot(void)
SEC("xdp")
__success __retval(7)
__log_level(2)
-__msg("loop header at 2, widening *fp-8 to 0..7 step 1")
+__msg(RANGE("2", "fp-8", 0, 7))
__naked void widen_any_zero_spill(void)
{
asm volatile (" \
@@ -1777,10 +1590,7 @@ __naked void widen_any_zero_spill(void)
SEC("xdp")
__success __retval(7)
__log_level(2)
-__msg("scev at header 2:")
-__msg_next("r6=(+ r6 1) / (linear r6 1)")
-__msg_next("*fp-8=%fp-8.5=phi(*fp-8, 7) / (any *fp-8 7)")
-__msg("loop header at 2, widening r6 to 0..2 step 1")
+__msg(RANGE("2", "R6", 0, 2))
__msg("3: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar() R10=fp0 fp-8=m???????")
__naked void widen_any_misc_slot(void)
{
@@ -1852,8 +1662,7 @@ extern int bpf_stream_print_stack(int stream_id) __weak __ksym;
SEC("syscall")
__success __retval(0)
__log_level(2)
-__msg("loop header at 1, header_count is 4 ")
-__not_msg("can't widen")
+__msg("loop at 1 has trip count 4{{$}}")
__naked void kfunc_implicit_arg_in_a_loop(void)
{
asm volatile (
@@ -1876,14 +1685,11 @@ void kfunc_root(void)
bpf_stream_print_stack(0);
}
-/* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */
+/* An unbound STACK_MISC slot must not prevent widening. */
SEC("tracepoint")
__success
__log_level(2)
-__msg("scev at header 4:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" *fp-8=%fp-8.7?")
-__msg("loop header at 4, widening r6 to 0..9 step 1")
+__msg(RANGE("4", "R6", 0, 9))
__naked void widen_subprog_probe_read(void)
{
asm volatile (
@@ -1909,7 +1715,7 @@ __naked void widen_subprog_probe_read(void)
*/
SEC("tracepoint")
__success __log_level(2)
-__msg("loop header at 2, header_count is 10")
+__msg("loop at 2 has trip count 10{{$}}")
__naked void stack_address_as_a_number(void)
{
asm volatile (" \
@@ -1934,9 +1740,8 @@ __naked void stack_address_as_a_number(void)
SEC("socket")
__success
__log_level(2)
-__msg("loop header at 4, header_count is 10")
-__msg("loop header at 4, widening r1 to 0..9 step 1")
-__msg("loop header at 4, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1")
+__msg(RANGE("4", "R1", 0, 9) "{{.*}}R6=scalar(var_off=(0x1; 0xfffffffffffffffe))")
+__msg("loop at 4 has trip count 10{{$}}")
__flag(BPF_F_TEST_STATE_FREQ)
__naked void widen_overflowing_range(void)
{
@@ -1956,7 +1761,7 @@ __naked void widen_overflowing_range(void)
}
/*
- * The bound slot is unallocated at the first loop header. SCEV must
+ * The bound slot is unallocated at the first loop header. The verifier must
* treat it as an unknown scalar. JSET limits main-pass iterations.
*
* r0 = 0;
@@ -1970,10 +1775,7 @@ __naked void widen_overflowing_range(void)
SEC("xdp")
__success __retval(0)
__log_level(2)
-__msg("scev at header 1:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg("loop header at 1, can't compute iterations count")
-__not_msg("widening")
+__msg("loop at 1 has no trip count")
__naked void eval_expr_unallocated_stack_bound(void)
{
asm volatile (" \
@@ -1997,7 +1799,7 @@ volatile __u32 data_bound = 10;
#define BOUND_IN_MAP_TEST(name, bound, count) \
SEC("socket") \
__success __retval(10) __log_level(2) \
-__msg("loop header at 3, header_count is [0.." #count "]") \
+__msg("loop at 3 has trip count " #count "{{$}}") \
__naked void name(void) \
{ \
asm volatile ( \
@@ -2018,9 +1820,9 @@ BOUND_IN_MAP_TEST(bound_in_data, data_bound, 16)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 3, header_count is 8")
-__msg("loop header at 3, widening r0 to 2..9 step 1")
+__msg(RANGE("3", "R0", 2, 9))
__msg("3: R0=scalar(smin=umin=smin32=umin32=2,smax=umax=smax32=umax32=9,{{.*}})")
+__msg("loop at 3 has trip count 8{{$}}")
__msg("processed 5 insns")
__naked void post_cond_jlt_with_base(void)
{
@@ -2037,7 +1839,7 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 1, header_count is 4 ")
+__msg("loop at 1 has trip count 4{{$}}")
__naked void latch_base_differs_from_step(void)
{
asm volatile (" \
@@ -2053,11 +1855,10 @@ __naked void latch_base_differs_from_step(void)
}
/* Additions and subtractions of constants are one step. Two 64-bit shifts by 32 are (u32)r6. */
-#define ONE_STEP_TEST(name, first, body, jump, expr, ret) \
+#define ONE_STEP_TEST(name, first, body, jump, ret) \
SEC("socket") \
__success __retval(ret) __log_level(2) \
-__msg(" r6=" expr) \
-__msg("loop header at 1, header_count is 8 ") \
+__msg("loop at 1 has trip count 8{{$}}") \
__naked void name(void) \
{ \
asm volatile ( \
@@ -2069,10 +1870,9 @@ __naked void name(void) \
::: __clobber_all); \
}
-ONE_STEP_TEST(add_add, 0, "r6 += 1; r6 += 1;", "< 16", "(+ r6 2) / (linear r6 2)", 16)
-ONE_STEP_TEST(sub, 16, "r6 -= 2;", "> 0", "(+ r6 -2) / (linear r6 -2)", 0)
-ONE_STEP_TEST(two_shifts_by_32, 0, "r6 += 2; r6 <<= 32; r6 >>= 32;", "< 16",
- "(zext32 (+ r6 2)) / (zext32 (linear r6 2))", 16)
+ONE_STEP_TEST(add_add, 0, "r6 += 1; r6 += 1;", "< 16", 16)
+ONE_STEP_TEST(sub, 16, "r6 -= 2;", "> 0", 0)
+ONE_STEP_TEST(two_shifts_by_32, 0, "r6 += 2; r6 <<= 32; r6 >>= 32;", "< 16", 16)
/*
* @shifts are not (u32)r7: a 32-bit shift by a register that has 32 in it shifts by 32 & 31 = 0.
@@ -2108,9 +1908,9 @@ NO_ZEXT32_TEST(arsh_by_32, 0x7ffffffe, "r7 <<= 32; r7 s>>= 32;")
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 1, header_count is 11")
-__msg("loop header at 1, widening r0 to 0..10 step 1")
+__msg(RANGE("1", "R0", 0, 10))
__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})")
+__msg("loop at 1 has trip count 11{{$}}")
__msg("processed 4 insns")
__naked void post_cond_jle(void)
{
@@ -2125,9 +1925,9 @@ __naked void post_cond_jle(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 1, header_count is 10")
-__msg("loop header at 1, widening r0 to 0..9 step 1")
+__msg(RANGE("1", "R0", 0, 9))
__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=9,{{.*}})")
+__msg("loop at 1 has trip count 10{{$}}")
__msg("processed 5 insns")
__naked void post_cond_jge(void)
{
@@ -2143,9 +1943,9 @@ __naked void post_cond_jge(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 1, header_count is 11")
-__msg("loop header at 1, widening r0 to 0..10 step 1")
+__msg(RANGE("1", "R0", 0, 10))
__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})")
+__msg("loop at 1 has trip count 11{{$}}")
__msg("processed 5 insns")
__naked void pre_cond_jge(void)
{
@@ -2161,15 +1961,9 @@ __naked void pre_cond_jge(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg_next(" scev at latch 2:")
-__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)")
-__msg("loop header at 1, widening r0")
__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})")
__msg("1: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})")
__msg("2: (55) if r0 != 0x3 goto pc-2")
-__msg("loop header at 1, clamping r0")
__msg("3: (95) exit")
__msg("processed 4 insns")
__naked void post_cond_jne(void)
@@ -2185,14 +1979,9 @@ __naked void post_cond_jne(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r0=(+ r0 -1) / (linear r0 -1)")
-__msg_next(" scev at latch 2:")
-__msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)")
-__msg("loop header at 1, header_count is 3 ")
-__msg("loop header at 1, widening r0 to 1..3 step 1")
+__msg(RANGE("1", "R0", 1, 3))
__msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) R10=fp0 loop_stack=1")
-__msg("loop header at 1, clamping r0 to 1..3 step 1")
+__msg("loop at 1 has trip count 3{{$}}")
__msg("processed 4 insns")
__naked void post_cond_jne_neg_step(void)
{
@@ -2212,12 +2001,10 @@ SEC("socket")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, header_count is 5 ")
-__msg("loop header at 2, widening r1 to 0..4 step 1")
-__msg("loop header at 2, widening r3 to 0..4 step 1")
-__msg("3: (55) if r3 != 0x5 goto pc+1 {{.*}}; R3=5")
-__msg("loop header at 2, clamping r3 to 0..4 step 1")
-__msg("from 6 to 2: within")
+__msg(RANGE("2", "R1", 0, 4) RANGE_TOO("R3", 0, 4))
+__msg("from 3 to 4: R1=4 R3=5")
+__msg("from 6 to 2: pruned")
+__msg("loop at 2 has trip count 5{{$}}")
__naked void clamp_dependent_overshoot(void)
{
asm volatile (" \
@@ -2233,17 +2020,15 @@ __naked void clamp_dependent_overshoot(void)
" ::: __clobber_all);
}
-/* Same with a 32-bit dependent register, SCEV (zext32 (linear r3 1)) */
+/* Same with a 32-bit dependent register */
SEC("socket")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, header_count is 5 ")
-__msg("loop header at 2, widening r1 to 0..4 step 1")
-__msg("loop header at 2, widening r3 to 0..4 step 1")
-__msg("3: (56) if w3 != 0x5 goto pc+1 {{.*}}; R3=5")
-__msg("loop header at 2, clamping r3 to 0..4 step 1")
-__msg("from 6 to 2: within")
+__msg(RANGE("2", "R1", 0, 4) RANGE_TOO("R3", 0, 4))
+__msg("from 3 to 4: R1=4 R3=5")
+__msg("from 6 to 2: pruned")
+__msg("loop at 2 has trip count 5{{$}}")
__naked void clamp_dependent_overshoot32(void)
{
asm volatile (" \
@@ -2262,18 +2047,13 @@ __naked void clamp_dependent_overshoot32(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg_next(" scev at latch 1:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg("loop header at 1, widening r0")
__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=3,{{.*}})")
-__msg("1: (15) if r0 == 0x3 goto pc+2 ; R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})")
+__msg("1: (15) if r0 == 0x3 goto pc+2")
+__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})")
__msg("2: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})")
__msg("3: (05) goto pc-3")
-__msg("loop header at 1, clamping r0")
-__msg("from 1 to 4: R0=3")
-__msg("4: R0=3")
+/* or 0: pass 0 takes the edge of may_goto */
+__msg("from 1 to 4: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=3,{{.*}}step=0+3)")
__msg("4: (95) exit")
__msg("processed 5 insns")
__naked void pre_cond_je1(void)
@@ -2290,15 +2070,9 @@ __naked void pre_cond_je1(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 1:")
-__msg_next(" r0=(zext32 (+ r0 1)) / (zext32 (linear r0 1))")
-__msg_next(" scev at latch 2:")
-__msg_next(" r0=(zext32 (+ r0 1)) / (zext32 (linear (+ r0 1) 1))")
-__msg("loop header at 1, widening r0")
__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})")
__msg("1: (04) w0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})")
__msg("2: (56) if w0 != 0x3 goto pc-2")
-__msg("loop header at 1, clamping r0")
__msg("3: (95) exit")
__msg("processed 4 insns")
__naked void post_cond_jne32(void)
@@ -2316,14 +2090,10 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev at header 1:")
-__msg_next(" r0=(zext32 (+ r0 3)) / (zext32 (linear r0 3))")
-__msg_next(" scev at latch 2:")
-__msg_next(" r0=(zext32 (+ r0 3)) / (zext32 (linear (+ r0 3) 3))")
__msg("entering loop 1")
-__msg_next("loop header at 1, header_count is 2")
-__msg("loop header at 1, widening r0 to 0..0xffffffff (s32 -1..2) step 1")
-__msg("1: R0=scalar(smin=0,smax=umax=0xffffffff,smin32=-1,smax32=2,{{.*}})")
+__msg(RANGE("1", "R0", 2, 0xffffffff))
+__msg("1: R0=scalar(smin=umin=2,smax=umax=0xffffffff,smin32=-1,smax32=2,{{.*}})")
+__msg("loop at 1 has trip count 2{{$}}")
__naked void post_cond_jne32_overflow(void)
{
asm volatile (" \
@@ -2342,8 +2112,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, header_count is 1431655767")
-__msg("loop header at 2, widening r0 to 0..0xffffffff step 1")
+__msg(RANGE("2", "R0", 0, 0xffffffff))
+__msg("loop at 2 has trip count 1431655767{{$}}")
__naked void post_cond_jne32_full_range(void)
{
asm volatile (" \
@@ -2361,8 +2131,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 1, header_count is 2")
-__msg("loop header at 1, widening r0 to -6..0xfffffffa (s32 -9..-6) step 1")
+__msg(RANGE("1", "R0", -6, 0xfffffff7))
+__msg("loop at 1 has trip count 2{{$}}")
__naked void post_cond_jne32_base_mismatch(void)
{
asm volatile (" \
@@ -2381,8 +2151,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 1, header_count is 2")
-__msg("loop header at 1, widening r0 to 0..0xffffffff (s32 -1..0) step 1")
+__msg(RANGE("1", "R0", 0, 0xffffffff))
+__msg("loop at 1 has trip count 2{{$}}")
__naked void post_cond_jne32_neg_step(void)
{
asm volatile (" \
@@ -2401,9 +2171,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, header_count is 2")
-__msg("loop header at 2, widening r0 to 0x8000000000000000..0x7fffffffffffffff (s32 1..3) step 2")
-__msg("2: R0=scalar({{.*}}step=1+2)")
+__msg("2: R0=scalar(umin=3,umax=0x8000000000000001,")
+__msg("loop at 2 has trip count 2{{$}}")
__naked void post_cond_jne32_signed_boundary(void)
{
asm volatile (" \
@@ -2421,10 +2190,9 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 1, header_count is 4")
-__msg("loop header at 1, widening r0 to 0..3 step 1")
-__msg("loop header at 1, clamping r0")
-__msg("4: R0=3")
+__msg(RANGE("1", "R0", 0, 3))
+__msg("loop at 1 has trip count 4{{$}}")
+__msg("4: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=3,{{.*}}step=0+3)")
__naked void pre_cond_je32(void)
{
asm volatile (" \
@@ -2441,7 +2209,7 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 4, can't compute iterations count")
+__msg("loop at 4 has no trip count")
__naked void post_cond_alu32_jmp64(void)
{
asm volatile (" \
@@ -2461,7 +2229,7 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, can't compute iterations count")
+__msg("loop at 2 has no trip count")
__naked void post_cond_alu64_jmp32(void)
{
asm volatile (" \
@@ -2483,8 +2251,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 3, header_count is 3")
-__msg("loop header at 3, widening r0 to 0xfffffffc..0xfffffffe (s32 -4..-2) step 1")
+__msg(RANGE("3", "R0", 0xfffffffc, 0xfffffffe))
+__msg("loop at 3 has trip count 3{{$}}")
__naked void post_cond_alu32_jmp64_bounded(void)
{
asm volatile (" \
@@ -2501,8 +2269,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 1, header_count is 3")
-__msg("loop header at 1, widening r0 to -6..-2 step 2")
+__msg(RANGE("1", "R0", -6, -2))
+__msg("loop at 1 has trip count 3{{$}}")
__naked void post_cond_alu64_jmp32_bounded(void)
{
asm volatile (" \
@@ -2517,7 +2285,7 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 1, header_count is [0..3] ")
+__msg("loop at 1 has trip count 3{{$}}")
__naked void one_backedge_two_exits(void)
{
asm volatile (" \
@@ -2544,17 +2312,11 @@ __naked __noinline __used static int returns_zero(void)
/*
* A loop that the compiler unrolls in part has an exit for every 'break' and 'return' in every
* copy of its body. How many there are says nothing about the number of iterations.
- * Scalar evolution takes a nested loop for one insn, and its exits are the successors of this insn.
- * The exit list is truncated: the enclosing loop has no SCEVs, the subprog is not analyzed.
+ * Only the first exit is kept.
*/
SEC("xdp")
__success __log_level(2)
-__msg("loop at 1, too many exits, too many exits of a nested loop")
-__msg("loop at 2, nested in 1, too many exits")
-__msg("Computing SCEV for subprog at 0:")
-__msg_next("Computing SCEV for subprog at {{[0-9]+}}:")
-__msg("loop header at 1, unsupported loop: too many exits of a nested loop")
-__msg("loop header at 2, header_count is [0..4] ")
+__msg("loop at 2 has trip count 4{{$}}")
__naked void loop_around_301_exits(void)
{
asm volatile (" \
@@ -2576,19 +2338,9 @@ __naked void loop_around_301_exits(void)
SEC("xdp")
__success
__log_level(2)
-__msg_next("scev at header 11:")
-__msg_next(" r0=(+ r0 1) / (linear r0 1)")
-__msg_next(" r1=(+ r1 2) / (linear r1 2)")
-__msg_next(" scev at latch 16:")
-__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)")
-__msg_next(" r1=(+ r1 2) / (linear (+ r1 2) 2)")
-__msg("loop header at 11, widening r0")
-__msg("loop header at 11, widening r1")
__msg("11: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=7,var_off=(0x0; 0x7)) R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=14,var_off=(0x0; 0xe),step=0+2)")
__msg("16: (a5) if r0 < 0x8 goto pc-6")
/* second iteration */
-__msg("loop header at 11, clamping r0")
-__msg("loop header at 11, clamping r1")
/* iteration convergence */
__not_msg("{{^}}11:")
/* loop exit */
@@ -2625,7 +2377,7 @@ __naked void correlated_regs(void)
* for (i = 0; i < 4; i++):
* for (j = 0; j < 4; j++):
* k += 1
- * k <<= 1 // make SCEV construction not possible
+ * k <<= 1 // not linear
* k >>= 1
* map[k] = 1 // k has to be known
*/
@@ -2634,7 +2386,7 @@ __success
__log_level(2)
__msg("19: (72) *(u8 *)(r4 +0) = 1 ; R4=map_value(id={{.*}},map=map,ks=4,vs=1024,imm=16)")
__not_msg("19: ")
-__msg("processed 105 insns")
+__msg("processed 57 insns")
__naked void nested_loops_precise_var1(void)
{
asm volatile (" \
@@ -2670,9 +2422,9 @@ __naked void nested_loops_precise_var1(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 2, header_count is 100 ")
-__msg("loop header at 4, header_count is [0..100] ")
-__msg("loop header at 6, header_count is [0..100] ")
+__msg("loop at 6 has trip count 100{{$}}")
+__msg("loop at 4 has trip count 100{{$}}")
+__msg("loop at 2 has trip count 100{{$}}")
__msg("processed 14 insns")
__naked void nested_loop_with_two_exits(void)
{
@@ -2699,6 +2451,7 @@ __naked void nested_loop_with_two_exits(void)
SEC("xdp")
__success
__log_level(2)
+__msg("loop at 4 has trip count 10{{$}}")
__naked void exit_loop_into_loop_header(void)
{
asm volatile (" \
@@ -2734,14 +2487,11 @@ __msg("loop at 5")
__msg("loop at 6, nested in 5")
__msg("loop at 11, irreducible")
__msg("loop at 12, nested in 11")
-/* entry via if r0 == 5 goto d_%= true branch */
-__msg("loop header at 12, header_count is 3 ")
-__msg("loop header at 12, widening r9 to 0..2 step 1")
-/* entry via if r0 == 5 goto d_%= false branch */
-__msg("loop header at 12, header_count is 3 ")
-__msg("loop header at 12, widening r9 to 0..2 step 1")
-__msg("12: R8=0 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) R10=fp0 iter(11)=0 loop_stack=11,12")
-__msg("12: R8=1 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) R10=fp0 iter(11)=0 loop_stack=11,12")
+NO_GUARD
+__msg("loop at 12 has trip count 3{{$}}")
+NO_GUARD
+__msg("loop at 11 has trip count 4{{$}}")
+NO_GUARD
__naked void enter_nested_loop_from_side(void)
{
asm volatile (" \
@@ -2772,8 +2522,8 @@ d_%=: r9 += 1; \
* which contains C (counter in r9).
* Entering B at 'body' saves and restores r6, making it appear invariant.
* Entering B at 'alternate' skips the save and modifies r6 instead.
- * Hence A must not infer a SCEV expression for r6.
- * SCEV expression for r9 in C should still be computed.
+ * Hence A must not have an equation for r6.
+ * r9 in C still moves with the trip index.
*
* 0: r6 = 0;
* do { // A
@@ -2804,12 +2554,7 @@ __log_level(2)
__msg("loop at 1{{$}}")
__msg("loop at 5, nested in 1, irreducible")
__msg("loop at 11, nested in 5")
-__msg("scev at header 1:")
-__msg_next(" r6=%r6.5?")
-__msg("scev at header 11:")
-__msg_next(" r9=(+ r9 1) / (linear r9 1)")
-__msg("loop header at 11, widening r9 to 0..2 step 1")
-__not_msg("loop header at 1, widening r6")
+__msg(RANGE("11", "R9", 0, 2))
__naked void nested_irreducible_loop(void)
{
asm volatile (" \
@@ -2840,9 +2585,7 @@ c_%=: r9 += 1; \
/*
* Loop A (counter in r6) contains irreducible loop B.
- * Loop B has an exit edge from A. This is a corner case
- * observed during testing: insn_aux_data[l1].need_scev == true,
- * but SCEV environment is not constructed.
+ * Loop B has an exit edge from A.
*
* r6 = 0;
* do { // A
@@ -2864,6 +2607,7 @@ c_%=: r9 += 1; \
SEC("xdp")
__success
__log_level(2)
+__msg("loop at 1 has trip count 4{{$}}")
__naked void nested_irreducible_loop2(void)
{
asm volatile (" \
@@ -2897,8 +2641,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 4, widening r7 to 0..18 step 2")
-__msg("R7=scalar(smin=smin32=0,smax=umax=smax32=umax32=18,var_off=(0x0; 0x1e),step=0+2)")
+__msg(RANGE("4", "R7", 0, 18))
+__msg("R7=scalar(smin=smin32=0,smax=umax=smax32=umax32=18,var_off=(0x0; 0x1e))")
__naked void widen_nonconst_base(void)
{
asm volatile (" \
@@ -2925,8 +2669,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 5, header_count is 3 ")
-__msg("5: R7=scalar({{.*}}smax=umax=smax32=umax32=20,{{.*}}step=0+4)")
+__msg("5: R7=scalar({{.*}}smax=umax=smax32=umax32=20,var_off=(0x0; 0x1c))")
+__msg("loop at 5 has trip count 3{{$}}")
__naked void widen_nonconst_base_refined(void)
{
asm volatile (" \
@@ -2957,11 +2701,11 @@ __naked void widen_nonconst_base_refined(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 1, widening r6 to 0..3 step 1")
-__msg("loop header at 2, widening r7 to 0..2 step 1")
+__msg(RANGE("1", "R6", 0, 3))
+__msg(RANGE("2", "R7", 0, 2))
__msg("exiting loop 2")
__msg("entering loop 4")
-__msg("loop header at 4, widening r7 to 1..3 step 1")
+__msg(RANGE("4", "R7", 1, 3))
__naked void sibling_inner_loops(void)
{
asm volatile (" \
@@ -2982,7 +2726,7 @@ SEC("socket")
__success
__log_level(2)
/* whatever is in the slot */
-__msg("loop header at 0, header_count is [0..11] ")
+NO_COUNT
__naked void uninit_slot_counter(void)
{
asm volatile (" \
@@ -3033,7 +2777,7 @@ __naked void conditional_assignment_on_second_iteration(void)
}
/*
- * The latch bound has header SCEV (any r7 100), so it is not invariant:
+ * The latch bound is r7 or 100, so it is not invariant:
*
* u64 i = 0, n = 3;
* for (;;) {
@@ -3043,20 +2787,12 @@ __naked void conditional_assignment_on_second_iteration(void)
* if (i == 1)
* n = 100;
* }
- *
- * The ANY expression must also appear at the latch, rather than a bare r7
- * that would incorrectly be treated as a loop-invariant bound.
*/
SEC("xdp")
__success
__log_level(2)
-__msg("scev at header 2:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" r7=%r7.6=phi(r7, 100) / (any r7 100)")
-__msg_next(" scev at latch 3:")
-__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)")
-__msg_next(" r7=r7 / (any r7 100)")
-__naked void no_widen_changing_latch_bound(void)
+__msg("loop at 2 has trip count 100{{$}}")
+__naked void changing_latch_bound(void)
{
asm volatile (" \
r6 = 0; \
@@ -3075,14 +2811,14 @@ exit_%=: \
}
/*
- * Join two stack pointer offsets via (any r7 r8), with r8 loop-invariant.
+ * Join two stack pointer offsets, r7 and r8. The loop does not change r8.
* The joined range [-16, -8] still points to initialized stack memory,
* so dereferencing r7 after the loop is safe.
*/
SEC("xdp")
__success __retval(0)
__log_level(2)
-__msg("loop header at 7, widening r7 to -16..-8 step 1")
+__msg(RANGE("7", "R7", -16, -8))
__msg("7: {{.*}}R7=fp(smin=smin32=-16,smax=smax32=-8,")
__msg("12: R7=fp(smin=smin32=-16,smax=smax32=-8,")
__naked void conditional_stack_pointer_assignment(void)
@@ -3120,8 +2856,7 @@ __naked void conditional_stack_pointer_assignment(void)
SEC("xdp")
__success __retval(0)
__log_level(2)
-__msg("loop header at {{[0-9]+}}, can't widen r6, expr is")
-__not_msg("widening")
+__msg("loop at 17 has trip count 3{{$}}")
__naked void no_widen_any_map_origin(void)
{
asm volatile (" \
@@ -3185,8 +2920,7 @@ __naked __noinline __used static int any_stack_origin_callee(void)
SEC("xdp")
__success __retval(0)
__log_level(2)
-__msg("loop header at {{[0-9]+}}, can't widen r6, expr is")
-__not_msg("widening")
+__not_msg("has no trip count")
__naked void no_widen_any_stack_origin(void)
{
asm volatile (" \
@@ -3219,9 +2953,8 @@ SEC("socket")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 3, widening r6 to 0..9 step 1")
-__msg("loop header at 3, clamping r6 to 0..9 step 1")
-__msg("from 12 to 6: within")
+__msg(RANGE("3", "R6", 0, 9))
+COUNT(10)
__naked void clamp_callee_state_pruning(void)
{
asm volatile (" \
@@ -3246,9 +2979,8 @@ SEC("socket")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 3, widening r6 to 0..9 step 1")
-__msg("loop header at 4, widening r7 to 0..2 step 1")
-__msg("loop header at 3, clamping r6 to 0..9 step 1")
+__msg(RANGE("3", "R6", 0, 9))
+__msg(RANGE("4", "R7", 0, 2))
__naked void clamp_nested_loop_state_pruning(void)
{
asm volatile (" \
@@ -3267,7 +2999,7 @@ __naked void clamp_nested_loop_state_pruning(void)
}
/*
- * Same as above, but the inner loop is not recognized by SCEV and is unrolled
+ * Same as above, but the inner loop has no linear counter
* with a precise r7. The states at its header still match between the outer
* iterations and must not be reported as an infinite loop: the path between
* them crosses the back edge of the terminating outer loop.
@@ -3276,9 +3008,8 @@ SEC("socket")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 3, widening r6 to 0..9 step 1")
-__msg("loop header at 5, can't compute iterations count")
-__msg("loop header at 3, clamping r6 to 0..9 step 1")
+__msg(RANGE("3", "R6", 0, 9))
+__msg("loop at 5 has trip count 2{{$}}")
__naked void clamp_nested_unrolled_loop_state_pruning(void)
{
asm volatile (" \
@@ -3297,15 +3028,12 @@ __naked void clamp_nested_unrolled_loop_state_pruning(void)
}
/*
- * A number in a part of the stack slot fp-8. There is no formula for it, but the state has its
- * value. If the loop makes the next value from it, each iteration has a new one: the loop is
- * verified one iteration at a time. If not, the body is verified at most two times.
+ * A number in a part of the stack slot fp-8. It does not take the trip count from the loop.
*/
-#define PARTIAL_SPILL_TEST(name, sz, count, msg, first, body) \
+#define PARTIAL_SPILL_TEST(name, sz, count, expect, first, body) \
SEC("socket") \
__success __log_level(2) \
-__msg("loop header at {{[0-9]+}}, header_count is " #count) \
-__msg("loop header at {{[0-9]+}}, " msg) \
+expect COUNT(count) \
__naked void name(void) \
{ \
asm volatile ( \
@@ -3324,18 +3052,18 @@ __naked void name(void) \
#define SPILL32_0 "*(u32 *)(r10 - 8) = r1;"
#define SPILL32_INC "r1 = *(u32 *)(r10 - 8); w1 += 1; *(u32 *)(r10 - 8) = r1;"
-#define NOT_UNROLLED "widening r6 to 0..0xf423f step 1"
+#define NOT_UNROLLED __msg(RANGE("{{[0-9]+}}", "R6", 0, 0xf423f))
/* 'if (...) cnt++' */
-PARTIAL_SPILL_TEST(partial_spill32_inc, u32, 10, "can't widen *fp-8, expr is %fp-8.{{[0-9]+}}=phi(*fp-8, (spill32 ",
+PARTIAL_SPILL_TEST(partial_spill32_inc, u32, 10, ONCE,
SPILL32_0, "if r6 & 1 goto 2f;" SPILL32_INC "2:")
-PARTIAL_SPILL_TEST(partial_spill16_inc, u16, 10, "can't widen *fp-8, expr is (spill16 ",
+PARTIAL_SPILL_TEST(partial_spill16_inc, u16, 10, ONCE,
"*(u16 *)(r10 - 8) = r1;", "r1 = *(u16 *)(r10 - 8); w1 += 1; *(u16 *)(r10 - 8) = r1;")
/* A loop in the loop does it */
-PARTIAL_SPILL_TEST(partial_spill32_inc_in_nested_loop, u32, 10, "can't widen *fp-8, expr is %fp-8.{{[0-9]+}}?",
+PARTIAL_SPILL_TEST(partial_spill32_inc_in_nested_loop, u32, 10, ONCE,
SPILL32_0, "r7 = 0; 2:" SPILL32_INC "r7 += 1; if r7 < 3 goto 2b;")
/* Two slots feed each other */
-PARTIAL_SPILL_TEST(partial_spill32_two_slots, u32, 10, "can't widen *fp-8, expr is (spill32 ",
+PARTIAL_SPILL_TEST(partial_spill32_two_slots, u32, 10, ONCE,
SPILL32_0 "*(u32 *)(r10 - 16) = r1;",
"r2 = *(u32 *)(r10 - 16);" SPILL32_INC "*(u32 *)(r10 - 16) = r1; *(u32 *)(r10 - 8) = r2;")
/* 'if (...) found = 1' */
@@ -3361,67 +3089,62 @@ PARTIAL_SPILL_TEST(partial_spill32_unknown, u32, 1000000, NOT_UNROLLED,
*/
#define SPILL32_LOOP(body) "*(u32 *)(r10 - 8) = r6; 1:" LIMIT_ITERATIONS body
#define SPILL32_LOAD "r1 = *(u32 *)(r10 - 8);"
-#define SPILL32_COUNTED(count, range) \
- __retval(count) \
- __msg("loop header at {{[0-9]+}}, header_count is [0.." #count "] ") \
- __msg("loop header at {{[0-9]+}}, widening *fp-8 to " range " ") \
- __msg("loop header at {{[0-9]+}}, clamping *fp-8 to " range " ")
+#define SPILL32_COUNTED(count, min, max) \
+ __retval(count) NO_GUARD __msg(RANGE("{{[0-9]+}}", "fp-8", min, max)) COUNT(count)
/* On big-endian '*(u32 *)(r10 - 8)' is the high half of the slot: there is no formula for it. */
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
/* Only the low half of the register that is stored at first gets to the slot */
RANGE_TEST(spill32_inc, "", 7ULL << 32, 7ULL << 32, 1, 5, 5,
SPILL32_LOOP(SPILL32_INC "if r1 != r7 goto 1b;"),
- __msg(" *fp-8=(spill32 (+ *fp-8 1)) / (spill32 (linear (zext32 *fp-8) 1))")
- SPILL32_COUNTED(5, "0..4"))
+ __retval(5) COUNT(5))
/* The jump tests a second load of the counter */
RANGE_TEST(spill32_second_load, "", 0, 0, 1, 5, 5,
SPILL32_LOOP(SPILL32_INC SPILL32_LOAD "if w1 != w7 goto 1b;"),
- SPILL32_COUNTED(5, "0..4"))
+ SPILL32_COUNTED(5, 0, 4))
/* What a compiler makes without 32-bit insns */
RANGE_TEST(spill32_add64, "", 0, 0, 1, 5, 5,
SPILL32_LOOP(SPILL32_LOAD "r1 += 1; *(u32 *)(r10 - 8) = r1; r1 <<= 32; r1 >>= 32;"
- "if r1 != r7 goto 1b;"), SPILL32_COUNTED(5, "0..4"))
+ "if r1 != r7 goto 1b;"), SPILL32_COUNTED(5, 0, 4))
/* 'first + step * count' is 0..8. The latch lets 5 through. */
RANGE_TEST(spill32_first_is_a_range, "", 0, 3, 1, 6, 6,
- SPILL32_LOOP(SPILL32_INC "if w1 < w7 goto 1b;"), SPILL32_COUNTED(6, "0..5"))
+ SPILL32_LOOP(SPILL32_INC "if w1 < w7 goto 1b;"), SPILL32_COUNTED(6, 0, 5))
RANGE_TEST(spill32_first_is_a_range_pre, "", 0, 3, 1, 6, 6,
SPILL32_LOOP(SPILL32_LOAD "if r1 >= r7 goto 2f;" SPILL32_INC "goto 1b;"),
- SPILL32_COUNTED(7, "0..6"))
+ SPILL32_COUNTED(7, 0, 6))
/* 2 and 0 in the slot are joined in front of the loop */
RANGE_TEST(spill32_first_is_precise, "", 0, 0, 1, 0, 1,
"if r7 == 0 goto +1; r6 = 2;" SPILL32_LOOP(SPILL32_INC "if w1 != 5 goto 1b;"),
- __msg("header_count is [0..5] ") __not_msg("header_count is [0..3] "))
+ __msg("has trip count 5{{$}}"))
/* The slot holds 64 bits when the loop starts. Nothing reads the high half: it's dead. */
RANGE_TEST(spill32_first_is_64bit, "", 0, 0, 1, 5, 5,
"*(u64 *)(r10 - 8) = r6; 1:" LIMIT_ITERATIONS SPILL32_INC "if w1 != w7 goto 1b;",
- SPILL32_COUNTED(5, "0..4"))
+ SPILL32_COUNTED(5, 0, 4))
/* A 64-bit spill of i. Its upper 32 bits are dead: the low 32 bits are the induction variable. */
#define SEXT32_SPILL_LOOP(bound) \
"1:" BUF("*(u32 *)(r10 - 16)") "w1 = *(u32 *)(r10 - 16); w1 += 1; r3 = (s32)r1;" \
"*(u64 *)(r10 - 16) = r3; if w1 != " #bound " goto 1b;"
SEXT32_TEST(sext32_iv_hi32_dead, 0, SEXT32_SPILL_LOOP(8), __success __retval(8)
- __msg(" *fp-16=(sext32 (+ *fp-16 1)) / (spill32 (linear (zext32 *fp-16) 1))")
- __msg("header_count is 8 ") __msg("widening *fp-16 to 0..7 "))
+ __msg(RANGE("{{[0-9]+}}", "fp-16", 0, 7)) __msg("has trip count 8{{$}}"))
SEXT32_TEST(sext32_iv_hi32_dead_oob, 0, SEXT32_SPILL_LOOP(9), OOB)
-/* The counter makes the address of a spill. Such a loop is not summarized, as with a register. */
+/* The iv makes the address of a spill. Nothing reads the spill, so no check fails. */
PARTIAL_SPILL_TEST(spill32_is_a_stack_offset, u32, 4,
- "can't widen *fp-8, expr is (spill32 (+ *fp-8 8)), requires exact stack-offset tracking",
+ __msg(RANGE("{{[0-9]+}}", "fp-8", 0, 24)),
SPILL32_0, SPILL32_LOAD "r2 = r10; r2 += -48; r2 += r1; *(u64 *)(r2 + 0) = r6;"
"w1 += 8; *(u32 *)(r10 - 8) = r1;")
/* But there is no exact address to lose when nothing is known about the number */
PARTIAL_SPILL_TEST(spill32_unknown_is_a_stack_offset, u32, 1000000,
- "widening *fp-8 to 0..0xffffffff step 1",
+ __msg(RANGE("{{[0-9]+}}", "fp-8", 0, 0xffffffff)),
"*(u32 *)(r10 - 8) = r0;", SPILL32_LOAD "r2 = r1; r2 &= 24; r3 = r10; r3 += -48; r3 += r2;"
"*(u64 *)(r3 + 0) = r6; w1 += 8; *(u32 *)(r10 - 8) = r1;")
#endif
/* A step above S16_MAX has no range, as with a register */
-PARTIAL_SPILL_TEST(spill32_large_step, u32, 4, "can't widen *fp-8, expr is (spill32 ",
+PARTIAL_SPILL_TEST(spill32_large_step, u32, 4, ONCE,
SPILL32_0, SPILL32_LOAD "w1 += 0x8000; *(u32 *)(r10 - 8) = r1;")
/* No formula: the load extends the sign, */
RANGE_TEST(spill32_sext_load, "", 0, 0, 1, 5, 5,
- SPILL32_LOOP(SPILL32_INC "r1 = *(s32 *)(r10 - 8); if r1 != r7 goto 1b;"), NO_COUNT)
+ SPILL32_LOOP(SPILL32_INC "r1 = *(s32 *)(r10 - 8); if r1 != r7 goto 1b;"), COUNT(5))
/* the sum is made from a different slot. */
RANGE_TEST(spill32_other_slot, "", 0, 0, 1, 5, 5,
"*(u32 *)(r10 - 16) = r6;"
@@ -3430,14 +3153,14 @@ RANGE_TEST(spill32_other_slot, "", 0, 0, 1, 5, 5,
/*
* r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3).
- * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3.
+ * The join of r7 and r8 must preserve base 1 and step gcd(6, 9) = 3.
*/
SEC("xdp")
__success __retval(1)
__log_level(2)
__msg("r7 += 1 {{.*}}step=1+6)")
__msg("r8 += 1 {{.*}}step=1+9)")
-__msg("loop header at 9, widening r7 to 1..28 step 3")
+__msg(RANGE("9", "R7", 1, 28))
__msg("9: {{.*}}R7=scalar({{.*}},step=1+3)")
__msg("14: R7=scalar({{.*}},step=1+3)")
__naked void conditional_scalar_assignment_gcd(void)
@@ -3492,25 +3215,24 @@ __naked void name(void) \
}
/* [0, max] that loses bits is in [0, max], and a bit that is 0 stays 0 */
-#define LOSES_BITS(max) __msg("r7 stays in 0.." #max ", it loses bits only")
-#define UNROLLED(expr) __msg("can't widen r7, expr is " expr) __not_msg("it loses bits only")
+#define LOSES_BITS(max) NO_GUARD __msg("R7=scalar({{.*}}var_off=(0x0; " #max "))") COUNT(8)
NOT_LINEAR_TEST(and_clears_a_bit, "r7 &= 255;", "r1 = 1; r1 <<= r6; r1 ^= -1; r7 &= r1;",
- LOSES_BITS(255))
-NOT_LINEAR_TEST(and_on_the_right, "r7 &= 255;", "r1 = r8; r1 &= r7; r7 = r1;", LOSES_BITS(255))
-NOT_LINEAR_TEST(and_0_or_64, "r7 &= 64;", "r7 &= r8;", LOSES_BITS(64))
+ LOSES_BITS(0xff))
+NOT_LINEAR_TEST(and_on_the_right, "r7 &= 255;", "r1 = r8; r1 &= r7; r7 = r1;", LOSES_BITS(0xff))
+NOT_LINEAR_TEST(and_0_or_64, "r7 &= 64;", "r7 &= r8;", LOSES_BITS(0x40))
NOT_LINEAR_TEST(zext32_or_nothing, "r7 >>= 8;", "if r8 == 0 goto +1; w7 = w7;",
- LOSES_BITS(72057594037927935))
-/* 64..127 can become 0 */
-NOT_LINEAR_TEST(and_64_to_127, "r7 &= 63; r7 |= 64;", "r7 &= 127;", UNROLLED("(& r7 127)"))
-/* '>>' does not make a number greater, but iterations one by one tell more: 0 after 8 shifts */
+ LOSES_BITS(0xffffffffffffff))
+/* bit 6 stays */
+NOT_LINEAR_TEST(and_64_to_127, "r7 &= 63; r7 |= 64;", "r7 &= 127;", __msg(RANGE("{{[0-9]+}}", "R7", 64, 127)))
+/* '>>' does not make a number greater */
NOT_LINEAR_TEST(rsh_on_one_path, "r7 &= 255;", "if r8 == 0 goto +1; r7 >>= 1;",
- UNROLLED("%r7.8=phi(r7, (>> r7 1))"))
-NOT_LINEAR_TEST(rsh_then_and, "r7 &= 255;", "r7 >>= 1; r7 &= r8;", UNROLLED("(& (>> r7 1) r8)"))
-NOT_LINEAR_TEST(lsh_grows, "r7 &= 15;", "r7 <<= 1;", UNROLLED("(<< r7 1)"))
+ LOSES_BITS(0xff))
+NOT_LINEAR_TEST(rsh_then_and, "r7 &= 255;", "r7 >>= 1; r7 &= r8;", LOSES_BITS(0xff))
+NOT_LINEAR_TEST(lsh_grows, "r7 &= 15;", "r7 <<= 1;", __msg("R7=scalar() "))
/* bpf_compute_const_regs() knows that r7 is the same constant in each iteration */
-#define NOT_CHANGED __msg("widening r6 to 0..7")
+#define NOT_CHANGED __msg(RANGE("{{[0-9]+}}", "R6", 0, 7))
NOT_LINEAR_TEST(same_constant, "r7 = 17;", "r7 &= 31;", NOT_CHANGED)
NOT_LINEAR_TEST(same_wide_constant, "r7 = 0xffffffff00000011 ll;", "r7 &= -239;", NOT_CHANGED)
@@ -3550,6 +3272,7 @@ __naked void name(void) \
#define VALID __success __retval(0)
#define MAP_OOB __failure __msg("invalid access to map value")
#define UNBOUND __failure __msg("unbounded min value")
+#define OUTSIDE __failure __msg("value is outside of the allowed memory range")
/* r3 is the address in r1 as a scalar. It does not bound the offset of r1. */
LINKED(pointer_is_not_scalar, __failure,
@@ -3642,7 +3365,7 @@ LINKED(operand_has_a_leader_but_no_value, __failure __msg("R1 unbounded memory a
"w3 = w8; r2 = r8; r2 >>= 32; r2 += r7; r8 = 0; if r7 != 0 goto out_%=;"
"r1 = r6; r1 += r2; r0 = r3;")
/* the bound is filled from 8 bytes: a 32-bit spill below 4 bytes that nobody knows */
-LINKED(bound_of_loop_above_a_32bit_spill, UNBOUND,
+LINKED(bound_of_loop_above_a_32bit_spill, MAP_OOB,
"*(u64 *)(r10 - 16) = r8; r3 = 5; *(u32 *)(r10 - 16) = r3; r1 = 0;"
"l_%=: r2 = *(u64 *)(r10 - 16); r1 += 1; if r1 < r2 goto l_%=; r1 *= 12; r1 += r6;")
LINKED(zero_extension_of_low_8_bits, MAP_OOB,
@@ -3782,10 +3505,10 @@ LINKED(copy_refined_by_jump_with_dead_branch, VALID,
"m_%=: if r2 > 62 goto out_%=; r1 = r6; r1 += r3;"
LINKED(congruent_phis, VALID, TWO_PATHS("r3 = r2; r3 += 1;", "r3 = r2; r3 += 1;"))
-LINKED(phis_with_other_delta_on_one_path, UNBOUND,
+LINKED(phis_with_other_delta_on_one_path, MAP_OOB,
TWO_PATHS("r3 = r2; r3 += 1;", "r3 = r2; r3 += 2;"))
/* One of three edges brings unrelated values */
-LINKED(phis_not_linked_on_one_of_three_edges, UNBOUND,
+LINKED(phis_not_linked_on_one_of_three_edges, __failure __msg("R1 unbounded memory access"),
"r3 = r8; r3 >>= 2; r2 = 0; if r7 == 20 goto m_%=;"
TWO_PATHS("r3 = r2; r3 += 1;", "r3 = r2; r3 += 1;"))
/* (zext32 x) on both paths: linked in 64 bits */
@@ -3887,7 +3610,7 @@ LINKED(irreducible_loop, VALID,
"if r7 > 5 goto m_%=; l_%=: r7 += 1;"
"m_%=: r2 = r8; r2 += 1; r2 += 1; if r8 > 61 goto out_%=; r1 = r6; r1 += r2;"
"r0 = *(u8 *)(r1 + 0); if r7 < 10 goto l_%=;")
-LINKED(irreducible_loop_fallthrough_pred, UNBOUND,
+LINKED(irreducible_loop_fallthrough_pred, OUTSIDE,
"r2 = r7; if r7 > 5 goto m_%=; l_%=: r7 += 1; r2 = r8;"
"m_%=: if r8 > 63 goto out_%=; r1 = r6; r1 += r2;"
"r0 = *(u8 *)(r1 + 0); if r7 < 10 goto l_%=;")
@@ -3896,7 +3619,7 @@ LINKED(irreducible_loop_next_bb, VALID,
"m_%=: r2 = r8; r2 += 1; r2 += 1; if r7 == 3 goto n_%=; r3 = 0;"
"n_%=: if r8 > 61 goto out_%=; r1 = r6; r1 += r2;"
"r0 = *(u8 *)(r1 + 0); if r7 < 10 goto l_%=;")
-LINKED(irreducible_loop_secondary_entry, UNBOUND,
+LINKED(irreducible_loop_secondary_entry, OUTSIDE,
"r9 = 0; o_%=: r9 += 1; call %[bpf_ktime_get_ns]; r8 = r0; r2 = r7; if r7 > 5 goto m_%=;"
"l_%=: r7 += 1; r2 = r8; r2 += 1; r2 += 1; if r7 == 3 goto m_%=; r3 = 0;"
"m_%=: if r8 > 61 goto out_%=; r1 = r6; r1 += r2;"
@@ -3915,7 +3638,7 @@ LINKED(natural_loop_is_secondary_entry, MAP_OOB,
* holds a reference, so may_goto goes to the exit of the outer loop: an edge that the verifier
* adds. Every iteration of the outer loop restores r3. At the header of the inner one it is r7.
*/
-LINKED(added_edge, UNBOUND,
+LINKED(added_edge, OUTSIDE,
"r1 = %[ringbuf] ll; r2 = 8; r3 = 0; call %[bpf_ringbuf_reserve]; if r0 == 0 goto out_%=;"
"*(u64 *)(r10 - 16) = r0; r3 = r8;"
"o_%=: r2 = r3; r3 = r7;"
@@ -3962,16 +3685,15 @@ __naked void break_pkt_pointers_id(void)
}
/*
- * A loop induction variable used to compute the base address of a store to the
- * stack must not be widened: the spill offset would become varying, which the
- * verifier does not track.
+ * A loop induction variable is the base address of a store to the stack.
+ * It is widened. The store has a varying offset. No check depends on what it writes.
*/
SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking")
-__naked void no_widen_stack_spill(void)
+__msg(RANGE("2", "R2", 0, 24))
+__naked void widen_stack_spill(void)
{
asm volatile (" \
r0 = 0; \
@@ -3988,17 +3710,16 @@ __naked void no_widen_stack_spill(void)
" ::: __clobber_all);
}
/*
- * Same hazard across a loop nest: the outer induction variable r2 addresses a
- * stack store performed inside the inner loop. The dependency is pulled up from
- * the inner loop, so the outer loop must not widen r2.
+ * The same across a loop nest: the outer induction variable r2 addresses a
+ * stack store performed inside the inner loop.
*/
SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking")
-__msg("loop header at 3, widening r1 to 0..1 step 1")
-__naked void no_widen_stack_spill_nested(void)
+__msg(RANGE("2", "R2", 0, 24))
+__msg(RANGE("3", "R1", 0, 1))
+__naked void widen_stack_spill_nested(void)
{
asm volatile (" \
r0 = 0; \
@@ -4042,16 +3763,6 @@ SEC("xdp")
__failure
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev at header 9:")
-__msg_next(" r6=(+ %r6.13? 1) / ?")
-__msg_next(" scev at latch 20:")
-__msg_next(" r6=(+ %r6.13? 1) / (+ %r6.13? 1)")
-__msg_next("scev at header 13:")
-__msg_next(" r9=(+ r9 1) / (linear r9 1)")
-__msg_next(" scev at latch 16:")
-__msg_next(" r6=-1000 / -1000")
-__msg_next(" r8=r6 / r6")
-__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)")
__msg("R6=-999")
__msg("R1 min value is negative")
__naked void nested_loop_exit_clobbers_reg(void)
@@ -4119,18 +3830,7 @@ SEC("xdp")
__success __retval(3)
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("r6 = *(u64 *)(r10 -8) {{.*}}; r6 %r6.4? -> r6")
-__msg("scev at header 1:")
-__msg_next(" r6=(+ r6 1) / (linear r6 1)")
-__msg_next(" scev at latch 16:")
-__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)")
-__msg("scev at header 2:")
-__msg_next(" r7=(+ r7 1) / (linear r7 1)")
-__msg("scev at header 4:")
-__msg_next(" r9=(+ r9 1) / (linear r9 1)")
-__msg(" scev at latch 9:")
-__msg_next(" r8=r6 / r6")
-__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)")
+__msg("loop at 1 has trip count 3{{$}}")
__naked void nested_loop_exit_preserves_reg(void)
{
asm volatile (
@@ -4164,15 +3864,14 @@ __naked void nested_loop_exit_preserves_reg(void)
}
/*
- * Complement of no_widen_stack_spill: a sub-register (1-byte) store to the stack
- * lands as STACK_MISC and carries no tracked value, so the induction variable
- * addressing it (r2) is still widened.
+ * As widen_stack_spill, with a 1-byte store. It lands as STACK_MISC.
+ * The induction variable that addresses it (r2) is widened.
*/
SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("widening r2 to 0..24 step 8")
+__msg(RANGE("{{[0-9]+}}", "R2", 0, 24))
__naked void widen_byte_stack_store(void)
{
asm volatile (" \
@@ -4191,16 +3890,16 @@ __naked void widen_byte_stack_store(void)
}
/*
- * A fill (BPF_LDX) at a varying stack offset loses precision just like a spill,
- * so the induction variable computing the load base (r2) must not be widened.
+ * A fill (BPF_LDX) at a varying stack offset loses precision just like a spill.
+ * No check depends on what the fill reads, so the load base (r2) is widened.
* The slots are initialized up front so the fill itself is a valid read.
*/
SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 6, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking")
-__naked void no_widen_stack_fill(void)
+__msg(RANGE("6", "R2", 0, 24))
+__naked void widen_stack_fill(void)
{
asm volatile (" \
r0 = 0; \
@@ -4231,17 +3930,15 @@ __naked void no_widen_stack_fill(void)
* r7 = 1;
* }
*
- * SCEV expression for r7 is (any r7 8) and r7 is used to
- * address the stack memory. Avoid widening it, otherwise
- * verifier won't know the type of the slots[r7] expression.
+ * r7 is r7 or 8, and it is used to
+ * address the stack memory. Pass 0 widens it and doesn't know
+ * the type of the slots[r7] expression. The next passes keep it apart.
*/
SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev at header 4:")
-__msg_next("r7=%r7.11=phi(r7, 8) / (any r7 8)")
-__msg("loop header at 4, can't widen r7, expr is {{.*}}, requires exact stack-offset tracking")
+__msg("pass 1")
__naked void no_widen_stack_fill_any(void)
{
asm volatile (" \
@@ -4274,9 +3971,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("can't widen {{.*}}, requires exact stack-offset tracking")
+ONCE
#ifndef __clang__
-/* The issue is unrelated to SCEV */
__skip("GCC emits a stack-pointer loop condition the verifier cannot resolve")
#endif
int no_widen_dynptr_kfunc_arg(struct xdp_md *ctx)
@@ -4299,8 +3995,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("header_count is 4 ")
-__msg("widening r6 to -16..-13 step 1")
+__msg(RANGE("{{[0-9]+}}", "R6", -16, -13))
+__msg("has trip count 4{{$}}")
__naked void ptr_stack(void)
{
asm volatile (" \
@@ -4324,7 +4020,7 @@ __naked __noinline __used static int ptr_stack_callee(void)
r0 = 0; \
1: r6 += 1; \
r0 += 1; \
- if r0 & 8 goto 4f; /* bound main pass iteration, but hide it from SCEV */ \
+ if r0 & 8 goto 4f; /* bound main pass iteration, it gives no count */ \
if r6 < r7 goto 1b; /* r6 and r7 are from different frames, can't be widened */ \
4: r0 = 0; \
exit; \
@@ -4335,9 +4031,8 @@ __naked __noinline __used static int ptr_stack_callee(void)
SEC("xdp")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r6 (fp), r7 (fp)")
-__not_msg("widening r6")
__naked void ptr_stack_other_frame(void)
{
asm volatile (" \
@@ -4376,21 +4071,22 @@ __naked void name(void) \
}
PTR_RANGE_TEST(ptr_first_is_below_or_at_fp, "r0 &= 63;", "r6", 1, "<=", "r7",
- __msg("loop header at {{[0-9]+}}, header_count is [0..64] "))
-PTR_RANGE_TEST(ptr_first_offset_is_wide, "r0 >>= 2;", "r6", 1, "<", "r7", NO_COUNT)
-PTR_RANGE_TEST(ptr_bound_offset_is_wide, "r0 >>= 2;", "r7", -1, ">", "r6", NO_COUNT)
+ __msg("loop at {{[0-9]+}} has trip count 64{{$}}"))
+PTR_RANGE_TEST(ptr_first_offset_is_wide, "r0 >>= 2;", "r6", 1, "<", "r7",
+ NO_COUNT)
+PTR_RANGE_TEST(ptr_bound_offset_is_wide, "r0 >>= 2;", "r7", -1, ">", "r6",
+ NO_COUNT)
/* Constants: the offset goes from -55 to 1, across 0 */
PTR_RANGE_TEST(ptr_const_crosses_fp, "r0 = 0;", "r6", 8, "<", "r7",
- __msg("loop header at {{[0-9]+}}, header_count is [0..8] "))
+ __msg("loop at {{[0-9]+}} has trip count 8{{$}}"))
/* 'for (p = buf; p != end; p += 21)': fp-63 and fp-42 get to fp */
PTR_RANGE_TEST(ptr_ne_mul21, "r0 &= 1; r0 *= 21;", "r6", 21, "!=", "r7",
- __msg("loop header at {{[0-9]+}}, header_count is [0..3] "))
+ __msg("loop at {{[0-9]+}} has trip count 3{{$}}"))
SEC("xdp")
__success
__log_level(2)
-__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)")
-__msg("loop header at 11, can't compute iterations count at 14")
+NO_COUNT
__naked void ptr_map_value_jmp32(void)
{
asm volatile (" \
@@ -4417,7 +4113,7 @@ __naked void ptr_map_value_jmp32(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 11, can't compute iterations count")
+__msg("loop at 11 has no trip count")
__naked void ptr_map_value_alu32_latch(void)
{
asm volatile (" \
@@ -4444,7 +4140,7 @@ __naked void ptr_map_value_alu32_latch(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 11, can't compute iterations count")
+NO_COUNT
__naked void ptr_map_value_alu32_latch2(void)
{
asm volatile (" \
@@ -4472,7 +4168,7 @@ __naked void ptr_map_value_alu32_latch2(void)
SEC("xdp")
__success
__log_level(2)
-__msg("loop header at 8, can't widen r0, expr is (zext32 (+ r0 1))")
+__msg("loop at 8 has trip count 4{{$}}")
__naked void ptr_map_value_alu32_non_latch(void)
{
asm volatile (" \
@@ -4498,9 +4194,8 @@ __naked void ptr_map_value_alu32_non_latch(void)
SEC("xdp")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r6 (pkt_meta), r7 (pkt)")
-__not_msg("widening r6")
__naked void ptr_packet_meta_other_origin(void)
{
asm volatile (" \
@@ -4525,8 +4220,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("header_count is 4 ")
-__msg("widening r6 to 0..3 step 1")
+__msg(RANGE("{{[0-9]+}}", "R6", 0, 3))
+__msg("has trip count 4{{$}}")
__naked void ptr_map_value_two_lookups(void)
{
asm volatile (" \
@@ -4568,9 +4263,8 @@ __naked void ptr_map_value_two_lookups(void)
SEC("xdp")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)")
-__not_msg("widening r6")
__naked void ptr_map_value_two_lookups_inner_map(void)
{
asm volatile (" \
@@ -4611,8 +4305,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("header_count is 4 ")
-__msg("widening r6 to 0..3 step 1")
+__msg(RANGE("{{[0-9]+}}", "R6", 0, 3))
+__msg("has trip count 4{{$}}")
__naked void ptr_map_value(void)
{
asm volatile (" \
@@ -4638,9 +4332,8 @@ __naked void ptr_map_value(void)
SEC("xdp")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)")
-__not_msg("widening r6")
__naked void ptr_map_value_other_map(void)
{
asm volatile (" \
@@ -4675,8 +4368,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("header_count is 4 ")
-__msg("widening r6 to 0..3 step 1")
+__msg(RANGE("{{[0-9]+}}", "R6", 0, 3))
+__msg("has trip count 4{{$}}")
__naked void ptr_mem(void)
{
asm volatile (" \
@@ -4705,9 +4398,8 @@ __naked void ptr_mem(void)
SEC("xdp")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r6 (ringbuf_mem), r7 (ringbuf_mem)")
-__not_msg("widening r6")
__naked void ptr_mem_other_reservation(void)
{
asm volatile (" \
@@ -4749,8 +4441,7 @@ SEC("iter/bpf_map_elem")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("header_count is 4 ")
-__msg("can't widen r6, expr is (+ r6 8)")
+ONCE
__naked void ptr_buf(void)
{
asm volatile (" \
@@ -4772,9 +4463,8 @@ __naked void ptr_buf(void)
SEC("iter/bpf_map_elem")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r6 (buf), r7 (buf)")
-__not_msg("widening r6")
__naked void ptr_buf_other_load(void)
{
asm volatile (" \
@@ -4800,9 +4490,8 @@ __naked void ptr_buf_other_load(void)
SEC("xdp")
__success
__log_level(2)
+NO_COUNT
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("scev: incompatible latch operands r1 (map_value), scalar 4")
-__not_msg("widening r1")
__naked void ptr_vs_scalar_no_widen(void)
{
asm volatile (" \
@@ -4832,8 +4521,8 @@ out_%=: \
SEC("xdp")
__success
__log_level(2)
+__msg("loop at 9 has trip count 3{{$}}")
__flag(BPF_F_TEST_STATE_FREQ)
-__msg("loop header at 9, can't widen r1, expr is (+ r1 8)")
__naked void no_widen_maybe_null(void)
{
asm volatile (" \
@@ -4924,8 +4613,7 @@ __msg("entering loop 1")
__msg("entering loop 4")
__msg("exiting loop 4")
__not_msg("entering loop 1")
-__msg("from 4 to 1:")
-__msg("loop_stack=1{{$}}")
+__msg("from 4 to 1: pruned")
__naked void nested_loop_backedge_to_outer_header(void)
{
asm volatile (" \
@@ -4948,8 +4636,8 @@ exit_%=: \
SEC("socket")
__success
__log_level(2)
-__msg("loop header at 2, header_count is 10")
-__msg("loop header at 2, widening r1 to 0..9 step 1")
+__msg(RANGE("2", "R1", 0, 9))
+__msg("loop at 2 has trip count 10{{$}}")
__flag(BPF_F_TEST_STATE_FREQ)
__naked void post_cond_ivar_src(void)
{
@@ -4967,9 +4655,9 @@ __naked void post_cond_ivar_src(void)
SEC("socket")
__success
__log_level(2)
-__msg(" backedge from 4, latch at 4, dominating exits at 2")
-__msg("loop header at 2, header_count is [0..15]")
-__msg("loop header at 2, widening r1 to 0..14 step 1")
+__msg(" backedge from 4, latch at 4")
+__msg(RANGE("2", "R1", 0, 14))
+__msg("loop at 2 has trip count 15{{$}}")
__flag(BPF_F_TEST_STATE_FREQ)
__naked void dominating_exit_bounds_count(void)
{
@@ -4991,10 +4679,8 @@ SEC("xdp")
__success
__log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
-__msg(" scev at latch 5:")
-__msg(" r7=(zext32 r1) / (zext32 (linear r1 2))")
-__msg("loop header at 1, header_count is 100 ")
-__msg("loop header at 1, widening r1 to 0..198 step 2")
+__msg(RANGE("1", "R1", 0, 198))
+__msg("loop at 1 has trip count 100{{$}}")
__naked void post_cond_jlt32_copy(void)
{
asm volatile (" \
@@ -5019,10 +4705,8 @@ __naked void post_cond_jlt32_copy(void)
SEC("socket")
__success __retval(0)
__log_level(2)
-__msg("header_count is 100000 ")
-__msg("can't widen r6, expr is %r6.13")
-__not_msg("widening r8")
-__msg("clamping r8 to 0..0x1869f step 1")
+__msg(RANGE("{{[0-9]+}}", "R8", 0, 0x1869f))
+__msg("has trip count 100000{{$}}")
__naked void index_loaded_in_previous_iteration(void)
{
asm volatile (" \
@@ -5053,7 +4737,7 @@ __naked void index_loaded_in_previous_iteration(void)
* iteration, 16 bits in the others: the state that comes back to the head next is not within it.
*/
SEC("socket")
-__failure __msg("R1 unbounded memory access")
+__failure __msg("R1 max value is outside of the allowed memory range")
__naked void first_iteration_loads_less(void)
{
asm volatile (" \
@@ -5330,7 +5014,7 @@ __naked void clamp_pkt_pointer_keeps_range_of_entry(void)
* in the second iteration no byte from r6 is in the packet.
*/
SEC("tc")
-__failure __msg("invalid access to packet, off=56 size=2, R6(id=2,off=56,r=56)")
+__failure __msg("invalid access to packet, off=56 size=2, R6(id={{[0-9]+}},off=56,r=56)")
__naked void widen_pkt_pointer_step_lowers_umax(void)
{
asm volatile (
@@ -5349,7 +5033,7 @@ __naked void widen_pkt_pointer_step_lowers_umax(void)
/* The pointer goes down. 8 bytes from the first r6 are in the packet, not 10. */
SEC("tc")
-__failure __msg("invalid access to packet, off=120 size=2, R6(id=2,off=120,r=120)")
+__failure __msg("invalid access to packet, off=120 size=2, R6(id={{[0-9]+}},off=120,r=120)")
__naked void widen_pkt_pointer_step_lowers_umax_down(void)
{
asm volatile (
@@ -5371,7 +5055,7 @@ __naked void widen_pkt_pointer_step_lowers_umax_down(void)
* r5 is 0..56 and has the range of r6. In the first iteration no byte from r5 is in the packet.
*/
SEC("tc")
-__failure __msg("invalid access to packet, off=56 size=2, R5(id=2,off=56,r=56)")
+__failure __msg("invalid access to packet, off=56 size=2, R5(id={{[0-9]+}},off=56,r=56)")
__naked void widen_pkt_pointer_negative_offset(void)
{
asm volatile (
@@ -5418,7 +5102,7 @@ __naked void clamp_pkt_pointer_step_lowers_umax(void)
/* r5 is r5 = r6 + 8 or r6 at the head. Here var_off lowers umax. No byte from r6 + 8 is valid. */
SEC("tc")
-__failure __msg("invalid access to packet, off=56 size=2, R5(id=2,off=56,r=56)")
+__failure __msg("invalid access to packet, off=56 size=2, R5(id={{[0-9]+}},off=56,r=56)")
__naked void union_pkt_pointer_var_off_lowers_umax(void)
{
asm volatile (
@@ -5461,8 +5145,8 @@ __naked void widen_pkt_pointer_keeps_valid_bytes(void)
SEC("tc")
__success
__log_level(2)
-__msg("widening r6 to 32..96 step 32")
-__msg("6: R6=pkt(id=1,r=0,smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=96")
+__msg(RANGE("{{[0-9]+}}", "R6", 32, 96))
+__msg("6: R6=pkt(id={{[0-9]+}},r=0,smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=96")
__naked void widen_pkt_pointer_beyond_pkt_end(void)
{
asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c
index 7406e8a7d18..7b1f830e5e2 100644
--- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c
+++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c
@@ -435,8 +435,7 @@ l1_%=: r1 = *(u64*)(r6 + 0); \
SEC("tc")
__description("unpriv: spill/fill of different pointers stx - ctx and sock")
-/* bpf_sk_release(skb) is an error too: "type=ctx expected=sock". This one is found first. */
-__failure __msg("R1 invalid mem access 'sock_or_null'")
+__failure __msg("type=ctx expected=sock")
__naked void pointers_stx_ctx_and_sock(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c
index 5357d667fa0..6f621c69ff0 100644
--- a/tools/testing/selftests/bpf/verifier/calls.c
+++ b/tools/testing/selftests/bpf/verifier/calls.c
@@ -1572,7 +1572,7 @@
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_XDP,
- .errstr = "cannot return stack pointer",
+ .errstr = "R0 !read_ok",
.result = REJECT,
},
{
diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c
index 345cecc722a..88939871651 100644
--- a/tools/testing/selftests/bpf/verifier/map_kptr.c
+++ b/tools/testing/selftests/bpf/verifier/map_kptr.c
@@ -373,7 +373,7 @@
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.fixup_map_kptr = { 1 },
.result = REJECT,
- .errstr = "Unreleased reference id=4 alloc_insn=20",
+ .errstr = "Unreleased reference id=\t alloc_insn=20",
.fixup_kfunc_btf_id = {
{ "bpf_kfunc_call_test_acquire", 15 },
}
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 45/49] selftests/bpf: Adjust tests that get another verdict or may_goto
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (43 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 44/49] selftests/bpf: Adjust tests to the new log text Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 46/49] selftests/bpf: Test barriers and far pointers Alexei Starovoitov
` (3 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
Programs that were rejected and load now:
- Unprivileged 'ptr += reg' gets a barrier or one mask for the joined
range. The tests check the xlated insns.
- conditional_assignment_on_second_iteration: the read is unproven in
pass 0. A later pass peels the trips, and no trip gets to the read.
- from_stack_in_small_chunks, partial_fill_from_cleaned_pointer_spill:
a load of a part of a spilled pointer. They load with CAP_PERFMON and
are rejected without it, as before.
- to_leak_tainted_dst_reg: a far pointer is no error. It loads with
CAP_PERFMON and is rejected without it, as before.
- test_get_stack_rawtp: the buffer is not two times larger any more.
- New: callback_full_frame, an arena stack entry with a full frame.
- Four programs are safe. Each gets an unsafe twin:
pointer_minus_index p + 63 - i, i is 0..63
callee_writes_memory_of_caller tests x <= 63, stores x / 2
callee_writes_through_alias the same through a second pointer
pkt_add_negative data + 20 + (-4..4), 54 bytes checked
Programs that load as before and get may_goto: the loop has no trip
count, and no error asks to peel it.
do { r8 += 1; r7 <<= 1; } while (r7 != 0x400);
One program is rejected now. clang makes one call of bpf_obj_drop()
for two types in local_kptr_stash.c:
if (...) r8 = plain_local; no special field
else r8 = node_data; a kptr
r1 = r8
call bpf_obj_drop
Declare bpf_obj_drop() with nomerge, as struct_ops_refcounted.c does
for bpf_task_release().
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
.../selftests/bpf/progs/local_kptr_stash.c | 3 +
.../bpf/progs/test_get_stack_rawtp.c | 20 +-----
.../bpf/progs/verifier_arena_stack.c | 55 ++++++++++++++++
.../bpf/progs/verifier_bounds_deduction.c | 14 ++--
.../selftests/bpf/progs/verifier_budget.c | 9 ++-
.../bpf/progs/verifier_direct_packet_access.c | 34 ++++++++--
.../selftests/bpf/progs/verifier_loop_walk.c | 8 ++-
.../selftests/bpf/progs/verifier_map_ptr.c | 18 +++--
.../selftests/bpf/progs/verifier_spill_fill.c | 18 ++---
.../selftests/bpf/progs/verifier_trip_count.c | 26 +++++---
.../selftests/bpf/progs/verifier_unpriv.c | 3 +-
.../bpf/progs/verifier_value_ptr_arith.c | 66 +++++++++++++++----
12 files changed, 201 insertions(+), 73 deletions(-)
diff --git a/tools/testing/selftests/bpf/progs/local_kptr_stash.c b/tools/testing/selftests/bpf/progs/local_kptr_stash.c
index d736506a4c8..d86ae65abe6 100644
--- a/tools/testing/selftests/bpf/progs/local_kptr_stash.c
+++ b/tools/testing/selftests/bpf/progs/local_kptr_stash.c
@@ -8,6 +8,9 @@
#include "../bpf_experimental.h"
#include "../test_kmods/bpf_testmod_kfunc.h"
+/* Do not make one call for the drop of plain_local and the drop of node_data */
+__attribute__((nomerge)) extern void bpf_obj_drop(void *p) __weak __ksym;
+
struct plain_local;
struct node_data {
diff --git a/tools/testing/selftests/bpf/progs/test_get_stack_rawtp.c b/tools/testing/selftests/bpf/progs/test_get_stack_rawtp.c
index b6a6eb279e5..41714c2182f 100644
--- a/tools/testing/selftests/bpf/progs/test_get_stack_rawtp.c
+++ b/tools/testing/selftests/bpf/progs/test_get_stack_rawtp.c
@@ -29,29 +29,11 @@ struct {
__type(value, struct stack_trace_t);
} stackdata_map SEC(".maps");
-/* Allocate per-cpu space twice the needed. For the code below
- * usize = bpf_get_stack(ctx, raw_data, max_len, BPF_F_USER_STACK);
- * if (usize < 0)
- * return 0;
- * ksize = bpf_get_stack(ctx, raw_data + usize, max_len - usize, 0);
- *
- * If we have value_size = MAX_STACK_RAWTP * sizeof(__u64),
- * verifier will complain that access "raw_data + usize"
- * with size "max_len - usize" may be out of bound.
- * The maximum "raw_data + usize" is "raw_data + max_len"
- * and the maximum "max_len - usize" is "max_len", verifier
- * concludes that the maximum buffer access range is
- * "raw_data[0...max_len * 2 - 1]" and hence reject the program.
- *
- * Doubling the to-be-used max buffer size can fix this verifier
- * issue and avoid complicated C programming massaging.
- * This is an acceptable workaround since there is one entry here.
- */
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 1);
__type(key, __u32);
- __type(value, __u64[2 * MAX_STACK_RAWTP]);
+ __type(value, __u64[MAX_STACK_RAWTP]);
} rawdata_map SEC(".maps");
SEC("raw_tracepoint/sys_enter")
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_stack.c b/tools/testing/selftests/bpf/progs/verifier_arena_stack.c
index b46e30a9c67..4feaebf0d72 100644
--- a/tools/testing/selftests/bpf/progs/verifier_arena_stack.c
+++ b/tools/testing/selftests/bpf/progs/verifier_arena_stack.c
@@ -737,6 +737,61 @@ __naked void callback(void)
: __clobber_all);
}
+static __naked __noinline __used long add_index_cb(void)
+{
+ asm volatile (
+ "r3 = *(u64 *)(r2 + 0);"
+ "r3 += r1;"
+ "*(u64 *)(r2 + 0) = r3;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/* takes a stack in arena, has all of its own frame in use, and bpf_loop() is inlined */
+static __naked __noinline __used long full_frame_cb(void)
+{
+ asm volatile (
+ ".8byte %[adj];"
+ ".8byte %[stx16];" /* *(u64 *)(r12 - 16) = r1 */
+ "*(u64 *)(r10 - %[depth]) = r1;"
+ "r3 = r2;"
+ "r1 = 3;"
+ "r2 = add_index_cb ll;"
+ "r4 = 0;"
+ "call %[bpf_loop];"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_loop), __imm_const(depth, 2008),
+ __imm_insn(adj, ADJ(16)),
+ __imm_insn(stx16, STX(BPF_DW, -16, BPF_REG_1))
+ : __clobber_all);
+}
+
+SEC("syscall")
+__description("arena stack: the arguments of a callback need no room of their own")
+__success __retval(6)
+__xlated("*(u64 *)(r10 -16) = r2")
+__xlated("*(u64 *)(r10 -24) = r3")
+__naked void callback_full_frame(void)
+{
+ asm volatile (
+ "r1 = 0;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "r1 = 2;"
+ "r2 = full_frame_cb ll;"
+ "r3 = r10;"
+ "r3 += -8;"
+ "r4 = 0;"
+ "call %[bpf_loop];"
+ "r0 = *(u64 *)(r10 - 8);"
+ "exit;"
+ :
+ : __imm(bpf_loop)
+ : __clobber_all);
+}
+
/* has no frame: its first insn is the call, not r12 += N */
static __naked __noinline __used long calls_loop_cb(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c b/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c
index 14e10760606..63baf2fc61f 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds_deduction.c
@@ -21,8 +21,11 @@ l0_%=: r0 -= r1; \
SEC("socket")
__description("check deducing bounds from const, 2")
-__success __failure_unpriv
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("nospec")
+__xlated_unpriv("r1 -= r0")
+#endif
__retval(1)
__naked void deducing_bounds_from_const_2(void)
{
@@ -53,8 +56,11 @@ l0_%=: r0 -= r1; \
SEC("socket")
__description("check deducing bounds from const, 4")
-__success __failure_unpriv
-__msg_unpriv("R6 has pointer with unsupported alu operation")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("nospec")
+__xlated_unpriv("r6 -= r0")
+#endif
__retval(0)
__naked void deducing_bounds_from_const_4(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_budget.c b/tools/testing/selftests/bpf/progs/verifier_budget.c
index 890b316166c..1bef19db503 100644
--- a/tools/testing/selftests/bpf/progs/verifier_budget.c
+++ b/tools/testing/selftests/bpf/progs/verifier_budget.c
@@ -1874,14 +1874,17 @@ __naked void may_goto_stays_in_the_loop(void)
);
}
-/* No state is within another one: the walk gets to the end of the loop, no may_goto */
+/* The loop has no trip count and no error asks for peeling: may_goto is added */
SEC("socket")
__description("budget: ten trips that differ")
__success __retval(10) __log_level(2)
-__msg("processed 34 insns")
+__msg("processed {{[0-9][0-9]}} insns")
__xlated("r8 += 1")
__xlated("r7 <<= 1")
-__xlated("if r7 != 0x400 goto pc-3")
+__xlated("...")
+__xlated("if r13 == 0x0 goto")
+__xlated("...")
+__xlated("if r7 != 0x400 goto pc-{{[0-9]+}}")
__naked void ten_trips_that_differ(void)
{
asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
index 7c3dc9643eb..7c29ffed743 100644
--- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
+++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
@@ -1060,16 +1060,36 @@ __naked void pkt_add_unknown_keeps_range(void)
"r0 = *(u64*)(r2 + 8);");
}
+#define PKT_ADD_NEGATIVE(off) \
+ PKT_54_BYTES("if r4 > 8 goto l0_%=;" \
+ "r4 += -4;" \
+ "r2 += " #off ";" \
+ "r2 += r4;" \
+ "r0 = *(u8*)(r2 + 0);")
+
+/* The offset is 20 + (-4..4) = 16..24. 54 bytes are checked from the start of the packet. */
SEC("tc")
__description("direct packet access: checked pointer plus unknown that may be negative")
-__failure __msg("invalid access to packet, off=16 size=1, R2(id={{[0-9]+}},off=16,r=0)")
-__naked void pkt_add_negative_clears_range(void)
+__success __retval(0)
+__naked void pkt_add_negative(void)
{
- PKT_54_BYTES("if r4 > 8 goto l0_%=;"
- "r4 += -4;"
- "r2 += 20;"
- "r2 += r4;"
- "r0 = *(u8*)(r2 + 0);");
+ PKT_ADD_NEGATIVE(20);
+}
+
+SEC("tc")
+__description("direct packet access: the same before the packet")
+__failure __msg("R2 min value is negative")
+__naked void pkt_add_negative_below(void)
+{
+ PKT_ADD_NEGATIVE(3);
+}
+
+SEC("tc")
+__description("direct packet access: the same after what is checked")
+__failure __msg("invalid access to packet")
+__naked void pkt_add_negative_above(void)
+{
+ PKT_ADD_NEGATIVE(50);
}
/* The byte is in the packet, but a range that umax gets to is not kept */
diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_walk.c b/tools/testing/selftests/bpf/progs/verifier_loop_walk.c
index 4baf4717591..cd834d776d8 100644
--- a/tools/testing/selftests/bpf/progs/verifier_loop_walk.c
+++ b/tools/testing/selftests/bpf/progs/verifier_loop_walk.c
@@ -65,17 +65,19 @@ WALKS(walks_32, 11)
/*
* for (i = 7; i >= 0; i--) { if (i >= n) continue; a[i] = 0; }
- * A compiler tests 'i >= n' in front of the loop too and jumps into the middle of it. Such a loop
- * has no count. It's walked to its end whichever way its jumps go: no may_goto is added.
+ * A compiler tests 'i >= n' before the loop too and jumps into the middle of it. Such a loop
+ * has no trip count, so may_goto is added.
*/
SEC("socket")
__description("walk: a loop with two entries")
__success __retval(0)
__arch_x86_64
__arch_arm64
+__xlated("if r13 == 0x0 goto")
+__xlated("...")
__xlated("w3 += -1")
__xlated("if w3 s< 0x0 goto pc+2")
-__xlated("if w3 < w2 goto pc-6")
+__xlated("if w3 < w2 goto pc-{{[0-9]+}}")
__xlated("goto pc-4")
__xlated("exit")
__naked void two_entries(void)
diff --git a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c
index e0a65835c86..43fd9e26c76 100644
--- a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c
+++ b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c
@@ -142,8 +142,13 @@ __naked void ptr_read_ops_field_accepted(void)
SEC("socket")
__description("bpf_map_ptr: r = 0, map_ptr = map_ptr + r")
-__success __failure_unpriv
-__msg_unpriv("R1 has pointer with unsupported alu operation")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r2 += -8")
+__xlated_unpriv("...")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r1 += r0")
+#endif
__retval(0)
__naked void map_ptr_map_ptr_r(void)
{
@@ -166,8 +171,13 @@ __naked void map_ptr_map_ptr_r(void)
SEC("socket")
__description("bpf_map_ptr: r = 0, r = r + map_ptr")
-__success __failure_unpriv
-__msg_unpriv("R0 has pointer with unsupported alu operation")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r2 += -8")
+__xlated_unpriv("...")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r1 += r0")
+#endif
__retval(0)
__naked void _0_r_r_map_ptr(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
index 958ee0d6419..a7b11630eb6 100644
--- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
+++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
@@ -1208,14 +1208,14 @@ __naked void old_stack_misc_vs_cur_scalar(void)
}
/*
- * stacksafe(): the state with STACK_MISC is there, the state with a stack spill
- * of a non-scalar arrives. A program that may read uninitialized stack may
- * take a pointer for a number: the second state is within the first one.
+ * One state has STACK_MISC, the other one has a stack spill of a non-scalar.
+ * The slot is live, so the states stay apart.
*/
SEC("socket")
__success __log_level(2)
__msg("8: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar()")
-__msg("processed 10 insns")
+__msg("8: (79) r1 = *(u64 *)(r10 -8) ; R1=ctx()")
+__msg("processed 12 insns")
__naked void old_stack_misc_vs_cur_ctx_ptr(void)
{
asm volatile(
@@ -1233,7 +1233,7 @@ __naked void old_stack_misc_vs_cur_ctx_ptr(void)
"*(u64*)(r10 - 8) = r0;"
"lock *(u64*)(r10 - 8) += r0;"
"2:"
- /* read fp-8, it's looked at once */
+ /* read fp-8 */
"r1 = *(u64*)(r10 - 8);"
"exit;"
:
@@ -1399,11 +1399,11 @@ __naked void var_off_write_over_scalar_spill(void)
SEC("socket")
__description("partial fill from cleaned pointer spill")
-__failure
+__success __caps_unpriv(CAP_BPF) __failure_unpriv
__log_level(2)
-__msg("1: (05) goto pc+0")
-__msg("2: (61) r0 = *(u32 *)(r10 -4)")
-__msg("invalid size of register fill")
+__msg_unpriv("1: (05) goto pc+0")
+__msg_unpriv("2: (61) r0 = *(u32 *)(r10 -4)")
+__msg_unpriv("invalid size of register fill")
__flag(BPF_F_TEST_STATE_FREQ)
__naked void partial_fill_from_cleaned_pointer_spill(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_trip_count.c b/tools/testing/selftests/bpf/progs/verifier_trip_count.c
index c8e3ddbc530..69288309ef0 100644
--- a/tools/testing/selftests/bpf/progs/verifier_trip_count.c
+++ b/tools/testing/selftests/bpf/progs/verifier_trip_count.c
@@ -2751,13 +2751,13 @@ __naked void uninit_slot_counter(void)
*
* R7 is always 10 at the real exit. Widening loses the correlation between
* R6 and R7, leaving R7 in [5, 10] both at the loop header and after the loop.
- * The verifier therefore rejects the possible invalid stack read.
+ * Pass 0 does not prove the read. A later pass peels the trips and does not get to it.
*/
SEC("xdp")
-__failure
+__success __retval(0)
__log_level(2)
-__msg("2: {{.*}}R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))")
-__msg("7: R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))")
+__msg("2: {{.*}}R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf)")
+__msg("7: R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf)")
__msg("invalid read from stack R10 off=0 size=8")
__naked void conditional_assignment_on_second_iteration(void)
{
@@ -3387,9 +3387,11 @@ LINKED(dividend_tested, VALID,
LINKED(operand_is_overwritten, MAP_OOB,
"r9 = r7; r9 *= 8; call %[bpf_get_prandom_u32]; r7 = r0; r7 &= 255;"
"if r7 > 7 goto out_%=; r1 = r6; r1 += r9;")
-/* a pointer is evaluated below an add only */
-LINKED(pointer_minus_index, __failure,
+/* r7 is 0..63, so 63 - r7 is 0..63 */
+LINKED(pointer_minus_index, VALID,
"r1 = r6; r1 += 63; r1 -= r7; if r7 > 63 goto out_%=;" USE_R6)
+LINKED(pointer_minus_index_oob, __failure __msg("R1 min value is"),
+ "r1 = r6; r1 += 63; r1 -= r7; if r7 > 64 goto out_%=;" USE_R6)
/* r3 is an address as a scalar. r6 + r7 * 2 has the offset [0, 14]: that is not the range of r3 */
LINKED(offset_of_pointer_is_not_scalar, UNBOUND,
"r2 = r7; r2 *= 2; r5 = r6; r5 += r2; r3 = r5; r3 = le64 r3; if r7 > 7 goto out_%=;"
@@ -3418,14 +3420,17 @@ static __naked __noinline __used void name(void) \
"if r3 <= 63 goto +2; r3 = 0; *(u64 *)(r1 + 0) = r3; r0 = 0; exit;" \
::: __clobber_all); \
}
-#define CALL(callee) \
- "*(u64 *)(r10 - 16) = r8; r1 = r10; r1 += -16; r4 = r1; r2 = r8; r2 >>= 1;" \
+#define CALL_R2(set_r2, callee) \
+ "*(u64 *)(r10 - 16) = r8; r1 = r10; r1 += -16; r4 = r1;" set_r2 \
"call " #callee "; r2 = *(u64 *)(r10 - 16); r1 = r6; r1 += r2;"
+#define CALL(callee) CALL_R2("r2 = r8; r2 >>= 1;", callee)
CLAMP(clamp, "")
LINKED(callee_tests_memory_of_caller, VALID, CALL(clamp))
CLAMP(store_and_clamp, "*(u64 *)(r1 + 0) = r2;")
-LINKED(callee_writes_memory_of_caller, __failure, CALL(store_and_clamp))
+/* It tests r8 and stores r8 / 2. If r8 is 0..63 then r8 / 2 is 0..31. Else it stores 0. */
+LINKED(callee_writes_memory_of_caller, VALID, CALL(store_and_clamp))
+LINKED(callee_writes_memory_of_caller_oob, MAP_OOB, CALL_R2("r2 = r7;", store_and_clamp))
/* a store defines the memory */
CLAMP(store_and_test, "r3 = r2; *(u64 *)(r1 + 0) = r3;")
LINKED(callee_tests_what_it_stored, VALID, CALL(store_and_test))
@@ -3434,7 +3439,8 @@ CLAMP(store_next_and_clamp, "*(u64 *)(r1 + 8) = r2;")
LINKED(callee_writes_next_field, VALID, CALL(store_next_and_clamp))
/* another address base can be another name of the memory */
CLAMP(store_alias_and_clamp, "*(u64 *)(r4 + 0) = r2;")
-LINKED(callee_writes_through_alias, __failure, CALL(store_alias_and_clamp))
+LINKED(callee_writes_through_alias, VALID, CALL(store_alias_and_clamp))
+LINKED(callee_writes_through_alias_oob, MAP_OOB, CALL_R2("r2 = r7;", store_alias_and_clamp))
static __naked __noinline __used void clamp_through_r0(void)
{
asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c
index 7b1f830e5e2..2d5a0263853 100644
--- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c
+++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c
@@ -286,8 +286,7 @@ __naked void mangle_pointer_on_stack_2(void)
SEC("socket")
__description("unpriv: read pointer from stack in small chunks")
-__failure __msg("invalid size")
-__failure_unpriv
+__success __failure_unpriv __msg_unpriv("invalid size")
__naked void from_stack_in_small_chunks(void)
{
asm volatile (" \
diff --git a/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c b/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c
index 5a3893151f6..3deafc3a9e0 100644
--- a/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c
+++ b/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c
@@ -41,8 +41,16 @@ struct {
SEC("socket")
__description("map access: known scalar += value_ptr unknown vs const")
-__success __failure_unpriv
-__msg_unpriv("R1 tried to add from different maps, paths or scalars")
+__success __success_unpriv
+/*
+ * r1 is 2 or 3, the values have 16 or 48 bytes. The paths are joined. The mask passes 0..3
+ * and makes 0 of all other numbers, so the read is in the value, also in speculation.
+ */
+#ifdef SPEC_V1
+__xlated_unpriv("w13 = 3")
+__xlated_unpriv("...")
+__xlated_unpriv("r13 &= r1")
+#endif
__retval(1)
__naked void value_ptr_unknown_vs_const(void)
{
@@ -79,8 +87,16 @@ l2_%=: r0 = 1; \
SEC("socket")
__description("map access: known scalar += value_ptr const vs unknown")
-__success __failure_unpriv
-__msg_unpriv("R1 tried to add from different maps, paths or scalars")
+__success __success_unpriv
+/*
+ * r1 is 2 or 3, the values have 16 or 48 bytes. The paths are joined. The mask passes 0..3
+ * and makes 0 of all other numbers, so the read is in the value, also in speculation.
+ */
+#ifdef SPEC_V1
+__xlated_unpriv("w13 = 3")
+__xlated_unpriv("...")
+__xlated_unpriv("r13 &= r1")
+#endif
__retval(1)
__naked void value_ptr_const_vs_unknown(void)
{
@@ -117,8 +133,16 @@ l2_%=: r0 = 1; \
SEC("socket")
__description("map access: known scalar += value_ptr const vs const (ne)")
-__success __failure_unpriv
-__msg_unpriv("R1 tried to add from different maps, paths or scalars")
+__success __success_unpriv
+/*
+ * r1 is 3 or 5, the values have 16 or 48 bytes. The paths are joined. The mask passes 0..5
+ * and makes 0 of all other numbers, so the read is in the value, also in speculation.
+ */
+#ifdef SPEC_V1
+__xlated_unpriv("w13 = 5")
+__xlated_unpriv("...")
+__xlated_unpriv("r13 &= r1")
+#endif
__retval(1)
__naked void ptr_const_vs_const_ne(void)
{
@@ -406,8 +430,17 @@ l2_%=: r0 = 1; \
SEC("socket")
__description("map access: mixing value pointer and scalar, 1")
-__success __failure_unpriv
-__msg_unpriv("R2 tried to add from different maps, paths or scalars, pointer arithmetic with it prohibited for !root")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("goto pc+2")
+__xlated_unpriv("...")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r2 += r3")
+__xlated_unpriv("...")
+__xlated_unpriv("exit")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r0 = *(u8 *)(r0 +0)")
+#endif
__retval(0)
__naked void value_pointer_and_scalar_1(void)
{
@@ -460,8 +493,17 @@ l4_%=: /* fake-dead code; targeted from branch A to \
SEC("socket")
__description("map access: mixing value pointer and scalar, 2")
-__success __failure_unpriv
-__msg_unpriv("R2 tried to add from different maps, paths or scalars, pointer arithmetic with it prohibited for !root")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("goto pc+2")
+__xlated_unpriv("...")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r2 += r3")
+__xlated_unpriv("...")
+__xlated_unpriv("exit")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r0 = *(u8 *)(r0 +0)")
+#endif
__retval(0)
__naked void value_pointer_and_scalar_2(void)
{
@@ -1420,8 +1462,8 @@ l0_%=: r0 = 1; \
SEC("socket")
__description("map access: trying to leak tainted dst reg")
-__failure __msg("math between map_value pointer and 4294967295 is not allowed")
-__failure_unpriv
+__success
+__failure_unpriv __msg_unpriv("R2 pointer arithmetic of map value goes out of range")
__naked void to_leak_tainted_dst_reg(void)
{
asm volatile (" \
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 46/49] selftests/bpf: Test barriers and far pointers
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (44 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 45/49] selftests/bpf: Adjust tests that get another verdict or may_goto Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 47/49] selftests/bpf: Test passes, instances, precision and the failing path Alexei Starovoitov
` (2 subsequent siblings)
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
verifier_barrier.c: check the place of the barrier for each kind of
edge: inlined bpf_loop(), exit of a callback, may_goto, recursion, a
call that never returns, far jumps.
Check what a speculative state must not decide for other states.
Check that a far pointer gives no compare, no access and no constant
offset. Check the log of a dropped path, of a leak and of a speculative
state.
far_map_value.c: a load through a far pointer to a read-only map value
is not folded to a constant.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
.../selftests/bpf/prog_tests/far_map_value.c | 38 +
.../selftests/bpf/prog_tests/verifier.c | 2 +
.../selftests/bpf/progs/verifier_barrier.c | 849 ++++++++++++++++++
3 files changed, 889 insertions(+)
create mode 100644 tools/testing/selftests/bpf/prog_tests/far_map_value.c
create mode 100644 tools/testing/selftests/bpf/progs/verifier_barrier.c
diff --git a/tools/testing/selftests/bpf/prog_tests/far_map_value.c b/tools/testing/selftests/bpf/prog_tests/far_map_value.c
new file mode 100644
index 00000000000..f2fb2aaaa26
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/far_map_value.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+
+#define FAR_OFF (1 << 29) /* BPF_MAX_VAR_OFF */
+
+/* See far_rodata in verifier_barrier.c. Here ld_imm64 makes the far pointer. */
+void serial_test_far_map_value(void)
+{
+ LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG);
+ struct bpf_insn insns[] = {
+ BPF_LD_MAP_VALUE(BPF_REG_1, 0, FAR_OFF),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -16),
+ BPF_LDX_MEM(BPF_B, BPF_REG_2, BPF_REG_1, 0),
+ BPF_MOV64_IMM(BPF_REG_3, 2),
+ BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 2),
+ BPF_MOV64_IMM(BPF_REG_0, 0),
+ BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, 1),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
+ BPF_EXIT_INSN(),
+ };
+ int map_fd, prog_fd;
+
+ map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, FAR_OFF + 8, 1, &opts);
+ if (map_fd == -ENOMEM) {
+ test__skip();
+ return;
+ }
+ if (!ASSERT_GE(map_fd, 0, "map_create"))
+ return;
+ ASSERT_OK(bpf_map_freeze(map_fd), "map_freeze");
+ insns[0].imm = map_fd;
+ prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns,
+ ARRAY_SIZE(insns), NULL);
+ if (ASSERT_GE(prog_fd, 0, "prog_load"))
+ close(prog_fd);
+ close(map_fd);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 67151c55b03..ab15e69a622 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -21,6 +21,7 @@
#include "verifier_arena_globals2.skel.h"
#include "verifier_array_access.skel.h"
#include "verifier_async_cb_context.skel.h"
+#include "verifier_barrier.skel.h"
#include "verifier_basic_stack.skel.h"
#include "verifier_bitfield_write.skel.h"
#include "verifier_bounds.skel.h"
@@ -231,6 +232,7 @@ void test_verifier_arena_stack(void) { RUN_TESTS(verifier_arena_stack);
void test_verifier_arena_large(void) { RUN(verifier_arena_large); }
void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); }
void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); }
+void test_verifier_barrier(void) { RUN(verifier_barrier); }
void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); }
void test_verifier_bitfield_write(void) { RUN(verifier_bitfield_write); }
void test_verifier_bounds(void) { RUN(verifier_bounds); }
diff --git a/tools/testing/selftests/bpf/progs/verifier_barrier.c b/tools/testing/selftests/bpf/progs/verifier_barrier.c
new file mode 100644
index 00000000000..7e68fc072e3
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_barrier.c
@@ -0,0 +1,849 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Where the barrier goes, and what a speculative state must not decide */
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, long);
+} array SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, int);
+} progs SEC(".maps");
+
+struct elem {
+ struct bpf_timer timer;
+ struct task_struct __kptr *task;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct elem);
+} timers SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_RINGBUF);
+ __uint(max_entries, 4096);
+} ring SEC(".maps");
+
+long data[8];
+
+void bpf_throw(u64 cookie) __ksym;
+void __bpf_trap(void) __ksym;
+void bpf_task_release(struct task_struct *task) __ksym;
+void bpf_rcu_read_lock(void) __ksym;
+void bpf_rcu_read_unlock(void) __ksym;
+
+#define OPS \
+ __imm_addr(data), __imm_addr(array), __imm_addr(progs), \
+ __imm(bpf_loop), __imm(bpf_for_each_map_elem), \
+ __imm(bpf_get_prandom_u32), __imm(bpf_get_smp_processor_id), \
+ __imm(bpf_tail_call), __imm_addr(timers), \
+ __imm(bpf_map_lookup_elem), __imm(bpf_timer_set_callback), \
+ __imm(bpf_kptr_xchg), __imm(bpf_ringbuf_reserve), __imm_addr(ring), \
+ __imm_const(skb_data, offsetof(struct __sk_buff, data)), \
+ __imm_const(skb_data_end, offsetof(struct __sk_buff, data_end))
+
+#define ASM(...) asm volatile (__VA_ARGS__ :: OPS : __clobber_all)
+
+#define SPECULATES __arch_x86_64 __arch_arm64 __caps_unpriv(CAP_BPF | CAP_NET_ADMIN)
+
+/* r6 = data, r7 = a number */
+#define START "r6 = %[data] ll; r7 = *(u64 *)(r6 + 0);"
+/* only a speculative state goes to @label */
+#define NEVER(label) "r5 = r7; r5 &= 1; if r5 > 1 goto " label ";"
+
+static __naked __used int nop_cb(void)
+{
+ asm volatile ("r0 = 0; exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("barrier: before the inlined bpf_loop()")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r3 = 0")
+__xlated_unpriv("nospec")
+__xlated_unpriv("if r1 <= 0x800000 goto")
+#endif
+__naked void inlined_loop(void)
+{
+ ASM(START
+ "r1 = 2;"
+ "r2 = nop_cb ll;"
+ "r3 = r7;"
+ "r4 = 0;"
+ NEVER("1f")
+ "r3 = 0;"
+"1: call %[bpf_loop];"
+ "r0 = 0;"
+ "exit;");
+}
+
+/* A speculative state leaves a number in the slot of the caller */
+static __naked __used int store_cb(void)
+{
+ ASM(START
+ "*(u64 *)(r4 + 0) = r7;"
+ "if r7 == 3 goto 1f;"
+ "*(u64 *)(r4 + 8) = r7;"
+"1:" NEVER("2f")
+ "*(u64 *)(r4 + 0) = r6;"
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: at the exit of a callback")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("*(u64 *)(r4 +0) = r6")
+__xlated_unpriv("...")
+__xlated_unpriv("r0 = 0")
+__xlated_unpriv("nospec")
+__xlated_unpriv("exit")
+#endif
+__naked void exit_of_callback(void)
+{
+ ASM(START
+ "*(u64 *)(r10 - 16) = r6;"
+ "*(u64 *)(r10 - 8) = r7;"
+ "r1 = %[array] ll;"
+ "r2 = store_cb ll;"
+ "r3 = r10;"
+ "r3 += -16;"
+ "r4 = 0;"
+ "call %[bpf_for_each_map_elem];"
+ "r1 = *(u64 *)(r10 - 16);"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: on the way out of a loop that is out of budget")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r0 = 1")
+__xlated_unpriv("exit")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r0 = 0")
+__xlated_unpriv("exit")
+#endif
+__naked void unwind_exit(void)
+{
+ ASM(START
+"1: r1 = *(u64 *)(r6 + 8);"
+ "if r1 != 0 goto 1b;"
+ "r0 = 1;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: before may_goto")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r1 = *(u64 *)(r6 +8)")
+__xlated_unpriv("nospec")
+__xlated_unpriv("...")
+__xlated_unpriv("if r1 > 0x7 goto")
+#endif
+__naked void in_front_of_guard(void)
+{
+ ASM(START
+"1: r1 = r6;"
+ NEVER("2f")
+ "r1 = *(u64 *)(r6 + 8);"
+"2: if r1 > 7 goto 1b;"
+ "r0 = 0;"
+ "exit;");
+}
+
+/* The code after the loop needs the result of the jump: may_goto has to leave the program */
+SEC("tc")
+__description("barrier: a speculative state does not select the way of may_goto")
+__success SPECULATES __success_unpriv
+__naked void way_of_guard(void)
+{
+ ASM(START
+ "call bpf_rcu_read_lock;"
+ NEVER("1f")
+ "call bpf_rcu_read_unlock;"
+"1: r1 = *(u64 *)(r6 + 8);"
+ "if r1 > 7 goto 1b;"
+ "r1 <<= 3;"
+ "r6 += r1;"
+ "r0 = *(u64 *)(r6 + 0);"
+ "r0 = 0;"
+ "exit;");
+}
+
+static __naked __noinline __used int recurse(void)
+{
+ ASM(START
+ "r8 = r1;"
+ "if r8 == 0 goto 2f;"
+ "r9 = r7;"
+ "r9 &= 1;"
+ "if r9 == 0 goto 1f;"
+ "call bpf_rcu_read_lock;"
+"1: r1 = r8;"
+ "r1 += -1;"
+ "call recurse;"
+ "if r9 == 0 goto 2f;"
+ "call bpf_rcu_read_unlock;"
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: a speculative state holds more in a recursion")
+__success SPECULATES __success_unpriv
+__naked void recursion(void)
+{
+ ASM(START
+ "call bpf_rcu_read_lock;"
+ NEVER("1f")
+ "call bpf_rcu_read_unlock;"
+"1: r1 = 1;"
+ "call recurse;"
+ "r0 = 0;"
+ "exit;");
+}
+
+/* It releases what its caller holds */
+static __naked __noinline __used int recurse_in_frames(void)
+{
+ asm volatile (
+ "r6 = r1;"
+ "call bpf_rcu_read_unlock;"
+ "call bpf_rcu_read_lock;"
+ "if r6 == 0 goto 1f;"
+ "r1 = r6;"
+ "r1 += -1;"
+ "call recurse_in_frames;"
+"1: r0 = 0;"
+ "exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("barrier: at the call of a recursion in the frames of the callers")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r1 += -1")
+__xlated_unpriv("nospec")
+__xlated_unpriv("call pc-")
+#endif
+__naked void recursion_in_frames(void)
+{
+ ASM("call bpf_rcu_read_lock;"
+ "r1 = 1;"
+ "call recurse_in_frames;"
+ "call bpf_rcu_read_unlock;"
+ "r0 = 0;"
+ "exit;");
+}
+
+#define CALLS_TWICE(name, next) \
+ static __noinline long name(long n) \
+ { \
+ long a = next(n); \
+ \
+ barrier_var(n); \
+ return a + next(n); \
+ }
+#define FOUR_LEVELS(name, next) \
+ CALLS_TWICE(name##_a, next) \
+ CALLS_TWICE(name##_b, name##_a) \
+ CALLS_TWICE(name##_c, name##_b) \
+ CALLS_TWICE(name, name##_c)
+
+static __noinline long level_0(long n)
+{
+ return n + data[1];
+}
+
+FOUR_LEVELS(level_4, level_0)
+FOUR_LEVELS(level_8, level_4)
+FOUR_LEVELS(level_12, level_8)
+FOUR_LEVELS(level_16, level_12)
+FOUR_LEVELS(level_20, level_16)
+FOUR_LEVELS(level_24, level_20)
+
+SEC("tc")
+__description("a speculative call has an instance like any other")
+__success SPECULATES __success_unpriv
+int calls_in_a_chain(void *ctx)
+{
+ long i = data[0] & 7;
+
+ barrier_var(i);
+ return data[i] + level_24(data[2]);
+}
+
+static __naked __noinline __used int unlock_never(void)
+{
+ ASM(START
+ NEVER("1f")
+ "goto 2f;"
+"1: call bpf_rcu_read_unlock;"
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: after the call, the function released what its caller holds")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("call pc+")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r6 = 0")
+#endif
+__naked void behind_call(void)
+{
+ ASM("call bpf_rcu_read_lock;"
+ "call unlock_never;"
+ "r6 = 0;"
+ "call bpf_rcu_read_unlock;"
+ "r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: a speculative state goes into a loop that never exits")
+__success SPECULATES __success_unpriv
+__naked void never_exits(void)
+{
+ ASM(START
+ "r0 = 0;"
+ NEVER("1f")
+ "exit;"
+"1: r0 += 1;"
+ "goto 1b;");
+}
+
+/* A speculative state has a number in r1. @insns write the stack, and no barrier follows. */
+#define WRITE_THEN_LOAD(insns) \
+ ASM(START \
+ "*(u64 *)(r10 - 8) = r7;" \
+ "r1 = r7;" \
+ NEVER("1f") \
+ "r1 = r6;" \
+"1:" insns \
+ "r0 = *(u8 *)(r1 + 0);" \
+ "exit;")
+
+SEC("tc")
+__description("barrier: an atomic does not end a speculative path")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("lock *(u64 *)(r10 -8) += r7")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r0 = *(u8 *)(r1 +0)")
+#endif
+__naked void behind_atomic(void)
+{
+ WRITE_THEN_LOAD("lock *(u64 *)(r10 - 8) += r7;");
+}
+
+SEC("tc")
+__description("barrier: a spill that is removed does not end a speculative path")
+__success __arch_x86_64 __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r0 = *(u32 *)(r0 +0)")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r0 = *(u8 *)(r1 +0)")
+#endif
+__naked void behind_fastcall(void)
+{
+ WRITE_THEN_LOAD("*(u64 *)(r10 - 16) = r1;"
+ "call %[bpf_get_smp_processor_id];"
+ "r1 = *(u64 *)(r10 - 16);");
+}
+
+static __naked __noinline __used int throw_if_zero(void)
+{
+ ASM("if r1 != 0 goto 1f;"
+ "call bpf_throw;"
+"1: r0 = 0;"
+ "exit;");
+}
+
+/* The code after the first call is dead. The load must not replace it. */
+SEC("tc")
+__description("barrier: after a call that never returns")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("r1 = 0")
+__xlated_unpriv("call pc+")
+__xlated_unpriv("nospec")
+__xlated_unpriv("goto pc-2")
+#endif
+__naked void never_returns(void)
+{
+ ASM(START
+ "if r7 == 5 goto 1f;"
+ "r6 = r7;"
+ "r1 = 0;"
+ "call throw_if_zero;"
+ "r0 = *(u8 *)(r6 + 1);"
+ "exit;"
+"1: r0 = *(u8 *)(r6 + 0);"
+ "r1 = 1;"
+ "call throw_if_zero;"
+ "r0 = 0;"
+ "exit;");
+}
+
+/* The add is rewritten to r3 += 4. A speculative state has the pointer in r2. */
+SEC("socket")
+__description("barrier: a speculative state does not fit the mask")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("if r1 == 0x5 goto")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r3 += r2")
+#endif
+__naked void other_mask(void)
+{
+ ASM(START
+ "r1 = 1;"
+ "r2 = 4;"
+ "r3 = r6;"
+ "if r7 < 9 goto 1f;"
+ "r1 = 5;"
+ "r2 = r6;"
+ "r3 = r7;"
+ "r3 &= 15;"
+"1: if r1 == 5 goto 2f;"
+ "r3 += r2;"
+ "r0 = *(u8 *)(r3 + 0);"
+"2: r0 = 0;"
+ "exit;");
+}
+
+static __naked __used int bad_timer_cb(void)
+{
+ asm volatile ("r0 = *(u64 *)(r0 + 0); exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("barrier: a speculative state sets no timer")
+__success SPECULATES __success_unpriv
+__naked void timer_never(void)
+{
+ ASM(START
+ "*(u64 *)(r10 - 8) = r7;"
+ "r1 = %[timers] ll;"
+ "r2 = r10;"
+ "r2 += -8;"
+ "call %[bpf_map_lookup_elem];"
+ "if r0 == 0 goto 2f;"
+ NEVER("1f")
+ "goto 2f;"
+"1: r1 = r0;"
+ "r2 = bad_timer_cb ll;"
+ "call %[bpf_timer_set_callback];"
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("a speculative state does not ask for a stack argument")
+__success SPECULATES __success_unpriv
+__naked void stack_arg_never(void)
+{
+ ASM(START
+ NEVER("1f")
+ "goto 2f;"
+"1: .8byte 0xfff86b7b;" /* *(u64 *)(r11 - 8) = r6 */
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("a trap that no path gets to is no bug of the verifier")
+__success SPECULATES __success_unpriv
+__naked void trap_never(void)
+{
+ ASM(START
+ "r8 = 0;"
+ "r9 = 0;"
+ "if r7 == 0 goto 1f;"
+ "r8 = 1;"
+ "r9 = 5;"
+"1: if r8 != 0 goto 2f;"
+ "if r9 != 0 goto 3f;"
+"2: r0 = 0;"
+ "exit;"
+"3: call __bpf_trap;"
+ "exit;");
+}
+
+#define COMPARE_WITH_END(size) \
+ ASM(START \
+ "r2 = *(u32 *)(r1 + %[skb_data]);" \
+ "r3 = *(u32 *)(r1 + %[skb_data_end]);" \
+ "r4 = *(" #size " *)(r6 + 8);" \
+ "r2 += 14;" \
+ "r2 += r4;" \
+ "if r2 > r3 goto 1f;" \
+ "*(u64 *)(r6 + 16) = r4;" \
+"1: r0 = 0;" \
+ "exit;")
+
+SEC("tc")
+__description("no compare of a far pointer to the packet")
+__success __caps_unpriv(CAP_BPF | CAP_NET_ADMIN)
+__failure_unpriv __msg_unpriv("R3 pointer comparison prohibited")
+__naked void compare_far(void)
+{
+ COMPARE_WITH_END(u32);
+}
+
+SEC("tc")
+__description("a pointer to the packet plus a header and a 16-bit length")
+__success __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __success_unpriv
+__naked void compare_near(void)
+{
+ COMPARE_WITH_END(u16);
+}
+
+/* Each trip adds a small value */
+SEC("tc")
+__description("no compare of a far pointer to the packet in a loop")
+__success __caps_unpriv(CAP_BPF | CAP_NET_ADMIN)
+__failure_unpriv __msg_unpriv("R3 pointer comparison prohibited")
+__naked void compare_far_loop(void)
+{
+ ASM(START
+ "r2 = *(u32 *)(r1 + %[skb_data]);"
+ "r3 = *(u32 *)(r1 + %[skb_data_end]);"
+ "r0 = 0;"
+"1: r2 += 0x4000;"
+ "if r2 > r3 goto 2f;"
+ "r0 += 1;"
+ "if r0 < 8 goto 1b;"
+"2: exit;");
+}
+
+/* The trip count shows that r6 is value + 2^32. It is not the offset of the kptr. */
+SEC("tc")
+__description("a trip count does not make a far offset a constant")
+__failure __msg("R1 doesn't have constant offset")
+__naked void far_by_trip_count(void)
+{
+ ASM("r7 = 0;"
+ "*(u64 *)(r10 - 8) = r7;"
+ "r1 = %[timers] ll;"
+ "r2 = r10;"
+ "r2 += -8;"
+ "call %[bpf_map_lookup_elem];"
+ "if r0 == 0 goto 2f;"
+ "r6 = r0;"
+ "r6 += 16;"
+"1: r8 = r6;"
+ "r6 += 0x7fffffff;"
+ "r6 += 0x7fffffff;"
+ "r6 += 2;"
+ "if r7 == 1 goto 2f;"
+ "r1 = r6;"
+ "r2 = 0;"
+ "call %[bpf_kptr_xchg];"
+ "if r0 == 0 goto 3f;"
+ "r1 = r0;"
+ "call bpf_task_release;"
+"3: r6 = r8;"
+ "r6 += 8;"
+ "r7 += 1;"
+ "goto 1b;"
+"2: r0 = 0;"
+ "exit;");
+}
+
+/* The offset is 2^32. Its low 32 bits are 0. */
+SEC("raw_tp.w")
+__description("no access through a far pointer")
+__failure __msg("R6 invalid variable buffer offset")
+__naked void access_far(void)
+{
+ ASM("r6 = *(u64 *)(r1 + 0);"
+ "r6 += 0x7fffffff;"
+ "r6 += 0x7fffffff;"
+ "r6 += 2;"
+ "r0 = *(u8 *)(r6 + 0);"
+ "exit;");
+}
+
+#define MOVES_AWAY(step) \
+ ASM(START \
+ "r2 = r6;" \
+"1: r2 += " #step ";" \
+ "r0 = *(u64 *)(r6 + 8);" \
+ "if r0 != 0 goto 1b;" \
+ "exit;")
+
+SEC("tc")
+__description("a pointer that is not used moves up without a bound")
+__success
+__naked void moves_up(void)
+{
+ MOVES_AWAY(1);
+}
+
+SEC("tc")
+__description("a pointer that is not used moves down without a bound")
+__success
+__naked void moves_down(void)
+{
+ MOVES_AWAY(-1);
+}
+
+/* r7, r8 = 0, 0 or 1, 5. The join makes the equation r8 = 5 * r7. What follows is dead. */
+#define DEAD_BY_EQUATION \
+ "call %[bpf_get_prandom_u32];" \
+ "r7 = 0;" \
+ "r8 = 0;" \
+ "if w0 & 1 goto 1f;" \
+ "r7 = 1;" \
+ "r8 = 5;" \
+"1: if r7 != 0 goto 9f;" \
+ "if r8 == 0 goto 9f;"
+
+SEC("iter/bpf_map_elem")
+__description("fixups: a dead tail call is removed")
+__success
+__xlated("if r7 != 0x0 goto pc+0")
+__xlated("r0 = 0")
+__xlated("exit")
+__naked void tail_call_never(void)
+{
+ ASM("r6 = r1;"
+ DEAD_BY_EQUATION
+ "r1 = r6;"
+ "r2 = %[progs] ll;"
+ "r3 = 0;"
+ "call %[bpf_tail_call];"
+"9: r0 = 0;"
+ "exit;");
+}
+
+__noinline int global_func(int x)
+{
+ return x + 1;
+}
+
+static __naked __noinline __used int calls_global(void)
+{
+ asm volatile ("r1 = 1; call global_func; exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("fixups: a dead load does not break the fastcall pattern")
+__success __arch_x86_64
+__xlated("r1 = 7")
+__xlated("r0 = {{.*}}")
+__xlated("r0 = &(void __percpu *)(r0)")
+__naked void one_by_one(void)
+{
+ ASM("r9 = *(u32 *)(r1 + %[skb_data]);"
+ "r6 = *(u32 *)(r1 + %[skb_data_end]);"
+ "call calls_global;"
+ "r1 = 7;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "call %[bpf_get_smp_processor_id];"
+ "r1 = *(u64 *)(r10 - 8);"
+ "if r9 > r6 goto 9f;"
+ DEAD_BY_EQUATION
+ "r0 = *(u64 *)(r10 - 8);"
+"9: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("log: a path that is dropped is not in the log")
+__failure __log_level(4) __not_msg("off=-2") __msg("value_size=64 off=64")
+__naked void log_of_dropped_path(void)
+{
+ /* After 'r2 == r3' any number is 2 or more, but -32..31 stays as it is */
+ ASM(START
+ "r3 = *(u64 *)(r6 + 16);"
+ "if r3 < 2 goto 2f;"
+ "r2 = *(u64 *)(r6 + 8);"
+ "r2 &= 63;"
+ "r2 += -32;"
+ "r9 = 0;"
+ "if r7 < 5 goto 1f;"
+ "r2 = *(u64 *)(r6 + 8);"
+ "r9 = 64;"
+"1: if r2 != r3 goto 2f;"
+ "if r2 > 40 goto 2f;"
+ "r4 = r6;"
+ "r4 += r2;"
+ "r0 = *(u8 *)(r4 - 2);"
+ "r6 += r9;"
+ "r0 = *(u64 *)(r6 + 0);"
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("log: the report of a leak starts at the acquire")
+__failure __msg("Unreleased reference") __msg("Causal path:")
+__not_msg("(15) if r0 == 0x0") __msg("acquired: owned resource")
+__naked void log_of_leak(void)
+{
+ ASM("r1 = %[ring] ll;"
+ "r2 = 8;"
+ "r3 = 0;"
+ "call %[bpf_ringbuf_reserve];"
+ "if r0 == 0 goto 1f;"
+ "r0 = 0;"
+"1: exit;");
+}
+
+static __naked __noinline __used void reserve(void)
+{
+ ASM("r1 = %[ring] ll;"
+ "r2 = 8;"
+ "r3 = 0;"
+ "call %[bpf_ringbuf_reserve];"
+ "exit;");
+}
+
+SEC("tc")
+__description("log: one insn acquires twice, each report has its acquire and one id")
+__failure __msg("Unreleased reference id=4 ") __msg("acquired: owned resource (id=4)")
+__msg("branch: took the false branch")
+__msg("Unreleased reference id=12 ") __msg("acquired: owned resource (id=12)")
+__not_msg("branch: took the false branch")
+__naked void log_of_two_leaks(void)
+{
+ ASM("call reserve;"
+ "if r0 == 0 goto 1f;"
+ "call reserve;"
+"1: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("log: no report for a speculative state")
+__success SPECULATES __success_unpriv __log_level(2)
+#ifdef SPEC_V1
+__msg_unpriv("R7 invalid mem access 'scalar'")
+#endif
+__not_msg_unpriv("Verification failed")
+__naked void log_of_speculative_state(void)
+{
+ ASM(START
+ NEVER("1f")
+ "r0 = 0;"
+ "exit;"
+"1: r0 = *(u64 *)(r7 + 0);"
+ "exit;");
+}
+
+static const volatile char bytes[4] = { 1, 2, 3, 4 };
+
+/* const_fold.c does not follow 'r3 *= 2'. If it has r2 = 3, then r2 == 4 is a verifier bug. */
+SEC("socket")
+__description("a far pointer to .rodata that comes back loads no constant")
+__success __retval(0) __xlated("r0 = r2")
+__naked void far_rodata(void)
+{
+ asm volatile (
+ "r1 = %[bytes] ll;"
+ "r1 += 0x20000001;"
+ "r1 += -0x1fffffff;"
+ "r2 = *(u8 *)(r1 + 0);"
+ "r3 = 2;"
+ "r3 *= 2;"
+ "r0 = 0;"
+ "if r2 != r3 goto 1f;"
+ "r0 = r2;"
+"1:"
+ "exit;"
+ :: __imm_addr(bytes) : __clobber_all);
+}
+
+/* The patches turn 8200 divisions into 32800 insns, too far for a 16-bit offset. */
+#define DIVS "r2 = r7; r3 = r7; .rept 8200; r2 /= r3; .endr;"
+
+SEC("tc")
+__description("barrier: where a far jump lands")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("8: gotol pc+32803")
+__xlated_unpriv("...")
+__xlated_unpriv("32812: nospec")
+#endif
+__naked void far_target(void)
+{
+ ASM(START
+ "r8 = r7;"
+ NEVER("1f")
+ "r8 = r6;"
+ DIVS
+"1: r0 = *(u64 *)(r8 + 8);"
+ "r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: after a far jump")
+__success SPECULATES __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("8: gotol pc+32804")
+__xlated_unpriv("9: nospec")
+#endif
+__naked void far_fall(void)
+{
+ ASM(START
+ "r8 = r6;"
+ "if r7 == 5 goto 1f;"
+ "r8 = r7;"
+"1: if r7 != 5 goto 2f;"
+ "r0 = *(u64 *)(r8 + 8);"
+ DIVS
+"2: r0 = 0;"
+ "exit;");
+}
+
+SEC("tc")
+__description("barrier: on the far exit that is out of budget")
+__success __arch_x86_64 __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("16: gotol pc+32810")
+__xlated_unpriv("...")
+__xlated_unpriv("32827: nospec")
+#endif
+__naked void far_unwind_exit(void)
+{
+ ASM(START
+"1: r1 = *(u64 *)(r6 + 8);"
+ "if r1 != 0 goto 1b;"
+ DIVS
+ "r0 = 1;"
+ "exit;");
+}
+
+/* the kfuncs have to be in BTF */
+void __kfunc_btf_root(void)
+{
+ bpf_rcu_read_lock();
+ bpf_rcu_read_unlock();
+ bpf_task_release(NULL);
+ __bpf_trap();
+ bpf_throw(0);
+}
+
+char _license[] SEC("license") = "GPL";
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 47/49] selftests/bpf: Test passes, instances, precision and the failing path
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (45 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 46/49] selftests/bpf: Test barriers and far pointers Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 8:30 ` sashiko-bot
2026-11-20 7:42 ` [PATCH bpf-next 48/49] selftests/bpf: Test equations Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 49/49] bpf, docs: Rewrite verifier.rst Alexei Starovoitov
48 siblings, 1 reply; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
verifier_passes.c has one group for each mechanism, for example deps,
live types, unreadable, peel, trip count, ids, instances, held, fixups,
pessimized, strings, failing_path, budget, log.
verifier_partition.c tests the order of the partitions, the undo, the
suspect loops, calls in frames, the load of a part of a spilled pointer
and arrays of pointers on the stack. Many valid programs have an unsafe
twin that has to be rejected.
liveness_many_calls has 12000 calls.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
.../selftests/bpf/prog_tests/verifier.c | 4 +
.../bpf/progs/verifier_callx_rodata.c | 25 +
.../bpf/progs/verifier_liveness_exp.c | 48 +
.../selftests/bpf/progs/verifier_partition.c | 1503 +++++++
.../selftests/bpf/progs/verifier_passes.c | 3636 +++++++++++++++++
.../bpf/progs/verifier_subprog_insn_stats.c | 14 +
.../selftests/bpf/progs/verifier_var_off.c | 46 +
7 files changed, 5276 insertions(+)
create mode 100644 tools/testing/selftests/bpf/progs/verifier_partition.c
create mode 100644 tools/testing/selftests/bpf/progs/verifier_passes.c
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index ab15e69a622..da112954a74 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -112,6 +112,7 @@
#include "verifier_netfilter_retcode.skel.h"
#include "verifier_bpf_fastcall.skel.h"
#include "verifier_or_jmp32_k.skel.h"
+#include "verifier_passes.skel.h"
#include "verifier_no_limits.skel.h"
#include "verifier_patch_list.skel.h"
#include "verifier_percpu_addr.skel.h"
@@ -120,6 +121,7 @@
#include "verifier_private_stack.skel.h"
#include "verifier_ptr_off.skel.h"
#include "verifier_ptr_to_buf.skel.h"
+#include "verifier_partition.skel.h"
#include "verifier_raw_stack.skel.h"
#include "verifier_raw_tp_writable.skel.h"
#include "verifier_reg_equal.skel.h"
@@ -324,6 +326,7 @@ void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode);
void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); }
void test_verifier_no_limits(void) { RUN_JITED(verifier_no_limits); }
void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); }
+void test_verifier_passes(void) { RUN(verifier_passes); }
/*
* Run serially with bpf_jit_harden at 0: blinding changes the xlated insns that
* the tests compare, and other tests set the sysctl.
@@ -344,6 +347,7 @@ void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup)
void test_verifier_private_stack(void) { RUN(verifier_private_stack); }
void test_verifier_ptr_off(void) { RUN(verifier_ptr_off); }
void test_verifier_ptr_to_buf(void) { RUN(verifier_ptr_to_buf); }
+void test_verifier_partition(void) { RUN(verifier_partition); }
void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); }
void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); }
void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); }
diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c
index 3545c2a2e8f..a7cceb1ff22 100644
--- a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c
+++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c
@@ -242,6 +242,31 @@ __naked void callx_rodata_callback_one_of_two(void)
: __clobber_all);
}
+/* each call is checked for each function in r2, and the states meet at the next insn */
+SEC("socket")
+__success __retval(0)
+__naked void callx_rodata_one_of_two_at_many_calls(void)
+{
+ asm volatile (
+ FUNC_TABLE2(cbs, loop_cb_stop, loop_cb_go_on)
+ "call %[bpf_get_prandom_u32];"
+ "r0 &= 1;"
+ "r0 <<= 3;"
+ "r6 = cbs_%= ll;"
+ "r6 += r0;"
+ "r6 = *(u64 *)(r6 + 0);"
+ ".rept 1024;"
+ "r2 = r6;"
+ "call %[bpf_get_prandom_u32];"
+ ".endr;"
+ "callx r6;"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
__naked __noinline __used
static unsigned long not_three(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
index bb343bad924..a6cacc04615 100644
--- a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
+++ b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
@@ -142,3 +142,51 @@ __naked int liveness_exponential_complexity(void)
"exit;"
::: __clobber_all);
}
+
+/* The number of calls has no limit of its own */
+SEC("?raw_tp")
+__success
+__naked int liveness_many_calls(void)
+{
+ asm volatile (
+ ".rept 12000;"
+ C(exp_sub7, 8)
+ ".endr;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/*
+ * The pointer to the stack moves to the next slot in each trip. So the loop takes a round of the
+ * analysis for each slot. The insns before the loop take one.
+ */
+SEC("?raw_tp")
+__success
+__naked int liveness_many_rounds(void)
+{
+ asm volatile (
+ "r6 = 0;"
+ "r2 = r10;"
+ ".rept 64;"
+ "r2 += -8;"
+ "*(u64 *)(r2 + 0) = r6;"
+ ".endr;"
+ ".rept 100000;"
+ "r0 = 0;"
+ ".endr;"
+ "1:"
+ "r2 = r10;"
+ "r2 += -512;"
+ ".rept 63;"
+ "r1 = *(u64 *)(r2 + 8);"
+ "*(u64 *)(r2 + 0) = r1;"
+ "r2 += 8;"
+ ".endr;"
+ "*(u64 *)(r10 - 8) = r10;"
+ "r6 += 1;"
+ "if r6 < 3 goto 1b;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
diff --git a/tools/testing/selftests/bpf/progs/verifier_partition.c b/tools/testing/selftests/bpf/progs/verifier_partition.c
new file mode 100644
index 00000000000..cf0b159a115
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_partition.c
@@ -0,0 +1,1503 @@
+// SPDX-License-Identifier: GPL-2.0
+/* How the verifier gets the precision that a program needs: partition, undo, suspect loops. */
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+#include "bpf_experimental.h"
+
+struct bytes2k {
+ char b[2048];
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct bytes2k);
+} map2k SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_RINGBUF);
+ __uint(max_entries, 4096);
+} ringbuf SEC(".maps");
+
+char small[14] SEC(".data.small");
+
+#define ASM(...) \
+ asm volatile (__VA_ARGS__ :: \
+ __imm_addr(map2k), __imm(bpf_get_prandom_u32), \
+ __imm(bpf_map_lookup_elem), __imm(bpf_loop), __imm_addr(ringbuf), \
+ __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), \
+ __imm(bpf_get_current_comm), __imm_addr(small) : __clobber_all)
+
+/* r9 = value of map2k */
+#define VALUE_IN_R9 \
+ "r1 = 0;" \
+ "*(u64 *)(r10 - 8) = r1;" \
+ "r1 = %[map2k] ll;" \
+ "r2 = r10;" \
+ "r2 += -8;" \
+ "call %[bpf_map_lookup_elem];" \
+ "if r0 == 0 goto 9f;" \
+ "r9 = r0;"
+
+#define EXIT_AT_9 "9: r0 = 0; exit;"
+#define LOAD(reg) "r1 = r9; r1 += " #reg "; r0 = *(u8 *)(r1 + 0);"
+#define RANDOM_BIT "r0 = *(u8 *)(r9 + 0); r0 &= 1;"
+#define MAY_GOTO(off) ".byte 0xe5; .byte 0; .short " #off "; .long 0;"
+#define INSNS(digits) __log_level(4) __msg("processed {{[0-9]{1," #digits "} }}insns")
+
+/* r6 = 0 + 1 + ... + (n - 1) */
+#define SUM_IN_R6(n) \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1: r6 += r7;" \
+ "r7 += 1;" \
+ "if r7 < " #n " goto 1b;"
+
+/* The last loop is infinite, and it decides only if the load runs */
+#define TWO_LOOPS(n) \
+ VALUE_IN_R9 \
+ SUM_IN_R6(n) \
+ "r8 = 0;" \
+"2:" RANDOM_BIT \
+ "if r0 == 0 goto 3f;" \
+ "r8 += 1;" \
+ "goto 2b;" \
+"3: if r8 <= 3 goto 9f;" \
+ LOAD(r6) \
+ EXIT_AT_9
+
+SEC("tc")
+__description("suspect: the loop that made data_deps imprecise, not the last one")
+__success INSNS(4)
+__naked void suspect_first_loop(void)
+{
+ ASM(TWO_LOOPS(8));
+}
+
+SEC("tc")
+__description("suspect: the sum is too large")
+__failure __msg("invalid access to map value, value_size=2048 off=2080 size=1")
+__naked void suspect_first_loop_bad(void)
+{
+ ASM(TWO_LOOPS(65));
+}
+
+/* for (r7 = 0; r7 < n; r7++) { if (random) r6 += 3; value[r6]; } */
+#define STEP_OF_3(n) \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1:" RANDOM_BIT \
+ "if r0 == 0 goto 2f;" \
+ "r6 += 3;" \
+"2:" LOAD(r6) \
+ "r7 += 1;" \
+ "if r7 < " #n " goto 1b;"
+
+SEC("tc")
+__description("peel: states that start a trip are joined")
+__success INSNS(5)
+__naked void join_first(void)
+{
+ ASM(VALUE_IN_R9 STEP_OF_3(600) EXIT_AT_9);
+}
+
+SEC("tc")
+__description("peel: states that start a trip are joined, one trip too many")
+__failure __msg("invalid access to map value, value_size=2048 off=2049 size=1")
+__naked void join_first_bad(void)
+{
+ ASM(VALUE_IN_R9 STEP_OF_3(683) EXIT_AT_9);
+}
+
+/* r6 + r7 is 100 * r8 */
+SEC("tc")
+__description("peeled_apart: the join of a trip is imprecise")
+__success INSNS(5)
+__naked void peeled_apart(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+ "r8 = 0;"
+"1:" RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ "r6 += 100;"
+ "goto 3f;"
+"2: r7 += 100;"
+"3: r8 += 1;"
+ "if r8 < 20 goto 1b;"
+ "r6 += r7;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("suspect: four loops are peeled in one pass")
+__success INSNS(5)
+__naked void loops_after_bulk(void)
+{
+ ASM(VALUE_IN_R9
+ ".rept 1500;"
+ RANDOM_BIT
+ "if r0 == 0 goto +1;"
+ "r8 = r0;"
+ ".endr;"
+ ".rept 4;"
+ STEP_OF_3(200)
+ ".endr;"
+ EXIT_AT_9);
+}
+
+/* The last 64 loops are suspects. The other loops have the same suspect bits. */
+SEC("tc")
+__description("suspect: many loops are widened and peeled")
+__success
+__naked void many_loops(void)
+{
+ ASM(".rept 20000;"
+ "r7 = 0;"
+"1: r7 += 1;"
+ "if r7 < 3 goto 1b;"
+ ".endr;"
+ VALUE_IN_R9
+ "r6 = 0;"
+ ".rept 64;"
+ "r7 = 0;"
+ "r8 = 1;"
+"1: r8 *= 2;"
+ "r7 += 1;"
+ "if r7 < 3 goto 1b;"
+ "r6 += r8;"
+ ".endr;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+/* Only the inner loop needs peeling. All trips of both are too many. */
+SEC("tc")
+__description("peel: the outer loop does not use up the budget of the inner loop")
+__success INSNS(6)
+__naked void inner_loop_only(void)
+{
+ ASM(VALUE_IN_R9
+ "r8 = 0;"
+"4: r6 = r8;"
+ "r7 = 0;"
+"1:" RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ "r6 += 3;"
+"2:" LOAD(r6)
+ "r7 += 1;"
+ "if r7 < 400 goto 1b;"
+ "r8 += 1;"
+ "if r8 < 400 goto 4b;"
+ EXIT_AT_9);
+}
+
+/* Only the outer loop needs peeling. The inner loop is infinite. */
+SEC("tc")
+__description("peel: the inner loop does not use up the budget of the outer loop")
+__success INSNS(6)
+__naked void outer_loop_only(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+"1:" RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ "r6 += 3;"
+"2: r8 = 0;"
+"3: r8 += 1;"
+ RANDOM_BIT
+ "if r0 != 0 goto 3b;"
+ "r2 = r8;"
+ "r2 &= 7;"
+ "r2 += r6;"
+ LOAD(r2)
+ "r7 += 1;"
+ "if r7 < 100 goto 1b;"
+ EXIT_AT_9);
+}
+
+/*
+ * A stack with depth r6 and a sum that grows. The jumps test a product, which shows nothing
+ * about r6 if it is a range.
+ */
+#define DEPTH(max) \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1:" MAY_GOTO(15) \
+ "r2 = r6;" \
+ "r2 *= r6;" \
+ RANDOM_BIT \
+ "if r0 == 0 goto 2f;" \
+ "if r2 >= " #max " goto 9f;" \
+ "r6 += 1;" \
+ "goto 1b;" \
+"2: if r2 == 0 goto 9f;" \
+ "r6 += -1;" \
+ "r1 = r9; r1 += r6; r0 = *(u8 *)(r1 + 2032);" \
+ "r7 += r0;" \
+ "goto 1b;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("loop_disjoint_apart: data_deps have a few values in the leftover")
+__success INSNS(5)
+__naked void loop_disjoint_apart(void)
+{
+ ASM(DEPTH(256));
+}
+
+SEC("tc")
+__description("loop_disjoint_apart: one value too many")
+__failure __msg("invalid access to map value, value_size=2048 off=2048")
+__naked void loop_disjoint_apart_bad(void)
+{
+ ASM(DEPTH(257));
+}
+
+/*
+ * The low bits of r6 are a ring index. r6 starts with 0 or with 1 << 24. It moves with the trip
+ * index only in the first trips.
+ */
+#define RING_WITH_TWO_STARTS(max) \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = 0;" \
+ RANDOM_BIT \
+ "if r0 == 0 goto 1f;" \
+ "r6 = 0x1000000;" \
+"1: r6 += 1;" \
+ "r2 = r6; r2 &= 15; r2 *= r2;" \
+ "if r2 <= " #max " goto 2f;" \
+ "r6 += -8;" \
+"2: r2 = r6; r2 &= 15;" \
+ "r1 = r9; r1 += r2; r0 = *(u8 *)(r1 + 2040);" \
+ "r7 += 1;" \
+ "if r7 < 1000000 goto 1b;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("loop_disjoint_apart: an xreg is linear only in the first trips")
+__success INSNS(4)
+__naked void linear_at_first(void)
+{
+ ASM(RING_WITH_TWO_STARTS(49));
+}
+
+SEC("tc")
+__description("loop_disjoint_apart: linear at first, one value too many")
+__failure __msg("invalid access to map value, value_size=2048 off=2048")
+__naked void linear_at_first_bad(void)
+{
+ ASM(RING_WITH_TWO_STARTS(64));
+}
+
+/* The address makes the load from the stack unproven. The stores of the loop do not matter. */
+SEC("tc")
+__description("deps: an access that is unproven because of its address")
+__success INSNS(5)
+__naked void address_is_unproven(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+"1:" RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ "r2 = r6;"
+ "r2 *= r6;"
+ "if r2 == 64 goto 3f;"
+ "r1 = r10;"
+ "r1 += -8;"
+ "r1 += r6;"
+ "*(u8 *)(r1 + 0) = r7;"
+ "r6 += 1;"
+"2: r7 += 1;"
+ "if r7 < 200 goto 1b;"
+"3: r2 = r6;"
+ "r2 *= r6;"
+ "if r2 == 0 goto 9f;"
+ "r1 = r10;"
+ "r1 += r6;"
+ "r2 = *(u8 *)(r1 - 9);"
+ LOAD(r2)
+ EXIT_AT_9);
+}
+
+/* r8 = -4..3 or any number, then 3..10 */
+#define RANGE_AROUND_ZERO \
+ "r8 = *(u64 *)(r9 + 8);" \
+ RANDOM_BIT \
+ "if r0 == 0 goto 5f;" \
+ "r8 &= 7;" \
+ "r8 += -4;" \
+"5: if r8 < 3 goto 9f;" \
+ "if r8 s< 0 goto 9f;" \
+ "if r8 s> 10 goto 9f;" \
+ "r9 += r8;" \
+ "r9 += -3;"
+
+SEC("tc")
+__description("failing_path: a jump cuts a range in two, the loop needs many trips")
+__success
+__naked void cut_range_on_path(void)
+{
+ ASM(VALUE_IN_R9
+ RANGE_AROUND_ZERO
+ "r6 = 0;"
+ "r7 = 0;"
+"1: r2 = r7;"
+ "r2 &= 1;"
+ "r6 += r2;"
+ LOAD(r6)
+ "r7 += 1;"
+ "if r7 < 4000 goto 1b;"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("a jump cuts a range in two in a pass")
+__success
+__naked void cut_range_in_pass(void)
+{
+ ASM(VALUE_IN_R9
+ "r8 = *(u64 *)(r9 + 8);"
+ "r8 &= 7;"
+ "r8 += -4;"
+ "if r8 < 3 goto 9f;"
+ "if r8 s< 0 goto 9f;"
+ "r9 += r8;"
+ "r0 = *(u8 *)(r9 - 3);"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("a jump cuts a range in two, the negative part gets through")
+__failure __msg("R9 min value is negative")
+__naked void cut_range_bad(void)
+{
+ ASM(VALUE_IN_R9
+ "r8 = *(u64 *)(r9 + 8);"
+ "r8 &= 7;"
+ "r8 += -4;"
+ "if r8 < 3 goto 9f;"
+ "r9 += r8;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+void __kfunc_btf_root(void)
+{
+ bpf_rcu_read_lock();
+ bpf_rcu_read_unlock();
+ bpf_preempt_disable();
+ bpf_preempt_enable();
+}
+
+#define LOCK_IF_COND(lock, unlock) \
+ RANDOM_BIT \
+ "r6 = r0;" \
+ "r7 = 0;" \
+ "if r6 == 0 goto +1;" \
+ "r7 = 1;" \
+ "if r6 == 0 goto +1;" \
+ "call " #lock ";" \
+ "if r7 == 0 goto +1;" \
+ "call " #unlock ";"
+
+SEC("tc")
+__description("a pass with an error ends at its budget")
+__success INSNS(6)
+__naked void pass_with_error_ends(void)
+{
+ ASM(VALUE_IN_R9
+ ".rept 60;"
+ LOCK_IF_COND(bpf_rcu_read_lock, bpf_rcu_read_unlock)
+ LOCK_IF_COND(bpf_preempt_disable, bpf_preempt_enable)
+ ".endr;"
+ EXIT_AT_9);
+}
+
+/* The first load is unproven in trip 0. No loop is a suspect, the second loop is not peeled. */
+SEC("tc")
+__description("suspect: the first error has none")
+__failure __msg("invalid access to map value, value_size=2048 off=65535 size=1") INSNS(4)
+__naked void first_error_has_no_suspect(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = *(u16 *)(r9 + 0);"
+ "r7 = r6;"
+ "r7 *= r6;"
+ "if r7 > 100 goto 9f;"
+ LOAD(r6)
+ "r6 = 0;"
+"1: r6 += 1;"
+ RANDOM_BIT
+ "if r0 != 0 goto 1b;"
+ "r6 >>= 32;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("log: the path-sensitive run is repeated")
+__failure __log_level(1) __msg("r7 *= r6") __msg("r1 += r6") __msg("unbounded min value")
+__naked void log_of_failing_path(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = *(u64 *)(r9 + 0);"
+ "r7 = r6;"
+ "r7 *= r6;"
+ "r8 = 0;"
+ "if r7 != 2 goto 1f;"
+ "r8 = 1;"
+"1: if r8 == 0 goto 9f;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void large_frame(void)
+{
+ asm volatile ("*(u64 *)(r10 - 1600) = 0; r0 = 0; exit;" ::: __clobber_all);
+}
+
+/* The store is at fp - 16. The join makes the frame too large for the call. No insn check fails. */
+SEC("tc")
+__description("partition: check_after_pass() finds the error")
+__load_if_large_stack()
+__success __log_level(2) __msg("combined stack size") __msg("pass 1")
+__naked void error_after_pass(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r7 = r0;"
+ "r8 = 1;"
+ "if w0 & 1 goto 1f;"
+ "r7 = 0;"
+ "r8 = 0;"
+"1: r7 &= 960;"
+ "if r8 != 0 goto 2f;"
+ "r7 = -r7;"
+ "r1 = r10;"
+ "r1 += r7;"
+ "*(u64 *)(r1 - 16) = 0;"
+"2: call large_frame;"
+ "r6 = 3;"
+"3: r6 += -1;"
+ "if r6 != 0 goto 3b;"
+ EXIT_AT_9);
+}
+
+/* All trips of the inner loop fit into the budget and do not help. r6 is 0, 1 or 2. */
+SEC("tc")
+__description("loop_disjoint_apart: replaces peeling")
+__success INSNS(5)
+__naked void loop_disjoint_apart_after_peel(void)
+{
+ ASM(VALUE_IN_R9
+ "r8 = 0;"
+"4: r6 = 0;"
+ "r7 = 0;"
+"1: r3 = r6;"
+ "r3 *= r6;"
+ RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ "if r3 >= 4 goto 3f;"
+ "r6 += 1;"
+"2: r7 += 1;"
+ "if r7 < 200 goto 1b;"
+"3: r1 = r9; r1 += r6; r0 = *(u8 *)(r1 + 2045);"
+ "r8 += 1;"
+ "if r8 < 50 goto 4b;"
+ EXIT_AT_9);
+}
+
+/* Peeling proves the store in the loop. The load after it needs LOOP_DISJOINT_APART. */
+SEC("tc")
+__description("loop_disjoint_apart: replaces peeling that helped")
+__success INSNS(5)
+__naked void loop_disjoint_apart_after_move(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+ "r8 = 0;"
+"1: r3 = r6;"
+ "r3 *= r6;"
+ RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ "if r3 >= 64 goto 3f;"
+ "r1 = r9; r1 += r6; *(u8 *)(r1 + 0) = r7;"
+ "r6 += 1;"
+"2: r8 += r0;"
+ "r7 += 1;"
+ "if r7 < 250 goto 1b;"
+"3: r1 = r9; r1 += r6; r0 = *(u8 *)(r1 + 2039);"
+ EXIT_AT_9);
+}
+
+/* The counter r6 starts with 0 or with 1 << 20. r3 is 2 or 3, r4 is 6. r8 is a ring index. */
+#define COUNTER_WITH_TWO_STARTS(off) \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = 0;" \
+ "r8 = 0;" \
+ RANDOM_BIT \
+ "if r0 == 0 goto 1f;" \
+ "r6 = 0x100000;" \
+"1: r3 = r6; r3 >>= 20; r3 += 2;" \
+ "r4 = 6; r4 /= r3; r4 *= r3;" \
+ "r1 = r9; r1 += r4; r0 = *(u8 *)(r1 + " #off ");" \
+ "r8 += 1;" \
+ "r2 = r8; r2 *= r8;" \
+ "if r2 <= 49 goto 2f;" \
+ "r8 = 0;" \
+"2: r1 = r9; r1 += r8; r0 = *(u8 *)(r1 + 2040);" \
+ "r6 += 1;" \
+ "r7 += 1;" \
+ "if r7 < 100000 goto 1b;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("loop_disjoint_apart: a counter keeps two entries apart")
+__success INSNS(5)
+__naked void counter_of_two_entries(void)
+{
+ ASM(COUNTER_WITH_TWO_STARTS(2041));
+}
+
+SEC("tc")
+__description("loop_disjoint_apart: a counter of two entries, one byte too far")
+__failure __msg("invalid access to map value, value_size=2048 off=2048 size=1")
+__naked void counter_of_two_entries_bad(void)
+{
+ ASM(COUNTER_WITH_TWO_STARTS(2042));
+}
+
+/* Both loops need peeling. The inner leftover is unproven in trip 0 of the outer loop. */
+SEC("tc")
+__description("partition: the leftover of an inner loop does not end it")
+__success INSNS(6)
+__naked void leftover_of_inner_loop(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r8 = 0;"
+"4: r7 = 0;"
+"1:" LOAD(r6)
+ "r6 += 1;"
+ "if r0 == 0 goto 2f;"
+ "r7 += 1;"
+ "r2 = r7;"
+ "r2 *= r7;"
+ "if r2 < 4096 goto 1b;"
+"2: r8 += 1;"
+ "r2 = r8;"
+ "r2 *= r8;"
+ "if r2 < 9 goto 4b;"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void load_at_r2(void)
+{
+ asm volatile ("r1 += r2; r0 = *(u8 *)(r1 + 0); exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("peel: a call in the loop is unproven too")
+__success INSNS(5)
+__naked void call_in_peeled_trips(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+"1:" RANDOM_BIT
+ "if r0 == 0 goto 2f;"
+ LOAD(r6)
+ "r6 += 1;"
+"2: r1 = r9;"
+ "r2 = r7;"
+ "call load_at_r2;"
+ "r7 += 1;"
+ "if r7 < 700 goto 1b;"
+ EXIT_AT_9);
+}
+
+#define DEEP_ERROR \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = *(u16 *)(r9 + 0);" \
+ "if r7 > 400 goto 9f;" \
+"1: if r7 == 0 goto 3f;" \
+ "r7 += -1;" \
+ RANDOM_BIT \
+ "if r0 == 0 goto 1b;" \
+ "r6 += 1;" \
+ "goto 1b;" \
+"3: r1 = r9; r1 += r6; r0 = *(u8 *)(r1 + 1748);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("failing_path: the error is 300 trips deep")
+__failure __msg("invalid access to map value") INSNS(5)
+__naked void deep_failing_path(void)
+{
+ ASM(DEEP_ERROR);
+}
+
+__noinline __used int global_no_error(int x)
+{
+ return x;
+}
+
+SEC("tc")
+__description("failing_path: a global function does not hide that a run is over its budget")
+__failure __msg("invalid access to map value") INSNS(5)
+__naked void deep_failing_path_global(void)
+{
+ ASM("r1 = 0; call global_no_error;" DEEP_ERROR);
+}
+
+/* Trip 1 subsumes trip 2. With the lock may_goto leaves the loop before the jump. */
+#define HALVE_UNTIL(bound) \
+ VALUE_IN_R9 \
+ "call bpf_rcu_read_lock;" \
+ "r6 = *(u8 *)(r9 + 0);" \
+ "r8 = 1;" \
+"1: if r6 < " #bound " goto 2f;" \
+ "r6 >>= 1;" \
+ "if r8 == 0 goto 1b;" \
+ "r8 += -1;" \
+ "goto 1b;" \
+"2: r6 <<= 8;" \
+ "r6 += r8;" \
+ LOAD(r6) \
+ "call bpf_rcu_read_unlock;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("peel_more: the next partition does not prune the peeled trips")
+__success INSNS(4)
+__naked void peel_more_after_prune(void)
+{
+ ASM(HALVE_UNTIL(8));
+}
+
+SEC("tc")
+__description("peel_more: the loop is left too early")
+__failure __msg("invalid access to map value, value_size=2048")
+__naked void peel_more_after_prune_bad(void)
+{
+ ASM(HALVE_UNTIL(9));
+}
+
+__noinline int read_u64(__u64 *p)
+{
+ return p ? 1 : 0;
+}
+
+static __naked __noinline __used void same(void)
+{
+ asm volatile ("r0 = r1; exit;" ::: __clobber_all);
+}
+
+static __naked __noinline __used void same_too(void)
+{
+ asm volatile ("r0 = r1; exit;" ::: __clobber_all);
+}
+
+/* The second call has the old exit state at once and the new one later */
+static __naked __noinline __used void returns_late(void)
+{
+ asm volatile ("r1 = 0; call same; r1 = 1; call same; exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("trips: a state gets to its trip after a later trip")
+__failure __msg("invalid access to map value, value_size=14 off=1048576")
+__naked void late_state_of_a_trip(void)
+{
+ asm volatile (
+ "call returns_late;"
+ "r1 = 0; call same_too;"
+ "r1 = 1; call same_too;"
+ "if r0 == 0 goto 9f;"
+ "r8 = 0;"
+ "r1 = 3; *(u64 *)(r10 - 8) = r1;"
+"1: call %[bpf_get_prandom_u32];"
+ "r6 = r0; r6 &= 3;"
+ "r7 = 0; r9 = 0;"
+ "*(u64 *)(r10 - 16) = r9;"
+ "*(u64 *)(r10 - 32) = r9;"
+ "*(u64 *)(r10 - 40) = r9;"
+"2: r0 = 0;"
+ "if r6 > 2 goto 3f;"
+ "if r8 == 0 goto 3f;"
+ "r2 = r9; r2 <<= 20;"
+ "r1 = %[small] ll; r1 += r2;"
+ "r2 = 1; *(u8 *)(r1 + 0) = r2;"
+"3: if r8 == 0 goto 4f;"
+ "r1 = *(u64 *)(r10 - 40);"
+ "if r1 == 0 goto 4f;"
+ "call returns_late;"
+ "if r0 == 0 goto 4f;"
+ "r9 = 1;"
+"4: r1 = *(u64 *)(r10 - 32);"
+ "r9 |= r1;"
+ "r1 = *(u64 *)(r10 - 40);"
+ "*(u64 *)(r10 - 32) = r1;"
+ "r1 = 1;"
+ "*(u64 *)(r10 - 40) = r1;"
+ "r6 += 1;"
+ "if r6 == 8 goto 5f;"
+ MAY_GOTO(5)
+ "r7 = *(u64 *)(r10 - 16);"
+ "r2 = *(u64 *)(r10 - 8);"
+ "r7 += r2;"
+ "*(u64 *)(r10 - 16) = r2;"
+ "goto 2b;"
+"5: r1 = %[small] ll; r1 += r7;"
+ "call read_u64;"
+ "r8 = 1;"
+ MAY_GOTO(1)
+ "goto 1b;"
+ EXIT_AT_9
+ :: __imm(bpf_get_prandom_u32), __imm_addr(small) : __clobber_all);
+}
+
+/* for (o = 0; o < 16; o++) for (u8 j = 0; j < 16 && can_loop; j++) a[k++] = 0; */
+#define GRID_IN_BYTES(at) \
+ VALUE_IN_R9 \
+ "w6 = 0; w8 = 0;" \
+"1: w7 = w8;" \
+ "w8 = w6; w8 += 16; w8 &= 255;" \
+"2:" MAY_GOTO(9) \
+ "w2 = w6; r1 = r9; r1 += r2; *(u8 *)(r1 + " #at ") = 0;" \
+ "w6 += 1;" \
+ "w2 = w6; w2 &= 255;" \
+ "if w8 == w2 goto 3f;" \
+ "goto 2b;" \
+"3: w8 = w7; w8 += 1; w7 &= 65535;" \
+ "if w7 < 15 goto 1b;" \
+ LOAD(r6) \
+ EXIT_AT_9
+
+SEC("tc")
+__description("trips: a distance is not larger than its numbers allow")
+__success INSNS(5)
+__naked void threshold_of_truncated_distance(void)
+{
+ ASM(GRID_IN_BYTES(1792));
+}
+
+SEC("tc")
+__description("trips: a distance, one byte too far")
+__failure __msg("invalid access to map value, value_size=2048")
+__naked void threshold_of_truncated_distance_bad(void)
+{
+ ASM(GRID_IN_BYTES(1793));
+}
+
+/*
+ * All trips are too many for the first budget. LOOP_DISJOINT_APART proves the load in the
+ * loop and makes the trips much more costly.
+ */
+#define BINARY_SEARCH(size) \
+ VALUE_IN_R9 \
+ "r4 = *(u64 *)(r9 + 8);" \
+ "r1 = r4;" \
+ "r1 &= 4;" \
+ "if r1 == 0 goto 9f;" \
+ "w6 = 0;" \
+ "w7 = " #size ";" \
+ "w3 = 0;" \
+ "r8 = -1;" \
+"1: w2 = w7;" \
+ "w2 += w6;" \
+ "w2 >>= 1;" \
+ "r5 = r2;" \
+ "r5 <<= 5;" \
+ "r1 = r9; r1 += r5; r0 = *(u64 *)(r1 + 0);" \
+ "if r0 == r4 goto 3f;" \
+ "if r0 < r4 goto 4f;" \
+ "w7 = w2;" \
+"4: w2 += 1;" \
+ "if r0 < r4 goto 5f;" \
+ "w2 = w6;" \
+"5: w6 = w2;" \
+ "r2 = r8;" \
+"3: if r0 == r4 goto 2f;" \
+ "if w3 > 126 goto 2f;" \
+ "w3 += 1;" \
+ "r8 = r2;" \
+ "if w6 < w7 goto 1b;" \
+"2: if r2 s< 0 goto 9f;" \
+ "r2 <<= 5;" \
+ LOAD(r2) \
+ EXIT_AT_9
+
+SEC("tc")
+__description("undo: a second choice is undone when the budget grows")
+__success INSNS(5)
+__naked void second_choice(void)
+{
+ ASM(BINARY_SEARCH(64));
+}
+
+SEC("tc")
+__description("undo: a second choice, one element too many")
+__failure __msg("invalid access to map value, value_size=2048 off=2048")
+__naked void second_choice_bad(void)
+{
+ ASM(BINARY_SEARCH(65));
+}
+
+/* r7 counts the trips of the inner loop, 16 * 16. Only its low byte ends that loop. */
+#define GRID_UNDER_LOCK(off) \
+ VALUE_IN_R9 \
+ "r6 = *(u64 *)(r9 + 0);" \
+ "if r6 != 2 goto 3f;" \
+ "call bpf_rcu_read_lock;" \
+"3: w2 = 0;" \
+ "w7 = 0;" \
+"1: w3 = w7;" \
+ "w3 += 16;" \
+ "w3 &= 255;" \
+"2: r1 = r9; r1 += r7; *(u8 *)(r1 + " #off ") = r2;" \
+ "w7 += 1;" \
+ "w5 = w7;" \
+ "w5 &= 255;" \
+ "if w3 != w5 goto 2b;" \
+ "w2 += 1;" \
+ "if w2 != 16 goto 1b;" \
+ LOAD(r7) \
+ "if r6 != 2 goto 9f;" \
+ "call bpf_rcu_read_unlock;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("peel_more: after an undo")
+__success INSNS(5)
+__naked void peel_more_after_undo(void)
+{
+ ASM(GRID_UNDER_LOCK(1792));
+}
+
+SEC("tc")
+__description("peel_more: after an undo, one byte too far")
+__failure __msg("invalid access to map value, value_size=2048 off=2048 size=1")
+__naked void peel_more_after_undo_bad(void)
+{
+ ASM(GRID_UNDER_LOCK(1793));
+}
+
+/* 16 suspects divide the budget. The block in front of the first one costs more than one part. */
+SEC("tc")
+__description("peel: a loop does not pay for the code in front of it")
+__success __log_level(2) __msg("pass 2") __not_msg("pass 3")
+__naked void long_block_before_loop(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ ".rept 600;"
+ "r0 = 0;"
+ ".endr;"
+ ".rept 16;"
+ "r7 = 1;"
+ "r8 = 0;"
+"1: r7 *= 2;"
+ "r8 += 1;"
+ "if r8 < 5 goto 1b;"
+ "r6 += r7;"
+ ".endr;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+#define WITHOUT_PERFMON __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __failure_unpriv
+
+#define LOOP_WITH_FP_16(cb) \
+ "call %[bpf_get_prandom_u32];" \
+ "if w0 == 7 goto 9f;" \
+ "r1 = 3;" \
+ "r2 = " #cb " ll;" \
+ "r3 = r10;" \
+ "r3 += -16;" \
+ "r4 = 0;" \
+ "call %[bpf_loop];"
+
+static __naked __noinline __used int store_number(void)
+{
+ ASM("r1 &= 7; *(u64 *)(r2 + 0) = r1; r0 = 0; exit;");
+}
+
+SEC("tc")
+__description("join: a part of a slot with two types is a number")
+__success WITHOUT_PERFMON __msg_unpriv("invalid size of register fill")
+__naked void part_of_two_types(void)
+{
+ ASM("r6 = r10;"
+ "r6 += -16;"
+ "*(u64 *)(r10 - 16) = r6;"
+ LOOP_WITH_FP_16(store_number)
+"9: r0 = *(u8 *)(r6 + 7);"
+ "exit;");
+}
+
+SEC("tc")
+__description("join: all of a slot with two types is no pointer")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void all_of_two_types(void)
+{
+ ASM("*(u64 *)(r10 - 16) = r10;"
+ LOOP_WITH_FP_16(store_number)
+"9: r1 = *(u64 *)(r10 - 16);"
+ "r0 = *(u8 *)(r1 - 8);"
+ "exit;");
+}
+
+SEC("tc")
+__description("fill: a loop over the bytes of a pointer")
+__success WITHOUT_PERFMON
+__naked void bytes_of_pointer(void)
+{
+ ASM(VALUE_IN_R9
+ "*(u64 *)(r10 - 16) = r9;"
+ "r6 = 0;"
+ "r7 = 0;"
+"1: r1 = r10;"
+ "r1 += -16;"
+ "r1 += r6;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r7 ^= r0;"
+ "r6 += 1;"
+ "if r6 < 8 goto 1b;"
+ "r9 = *(u64 *)(r10 - 16);"
+ "*(u8 *)(r9 + 0) = r7;"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fill: a part of a pointer is not its offset")
+__failure __msg("invalid access to map value, value_size=2048")
+__naked void part_of_pointer_is_unknown(void)
+{
+ ASM(VALUE_IN_R9
+ "*(u64 *)(r10 - 16) = r9;"
+ "r6 = *(u16 *)(r10 - 16);"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+#define RESERVE_TO_FP_16 \
+ "r1 = %[ringbuf] ll;" \
+ "r2 = 8;" \
+ "r3 = 0;" \
+ "call %[bpf_ringbuf_reserve];" \
+ "if r0 == 0 goto 9f;" \
+ "*(u64 *)(r10 - 16) = r0;" \
+ "r1 = *(u32 *)(r10 - 16);"
+
+SEC("tc")
+__description("fill: a part of a reference does not release it")
+__failure __msg("R1 type=scalar expected=ringbuf_mem")
+__naked void part_of_reference_released(void)
+{
+ ASM(RESERVE_TO_FP_16 "r2 = 0; call %[bpf_ringbuf_discard];" EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fill: the reference is held after a load of a part")
+__failure __msg("Unreleased reference")
+__naked void part_of_reference_leaked(void)
+{
+ ASM(RESERVE_TO_FP_16 EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fill: a part of a dynptr")
+__failure __msg("invalid read from stack")
+int part_of_dynptr(struct __sk_buff *skb)
+{
+ struct bpf_dynptr ptr;
+
+ bpf_dynptr_from_skb(skb, 0, &ptr);
+ return *(volatile __u32 *)&ptr;
+}
+
+/* fp-16 is 0. The helper writes @size bytes at fp-32 or at fp-32 + @off. */
+#define HELPER_WRITES(off, size) \
+ "r1 = 0;" \
+ "*(u64 *)(r10 - 16) = r1;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = 0;" \
+ "r2 = 8;" \
+ "if w0 & 1 goto 1f;" \
+ "r6 = " #off ";" \
+ "r2 = " #size ";" \
+"1: r1 = r10;" \
+ "r1 += -32;" \
+ "r1 += r6;" \
+ "call %[bpf_get_current_comm];" \
+ "r1 = *(u64 *)(r10 - 16);" \
+ "if r1 == 0 goto 9f;" \
+ "r0 = *(u8 *)(r1 + 0);" \
+ EXIT_AT_9
+
+SEC("raw_tp")
+__description("control_deps: the address that a helper writes to")
+__success
+__naked void helper_writes_around_zero(void)
+{
+ ASM(HELPER_WRITES(24, 7));
+}
+
+SEC("raw_tp")
+__description("control_deps: the address that a helper writes to, over the zero")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void helper_writes_over_zero(void)
+{
+ ASM(HELPER_WRITES(16, 8));
+}
+
+SEC("raw_tp")
+__description("control_deps: the size that a helper writes, over the zero")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void helper_writes_up_to_zero(void)
+{
+ ASM(HELPER_WRITES(8, 9));
+}
+
+/* fp-16 has a pointer. The helper writes at fp-24 or at fp-24 + @step. */
+#define WRITE_NEAR_PTR(step, size) \
+ VALUE_IN_R9 \
+ "*(u64 *)(r10 - 16) = r9;" \
+ RANDOM_BIT \
+ "r0 *= " #step ";" \
+ "r1 = r10;" \
+ "r1 += -24;" \
+ "r1 += r0;" \
+ size \
+ "call %[bpf_get_current_comm];" \
+ "r1 = *(u64 *)(r10 - 16);" \
+ "r0 = *(u8 *)(r1 + 0);" \
+ EXIT_AT_9
+
+SEC("raw_tp")
+__description("step: a helper writes before or after the pointer")
+__success __log_level(2) __not_msg("pass 1")
+__naked void write_around_ptr(void)
+{
+ ASM(WRITE_NEAR_PTR(16, "r2 = 8;"));
+}
+
+SEC("raw_tp")
+__description("step: the second address is the pointer")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void write_over_ptr(void)
+{
+ ASM(WRITE_NEAR_PTR(8, "r2 = 8;"));
+}
+
+SEC("raw_tp")
+__description("step: the size gets to the pointer")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void write_up_to_ptr(void)
+{
+ ASM(WRITE_NEAR_PTR(15, "r2 = 9;"));
+}
+
+SEC("raw_tp")
+__description("step: the size is 24 at the first address and 8 at the second one")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void write_rest_over_ptr(void)
+{
+ ASM(WRITE_NEAR_PTR(16, "r2 = 24; r2 -= r0;"));
+}
+
+/* p[0] = @first, p[1] = @second, r6 = 0 or 1, r1 = p[r6] */
+#define ONE_OF_TWO_PTRS(first, second) \
+ VALUE_IN_R9 \
+ "r1 = " first ";" \
+ "*(u64 *)(r10 - 24) = r1;" \
+ "r1 = " second ";" \
+ "*(u64 *)(r10 - 16) = r1;" \
+ RANDOM_BIT \
+ "r6 = r0;" \
+ "r0 <<= 3;" \
+ "r1 = r10;" \
+ "r1 += -24;" \
+ "r1 += r0;" \
+ "r1 = *(u64 *)(r1 + 0);"
+
+SEC("tc")
+__description("fork: load one of two pointers")
+__success __retval(0)
+#ifdef SPEC_V1
+WITHOUT_PERFMON __msg_unpriv("R1 variable stack access prohibited for !root")
+#endif
+__naked void load_one_of_two_ptrs(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "%[small] ll") "r0 = *(u8 *)(r1 + 13);" EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fork: one object is too small")
+__failure __msg("invalid access to map value, value_size=14 off=14 size=1")
+__naked void load_one_of_two_ptrs_bad(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "%[small] ll") "r0 = *(u8 *)(r1 + 14);" EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fork: a pointer or a number")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void load_ptr_or_number(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "7") "r0 = *(u8 *)(r1 + 0);" EXIT_AT_9);
+}
+
+#define P_AT_100_I "r6 *= 100; r1 += r6; r0 = *(u8 *)(r1 + 0);"
+
+SEC("tc")
+__description("fork: the index has one value in each state")
+__success __retval(0)
+__naked void index_follows_slot(void)
+{
+ ASM(ONE_OF_TWO_PTRS("%[small] ll", "r9") P_AT_100_I EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fork: the large index goes with the small object")
+__failure __msg("invalid access to map value, value_size=14 off=100 size=1")
+__naked void index_follows_slot_bad(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "%[small] ll") P_AT_100_I EXIT_AT_9);
+}
+
+/* p[0], p[1], p[4] and p[5] are pointers, p[2] and p[3] are numbers */
+#define P_AT_MASK(mask) \
+ VALUE_IN_R9 \
+ "r1 = 7;" \
+ "*(u64 *)(r10 - 56) = r9; *(u64 *)(r10 - 48) = r9;" \
+ "*(u64 *)(r10 - 40) = r1; *(u64 *)(r10 - 32) = r1;" \
+ "*(u64 *)(r10 - 24) = r9; *(u64 *)(r10 - 16) = r9;" \
+ "r0 = *(u8 *)(r9 + 0);" \
+ "r0 &= " #mask " * 8;" \
+ "r1 = r10;" \
+ "r1 += r0;" \
+ "r1 = *(u64 *)(r1 - 56);" \
+ "r0 = *(u8 *)(r1 + 0);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("fork: the index does not get to the numbers")
+__success __retval(0)
+__naked void index_with_gap(void)
+{
+ ASM(P_AT_MASK(5));
+}
+
+SEC("tc")
+__description("fork: the index gets to the numbers")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void index_with_gap_bad(void)
+{
+ ASM(P_AT_MASK(3));
+}
+
+/* The load from @map2k needs pass 1. One of the words that are added is a pointer. */
+SEC("tc")
+__description("fork: a word that is used as a number stays a number")
+__success __log_level(2) __msg("pass 1") __not_msg("fork R")
+__naked void word_stays_number(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "7")
+ "r6 = *(u64 *)(r9 + 8);"
+ "r6 += r1;"
+ "r7 = 0;"
+ "if r6 < 2048 goto +1;"
+ "r7 = -22;"
+ "call %[bpf_get_prandom_u32];"
+ "if r7 != 0 goto 9f;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+/* Pass 0 joins two addresses, so r1 is a number. The size is a word of the table. */
+SEC("raw_tp")
+__description("fork: another argument of the call stays a number")
+__success __log_level(2) __msg("pass 1")
+__naked void size_stays_number(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "%[small] ll")
+ "r7 = r1;"
+ "if r7 > 14 goto 9f;"
+ "if r7 == 0 goto 9f;"
+ "r8 = r10; r8 += -24;"
+ RANDOM_BIT
+ "if r0 == 0 goto +1;"
+ "r8 += 8;"
+ "r1 = *(u64 *)(r8 + 0);"
+ "r2 = r7;"
+ "call %[bpf_get_current_comm];"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("fork: an atomic add through one of two pointers")
+__success __retval(0)
+__naked void atomic_through_ptr(void)
+{
+ ASM(ONE_OF_TWO_PTRS("r9", "r9") "r2 = 1; lock *(u64 *)(r1 + 8) += r2;" EXIT_AT_9);
+}
+
+char large[64] SEC(".data.large");
+
+#define SUM_BY_BPF_FOR(off) \
+ char *volatile p[20]; \
+ int i, sum = 0; \
+ \
+ for (i = 0; i < 20; i++) \
+ p[i] = i & 1 ? small : large; \
+ bpf_for(i, 0, 20) \
+ sum += p[i][off]; \
+ return sum
+
+SEC("tc")
+__description("fork: the index is from bpf_for")
+__success __retval(0)
+int index_from_bpf_for(void *ctx)
+{
+ SUM_BY_BPF_FOR(13);
+}
+
+SEC("tc")
+__description("fork: the index is from bpf_for, some objects are too small")
+__failure __msg("invalid access to map value, value_size=14 off=14 size=1")
+int index_from_bpf_for_bad(void *ctx)
+{
+ SUM_BY_BPF_FOR(14);
+}
+
+static __naked __noinline __used void below_2k(void)
+{
+ asm volatile ("r0 = 0; if r1 >= 2048 goto +1; r0 = 1; exit;");
+}
+
+static __naked __noinline __used void other_below_2k(void)
+{
+ asm volatile ("r0 = 0; r1 = r2; if r1 >= 2048 goto +1; r0 = 1; exit;");
+}
+
+static __naked __noinline __used void load_if_r1_is_small(void)
+{
+ asm volatile ("r0 = 0; if r1 >= 14 goto 1f;"
+ "r1 = %[small] ll; r1 += r2; r0 = *(u8 *)(r1 + 0);"
+ "1: exit;" :: __imm_addr(small) : __clobber_all);
+}
+
+/* r6 is any number. @func gets r6 + @add. */
+#define CALL_WITH_R6(func, add) \
+ VALUE_IN_R9 \
+ "r6 = *(u64 *)(r9 + 0);" \
+ "r2 = *(u64 *)(r9 + 8);" \
+ "r1 = r6;" \
+ "r1 += " #add ";" \
+ "call " #func ";"
+
+SEC("tc")
+__description("in frames: the function tests the argument for the caller")
+__success
+__naked void callee_tests_arg(void)
+{
+ ASM(CALL_WITH_R6(below_2k, 0) "if r0 == 0 goto 9f;" LOAD(r6) EXIT_AT_9);
+}
+
+SEC("tc")
+__description("in frames: the function tests the argument minus one")
+__failure __msg("invalid access to map value, value_size=2048 off=2048")
+__naked void callee_tests_smaller_arg(void)
+{
+ ASM(CALL_WITH_R6(below_2k, -1) "if r0 == 0 goto 9f;" LOAD(r6) EXIT_AT_9);
+}
+
+SEC("tc")
+__description("in frames: the function tests another number")
+__failure __msg("unbounded min value is not allowed")
+__naked void callee_tests_other_arg(void)
+{
+ ASM(CALL_WITH_R6(other_below_2k, 0) "if r0 == 0 goto 9f;" LOAD(r6) EXIT_AT_9);
+}
+
+SEC("tc")
+__description("in frames: the function returns the argument")
+__success
+__naked void callee_returns_arg(void)
+{
+ ASM(CALL_WITH_R6(same, 0) "if r0 >= 2048 goto 9f;" LOAD(r6) EXIT_AT_9);
+}
+
+#define TWO_ARGS(add) \
+ "call %[bpf_get_prandom_u32];" \
+ "r1 = r0;" \
+ "r2 = r0;" \
+ "r2 += " #add ";" \
+ "call load_if_r1_is_small;" \
+ "exit;"
+
+SEC("tc")
+__description("in frames: two arguments are equal")
+__success
+__naked void args_are_equal(void)
+{
+ ASM(TWO_ARGS(0));
+}
+
+SEC("tc")
+__description("in frames: two arguments differ by one")
+__failure __msg("invalid access to map value, value_size=14 off=14")
+__naked void args_differ(void)
+{
+ ASM(TWO_ARGS(1));
+}
+
+/* r0 = (f(r1) + f(r1 + 1)) & 7. In frames 20 levels are 2^20 calls. */
+#define TREE(name, f) \
+static __naked __noinline __used void name(void) \
+{ \
+ asm volatile ("r6 = r1; call " #f "; r7 = r0;" \
+ "r1 = r6; r1 += 1; call " #f ";" \
+ "r0 += r7; r0 &= 7; exit;"); \
+}
+
+TREE(t1, same) TREE(t2, t1) TREE(t3, t2) TREE(t4, t3) TREE(t5, t4)
+TREE(t6, t5) TREE(t7, t6) TREE(t8, t7) TREE(t9, t8) TREE(t10, t9)
+TREE(t11, t10) TREE(t12, t11) TREE(t13, t12) TREE(t14, t13) TREE(t15, t14)
+TREE(t16, t15) TREE(t17, t16) TREE(t18, t17) TREE(t19, t18) TREE(t20, t19)
+
+/* The second load reads from the first one, so its deps come later */
+SEC("tc")
+__description("in frames: later deps do not add calls")
+__success INSNS(4)
+__naked void later_deps_add_no_calls(void)
+{
+ ASM(CALL_WITH_R6(below_2k, 0)
+ "if r0 == 0 goto 9f;"
+ LOAD(r6)
+ "r8 = r0; r8 &= 1;"
+ "r1 = *(u64 *)(r9 + 8);"
+ "call t20;"
+ "r8 += r0;"
+ "r6 = *(u64 *)(r9 + 16);"
+ "r7 = 0;"
+ "if r6 < 2040 goto +1;"
+ "r7 = 1;"
+ "if r7 != 0 goto 9f;"
+ "r6 += r8;"
+ LOAD(r6)
+ EXIT_AT_9);
+}
+
+/* r8 is fp-8 or fp-@low. The pointer at fp-16 is used after @write. */
+#define WRITE_AROUND_PTR(low, write) \
+ "*(u64 *)(r10 - 16) = r1;" \
+ "r8 = r10; r8 += -8;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 > 5 goto +2;" \
+ "r8 = r10; r8 += -" #low ";" \
+ "r1 = r8;" \
+ write \
+ "r1 = *(u64 *)(r10 - 16);" \
+ "r0 = *(u32 *)(r1 + 0);" \
+ "exit;"
+
+SEC("tc")
+__description("deps: a store to one of two slots keeps the slot between them")
+__success
+__naked void store_keeps_slot_between(void)
+{
+ ASM(WRITE_AROUND_PTR(24, "*(u64 *)(r1 + 0) = 1;"));
+}
+
+SEC("tc")
+__description("deps: a store to one of two slots forgets both")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void store_forgets_slot(void)
+{
+ ASM(WRITE_AROUND_PTR(16, "*(u64 *)(r1 + 0) = 1;"));
+}
+
+static __naked __noinline __used void read_up_to_8_bytes(void)
+{
+ asm volatile ("r6 = r1;"
+ "call %[bpf_get_prandom_u32];"
+ "r1 = r6; r2 = r0; r2 &= 7; r2 += 1; r3 = 0;"
+ "call %[bpf_probe_read_kernel];"
+ "exit;"
+ :: __imm(bpf_get_prandom_u32), __imm(bpf_probe_read_kernel)
+ : __clobber_all);
+}
+
+SEC("tc")
+__description("deps: a helper keeps the slot between two buffers of the caller")
+__success
+__naked void helper_keeps_slot_of_caller(void)
+{
+ ASM(WRITE_AROUND_PTR(24, "call read_up_to_8_bytes;"));
+}
+
+SEC("tc")
+__description("deps: a helper writes to a slot of the caller")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void helper_forgets_slot_of_caller(void)
+{
+ ASM(WRITE_AROUND_PTR(17, "call read_up_to_8_bytes;"));
+}
+
+SEC("tc")
+__description("fork: a load through a value from an unproven insn asks for none")
+__success
+__naked void made_up_ptr_needs_no_fork(void)
+{
+ asm volatile (
+ "r9 = r1;"
+ "call %[bpf_get_prandom_u32];"
+ /* r6 is a number, or the bits of a pointer */
+ "*(u64 *)(r10 - 8) = r9;"
+ "*(u64 *)(r10 - 16) = r0;"
+ "r0 &= 8;"
+ "r1 = r10;"
+ "r1 += -16;"
+ "r1 += r0;"
+ "r6 = *(u64 *)(r1 + 0);"
+ "if r6 > 7 goto 9f;"
+ /* r7 = 32, the first pass does not know it */
+ "r8 = 0;"
+ "r7 = 1;"
+"1: r7 *= 2;"
+ "r8 += 1;"
+ "if r8 < 5 goto 1b;"
+ "*(u64 *)(r10 - 32) = r9;"
+ "r1 = r10;"
+ "r1 += -64;"
+ "r1 += r7;"
+ "r1 = *(u64 *)(r1 + 0);"
+ "r1 = *(u32 *)(r1 + 76);"
+ "r2 = *(u32 *)(r9 + 80);"
+ "r3 = r1;"
+ "r3 += 8;"
+ "if r3 > r2 goto 9f;"
+ "r1 += r6;"
+ "r1 = *(u8 *)(r1 + 0);"
+"9: r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_passes.c b/tools/testing/selftests/bpf/progs/verifier_passes.c
new file mode 100644
index 00000000000..a6edb33d001
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_passes.c
@@ -0,0 +1,3636 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Pass 0 joins all numbers. An error selects what the next pass keeps apart. */
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+#include "bpf_experimental.h"
+#include <bpf/bpf_core_read.h>
+#include "bpf_kfuncs.h"
+
+struct bytes64 {
+ char b[64];
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct bytes64);
+} map64 SEC(".maps");
+
+struct with_lock {
+ long val;
+ struct bpf_spin_lock lock;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct with_lock);
+} map_lock SEC(".maps");
+
+struct with_timer {
+ struct bpf_timer timer;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct with_timer);
+} map_timer SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_USER_RINGBUF);
+ __uint(max_entries, 4096);
+} user_rb SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_RINGBUF);
+ __uint(max_entries, 4096);
+} rb SEC(".maps");
+
+char out[8];
+
+extern void __bpf_trap(void) __weak __ksym;
+
+#define OPS \
+ __imm_addr(map64), __imm_addr(map_timer), __imm_addr(map_lock), \
+ __imm(bpf_spin_lock), __imm(bpf_spin_unlock), __imm(bpf_loop), \
+ __imm(bpf_get_prandom_u32), __imm(bpf_map_lookup_elem), \
+ __imm(bpf_timer_set_callback), __imm(bpf_iter_num_new), \
+ __imm(bpf_iter_num_next), __imm(bpf_iter_num_destroy)
+
+#define ASM(...) asm volatile (__VA_ARGS__ :: OPS : __clobber_all)
+
+/* r9 = value of map64 */
+#define VALUE_IN_R9 \
+ "r1 = 0;" \
+ "*(u64 *)(r10 - 8) = r1;" \
+ "r1 = %[map64] ll;" \
+ "r2 = r10;" \
+ "r2 += -8;" \
+ "call %[bpf_map_lookup_elem];" \
+ "if r0 == 0 goto 9f;" \
+ "r9 = r0;"
+
+#define EXIT_AT_9 "9: r0 = 0; exit;"
+#define MAY_GOTO(off) ".byte 0xe5; .byte 0; .short " #off "; .long 0;"
+#define INSNS(digits) __log_level(4) __msg("processed {{[0-9]{1," #digits "} }}insns")
+#define FEW_INSNS INSNS(5)
+
+/* r7, r8 = 0, 0 or 1, r0: no equation. Only the join makes a path to what follows. */
+#define PATH_OF_THE_JOIN \
+ "call %[bpf_get_prandom_u32];" \
+ "r7 = 0;" \
+ "r8 = 0;" \
+ "if w0 & 1 goto 1f;" \
+ "r7 = 1;" \
+ "r8 = r0;" \
+"1: if r7 != 0 goto 9f;" \
+ "if r8 == 0 goto 9f;"
+
+/* The example at bpf_compute_deps() and bpf_must_keep_apart() */
+#define JOIN_RULE(bound) \
+ VALUE_IN_R9 \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = r0;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r7 = 0;" \
+ "if r6 <= " #bound " goto 5f;" \
+ "r7 = -22;" \
+"5: r8 = 1;" \
+ "if r0 == 0 goto 8f;" \
+ "r8 = 2;" \
+"8: if r8 == 3 goto 9f;" \
+ "if r7 != 0 goto 9f;" \
+ "r2 = r6;" \
+ "r2 <<= 3;" \
+ "r9 += r2;" \
+ "r0 = *(u64 *)(r9 + 0);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("deps: data and control differ, keep apart. Control only, join")
+__success __log_level(2)
+__msg("pass 1")
+__msg("R8=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=2")
+__not_msg("pass 2")
+__naked void join_rule(void)
+{
+ ASM(JOIN_RULE(7));
+}
+
+SEC("tc")
+__description("deps: the ok_state is out of bounds")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=8")
+__naked void join_rule_bad(void)
+{
+ ASM(JOIN_RULE(8));
+}
+
+SEC("tc")
+__description("deps: no error, no second pass")
+__success __log_level(2) __not_msg("pass 1")
+__naked void one_pass(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = 0;"
+ "if r0 == 0 goto 1f;"
+ "r6 = 56;"
+"1: r9 += r6;"
+ "r0 = *(u64 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("an error in a state that is not imprecise is real")
+__failure __log_level(2) __msg("invalid access to map value") __not_msg("pass 1")
+__naked void precise_error(void)
+{
+ ASM(VALUE_IN_R9
+ "r0 = *(u64 *)(r9 + 64);"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("transitive_deps: two control deps go together")
+__success __log_level(2) __msg("pass 1")
+__naked void transitive_deps(void)
+{
+ ASM(PATH_OF_THE_JOIN
+ "r0 = *(u64 *)(r10 + 0);"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void same(void)
+{
+ asm volatile ("r0 = r1; exit;");
+}
+
+#define TWO_CALLS(first) \
+ VALUE_IN_R9 \
+ "r1 = " #first ";" \
+ "call same;" \
+ "r6 = r0;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r1 = r0;" \
+ "call same;" \
+ "r9 += r6;" \
+ "r0 = *(u8 *)(r9 + 0);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("instances: the result of a call goes with its arguments")
+__success
+__naked void call_instance(void)
+{
+ ASM(TWO_CALLS(63));
+}
+
+SEC("tc")
+__description("instances: the argument of the call is too large")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void call_instance_bad(void)
+{
+ ASM(TWO_CALLS(64));
+}
+
+static __naked __noinline __used void read_at_sum(void)
+{
+ asm volatile ("r1 = *(u64 *)(r1 + 0); r3 += r1; r3 += r2; r0 = *(u8 *)(r3 + 0); exit;");
+}
+
+/* r6, r7 = 0..7, 0..55 or 0..55, 0..7 */
+SEC("tc")
+__description("deps: the deepest slot is live only before a call")
+__success
+__naked void slot_is_live_at_call(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "call %[bpf_get_prandom_u32];"
+ "r7 = r0;"
+ "call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 1f;"
+ "r6 &= 7;"
+ "r7 &= 55;"
+ "goto 2f;"
+"1: r6 &= 55;"
+ "r7 &= 7;"
+"2: r1 = r10;"
+ "r1 += -448;"
+ "r2 = r6;"
+ "r3 = r9;"
+ "*(u64 *)(r1 + 0) = r7;"
+ "call read_at_sum;"
+ EXIT_AT_9);
+}
+
+/* x * x - 3 * x + 2 is 0 for 1 and for 2 */
+#define ROOTS(second) \
+ VALUE_IN_R9 \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = 1;" \
+ "if r0 == 0 goto 1f;" \
+ "r6 = " #second ";" \
+"1: r1 = r6;" \
+ "r1 *= r6;" \
+ "r6 *= 3;" \
+ "r1 -= r6;" \
+ "r1 += 2;" \
+ "r9 += r1;" \
+ "r0 = *(u8 *)(r9 + 63);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("keep_disjoint_apart: one xreg of data_deps, no gap")
+__success
+__naked void disjoint_apart(void)
+{
+ ASM(ROOTS(2));
+}
+
+SEC("tc")
+__description("keep_disjoint_apart: one value is no root")
+__failure __msg("invalid access to map value, value_size=64 off=65 size=1")
+__naked void disjoint_apart_bad(void)
+{
+ ASM(ROOTS(0));
+}
+
+#define PTR_OR_NUMBER(jump) \
+ VALUE_IN_R9 \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = 0;" \
+ "if r0 == 0 goto 1f;" \
+ "r6 = r9;" \
+"1: *(u64 *)(r10 - 16) = r6;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 == 0 goto 2f;" \
+ "r0 = 0;" \
+"2: r1 = *(u64 *)(r10 - 16);" \
+ jump \
+ "r0 = *(u8 *)(r1 + 0);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("live types: a pointer or a number")
+__success __log_level(2) __not_msg("pass 1")
+__naked void unreadable(void)
+{
+ ASM(PTR_OR_NUMBER("if r1 == 0 goto 9f;"));
+}
+
+SEC("tc")
+__description("unreadable: the number is used as a pointer")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void unreadable_bad(void)
+{
+ ASM(PTR_OR_NUMBER(""));
+}
+
+/* Only one path allocates the slot of the caller. It is dead when the paths meet. */
+static __naked __noinline __used void read_on_one_path(void)
+{
+ ASM("r6 = r1;"
+ "call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 1f;"
+ "r0 = *(u64 *)(r6 + 0);"
+"1: r0 = 0;"
+ ".rept 100; r0 ^= 1; .endr;"
+ "exit;");
+}
+
+SEC("tc")
+__description("live types: a dead slot keeps no states apart")
+__success __log_level(4) __msg("processed 1{{[0-9]{2} }}insns")
+__naked void dead_slot(void)
+{
+ ASM("r1 = r10; r1 += -8; call read_on_one_path; exit;");
+}
+
+static __naked __noinline __used void read_if_set(void)
+{
+ asm volatile (
+ "r0 = 0;"
+ "if r1 == 0 goto 1f;"
+ "r0 = *(u32 *)(r1 + 0);"
+"1: exit;");
+}
+
+#define PTR_OR_NUMBER_ARG(n) \
+ "call read_if_set;" \
+ "r1 = " #n ";" \
+ "call read_if_set;" \
+ "exit;"
+
+SEC("tc")
+__description("unreadable: the argument is a pointer or a number")
+__success __log_level(2) __msg("R1 !read_ok") __msg("pass 1") __not_msg("pass 2")
+__naked void unreadable_arg(void)
+{
+ ASM(PTR_OR_NUMBER_ARG(0));
+}
+
+SEC("tc")
+__description("unreadable: the argument is not a pointer")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void unreadable_arg_bad(void)
+{
+ ASM(PTR_OR_NUMBER_ARG(1));
+}
+
+/* r6 is the start of a header, r2 is its end. The header has 22 or 14 bytes. */
+#define PKT_TWO_LENGTHS(off) \
+ "r9 = r1;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = *(u32 *)(r9 + 76);" \
+ "r3 = *(u32 *)(r9 + 80);" \
+ "r1 = r0;" \
+ "r1 &= 63;" \
+ "r6 += r1;" \
+ "r2 = r6;" \
+ "r2 += 22;" \
+ "if r0 & 64 goto 1f;" \
+ "r2 += -8;" \
+"1: if r2 > r3 goto 9f;" \
+ "r0 = *(u8 *)(r6 + " #off ");" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("join point: two packet pointers have two distances")
+__success
+__naked void pkt_two_lengths(void)
+{
+ ASM(PKT_TWO_LENGTHS(13));
+}
+
+SEC("tc")
+__description("join point: two packet pointers have two distances, out of bounds")
+__failure __msg("invalid access to packet")
+__naked void pkt_two_lengths_bad(void)
+{
+ ASM(PKT_TWO_LENGTHS(14));
+}
+
+#define PKT_CHECKED(ret) \
+ "r6 = *(u32 *)(r1 + 76);" \
+ "r2 = *(u32 *)(r1 + 80);" \
+ "r3 = r6;" \
+ "r3 += 8;" \
+ "r7 = " #ret ";" \
+ "if r3 > r2 goto 1f;" \
+ "r7 = 0;" \
+"1: if r7 != 0 goto 9f;" \
+ "r0 = *(u8 *)(r6 + 7);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("live types: a packet pointer with and without checked bytes")
+__success __log_level(2) __not_msg("pass 1")
+__naked void pkt_checked(void)
+{
+ ASM(PKT_CHECKED(1));
+}
+
+SEC("tc")
+__description("live types: the result of the check is lost")
+__failure __msg("invalid access to packet")
+__naked void pkt_checked_bad(void)
+{
+ ASM(PKT_CHECKED(0));
+}
+
+#define ONE_OF_TWO_MEMBERS(size) \
+ asm volatile ( \
+ "call %[bpf_get_current_task_btf];" \
+ "r6 = r0;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if w0 & 1 goto 1f;" \
+ "r6 += %[pid];" \
+ "goto 2f;" \
+"1: r6 += %[flags];" \
+"2: r0 = *(" #size " *)(r6 + 0);" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_get_current_task_btf), __imm(bpf_get_prandom_u32), \
+ __imm_const(pid, offsetof(struct task_struct, pid)), \
+ __imm_const(flags, offsetof(struct task_struct, flags)) \
+ : __clobber_all)
+
+SEC("raw_tp")
+__description("join rule: a pointer to one of two members")
+__success __log_level(2) __msg("pass 1")
+__naked void one_of_two_members(void)
+{
+ ONE_OF_TWO_MEMBERS(u32);
+}
+
+SEC("raw_tp")
+__description("join rule: the load is larger than the members")
+__failure __msg("access beyond the end of member")
+__naked void one_of_two_members_bad(void)
+{
+ ONE_OF_TWO_MEMBERS(u64);
+}
+
+/* r6 is -3..4 with 10 bytes after it, or -1..6 with 8 bytes */
+#define PKT_BEFORE_START(off) \
+ "r9 = r1;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = *(u32 *)(r9 + 76);" \
+ "r2 = *(u32 *)(r9 + 80);" \
+ "r3 = r6;" \
+ "r3 += 14;" \
+ "if r3 > r2 goto 9f;" \
+ "r1 = r0;" \
+ "r1 &= 7;" \
+ "if r0 > 99 goto 1f;" \
+ "r6 += 4;" \
+ "r6 -= r1;" \
+ "goto 2f;" \
+"1: r6 += 6;" \
+ "r6 -= r1;" \
+"2: r6 += 3;" \
+ "r0 = *(u8 *)(r6 + " #off ");" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("join: a packet pointer with a negative offset keeps its checked bytes")
+__success __log_level(2) __not_msg("pass 1")
+__naked void pkt_before_start(void)
+{
+ ASM(PKT_BEFORE_START(4));
+}
+
+SEC("tc")
+__description("join: a packet pointer with a negative offset, one byte too far")
+__failure __msg("invalid access to packet")
+__naked void pkt_before_start_bad(void)
+{
+ ASM(PKT_BEFORE_START(5));
+}
+
+/* The argument is the end of the packet, so each call is checked in frames */
+static __naked __noinline __used void many_insns(void)
+{
+ asm volatile (
+ "r0 = r1;"
+ "r0 = 0;"
+ ".rept 10000; r0 ^= 1; .endr;"
+ "exit;");
+}
+
+static __naked __noinline __used void many_calls(void)
+{
+ asm volatile (
+ "r6 = r1;"
+ ".rept 11; r1 = r6; call many_insns; .endr;"
+ "exit;");
+}
+
+SEC("tc")
+__description("live types: long code with one state does not use the budget")
+__success __log_level(4) __msg("processed 1{{[0-9]{6} }}insns")
+__naked void types_after_long_code(void)
+{
+ ASM("r6 = *(u32 *)(r1 + 80);"
+ ".rept 10; r1 = r6; call many_calls; .endr;"
+ PTR_OR_NUMBER("if r1 == 0 goto 9f;"));
+}
+
+/* 1 << 40 states, or one */
+#define MANY_TYPES(check) \
+ VALUE_IN_R9 \
+ "r8 = r10;" \
+ "r8 += -8;" \
+ ".rept 40;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r1 = 0;" \
+ "if r0 == 0 goto +1;" \
+ "r1 = r9;" \
+ "r8 += -8;" \
+ "*(u64 *)(r8 + 0) = r1;" \
+ ".endr;" \
+ ".rept 40;" \
+ "r1 = *(u64 *)(r8 + 0);" \
+ "r8 += 8;" \
+ ".endr;" \
+ check \
+ "r0 = *(u8 *)(r1 + 0);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("live types: over the budget")
+__success
+__naked void many_types(void)
+{
+ ASM(MANY_TYPES("if r1 == 0 goto 9f;"));
+}
+
+SEC("tc")
+__description("live types: over the budget, the number is used as a pointer")
+__failure __msg("R1 invalid mem access 'scalar'")
+__naked void many_types_bad(void)
+{
+ ASM(MANY_TYPES(""));
+}
+
+static __naked __noinline __used void leave_fp(void)
+{
+ asm volatile (
+ "r0 = r10;"
+ "r0 += -8;"
+ "exit;");
+}
+
+SEC("tc")
+__description("unreadable: a function leaves a pointer to its stack in r0")
+__success
+__naked void fp_in_r0(void)
+{
+ ASM("call leave_fp; r0 = 0; exit;");
+}
+
+SEC("tc")
+__description("unreadable: the caller reads that pointer")
+__failure __msg("the caller cannot read the stack pointer in R0") __msg("R0 !read_ok")
+__msg("invalidated: R0: the caller cannot read")
+__naked void fp_in_r0_bad(void)
+{
+ ASM("call leave_fp; r0 = *(u64 *)(r0 + 0); exit;");
+}
+
+SEC("tc")
+__description("unreadable: a callback has to return a number")
+__failure __not_msg("the caller cannot read") __msg("R0 not a scalar value")
+__naked void fp_in_r0_of_callback(void)
+{
+ ASM("r1 = 1;"
+ "r2 = leave_fp ll;"
+ "r3 = 0;"
+ "r4 = 0;"
+ "call %[bpf_loop];"
+ "r0 = 0;"
+ "exit;");
+}
+
+SEC("socket")
+__description("unreadable: an atomic insn doesn't make a number of the slot")
+__success __failure_unpriv __msg_unpriv("leaking pointer from stack off -16")
+__naked void unreadable_atomic(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = *(u64 *)(r9 + 8);"
+ "*(u64 *)(r10 - 16) = r9;"
+ "if r6 == 0 goto 1f;"
+ "r1 = 5;"
+ "*(u64 *)(r10 - 16) = r1;"
+"1: r1 = 0;"
+ "lock *(u64 *)(r10 - 16) += r1;"
+ "r1 = *(u64 *)(r10 - 16);"
+ "*(u64 *)(r9 + 0) = r1;"
+ EXIT_AT_9);
+}
+
+/* r6 goes up to 5, r7 goes down from @start to 0. The sum goes down. */
+#define HALVES(start) \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = " #start ";" \
+"1:" MAY_GOTO(8) \
+ "r1 = r9;" \
+ "r1 += r6;" \
+ "r1 += r7;" \
+ "r0 = *(u8 *)(r1 + 55);" \
+ "r6 += 6;" \
+ "r6 >>= 1;" \
+ "r7 >>= 1;" \
+ "goto 1b;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("peel: one trip")
+__success __log_level(2) __msg("pass 1") __msg(")=0 loop_stack=")
+__naked void peel_1(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r7 = r0;"
+ "r7 &= 0xff;"
+ "r8 = 0;"
+"1:" MAY_GOTO(8)
+ "if r8 == 0 goto 2f;"
+ "r1 = r9;"
+ "r1 += r7;"
+ "r0 = *(u8 *)(r1 + 0);"
+"2: r7 = r6;"
+ "r7 &= 0x3f;"
+ "r8 = r6;"
+ "goto 1b;"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("peel: more than one trip")
+__success __log_level(2) __msg("pass 1") __msg(")=1 loop_stack=")
+__naked void peel_2(void)
+{
+ ASM(HALVES(8));
+}
+
+SEC("tc")
+__description("peel: error in a peeled trip")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void peel_bad(void)
+{
+ ASM(HALVES(9));
+}
+
+SEC("tc")
+__description("peel: all trips, the trip count is proven")
+__success
+__xlated("r6 += 1")
+__xlated("if r6 < 0x20 goto")
+__naked void peel_trip_count(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+"1: r1 = r6;"
+ "r1 &= 1;"
+ "r7 += r1;"
+ "r6 += 1;"
+ "if r6 < 32 goto 1b;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 47);"
+ EXIT_AT_9);
+}
+
+#define PERIOD_3(test) \
+ VALUE_IN_R9 \
+ "r6 = 1;" \
+ "r7 = 20;" \
+ "r8 = 40;" \
+"1:" test \
+ "r2 = r9;" \
+ "r2 += r6;" \
+ "r2 += r7;" \
+ "r1 = *(u8 *)(r2 + 3);" \
+ "r1 = r6;" \
+ "r6 = r7;" \
+ "r7 = r8;" \
+ "r8 = r1;" \
+ "goto 1b;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("peel: three trips, a state repeats with a period")
+__success __log_level(2) __msg("pass 1") __msg(")=2 loop_stack=")
+__naked void trips_repeat_with_a_period(void)
+{
+ ASM(PERIOD_3(MAY_GOTO(9)));
+}
+
+SEC("tc")
+__description("peel: a state repeats with a period, may_goto is added")
+__success
+__xlated("r1 = *(u8 *)(r2 +3)")
+__naked void trips_repeat_without_may_goto(void)
+{
+ ASM(PERIOD_3("call %[bpf_get_prandom_u32]; if r0 == 0 goto 9f;"));
+}
+
+/* 2, 4, 16, 256, and the outer loop starts again. A trip of the last time subsumes the trip. */
+SEC("tc")
+__description("peel: the state that continues is from an earlier trip of the outer loop")
+__arch_x86_64 __arch_arm64
+__success
+__xlated("call unknown")
+__naked void prune_by_outer_loop(void)
+{
+ ASM(VALUE_IN_R9
+ "r1 = r10;"
+ "r1 += -16;"
+ "r2 = 0;"
+ "r3 = 10;"
+ "call %[bpf_iter_num_new];"
+"1: r1 = r10;"
+ "r1 += -16;"
+ "call %[bpf_iter_num_destroy];"
+ "r1 = r10;"
+ "r1 += -16;"
+ "r2 = 0;"
+ "r3 = 10;"
+ "call %[bpf_iter_num_new];"
+ "r7 = 2;"
+"2: if r7 > 200 goto 1b;"
+ "r1 = r10;"
+ "r1 += -16;"
+ "call %[bpf_iter_num_next];"
+ "if r0 == 0 goto 8f;"
+ "r1 = r9;"
+ "r1 += r7;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r7 *= r7;"
+ "goto 2b;"
+"8: r1 = r10;"
+ "r1 += -16;"
+ "call %[bpf_iter_num_destroy];"
+ EXIT_AT_9);
+}
+
+/* r7 is 0..8 in trip 9 and in the leftover. r8 only decides when the loop is left. */
+SEC("tc")
+__description("failing_path: after a loop, trips are skipped")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void leftover_is_precise(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = 0;"
+ "r8 = 0;"
+"1: call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 2f;"
+ "if r7 >= 8 goto 2f;"
+ "r7 += 1;"
+"2: r8 += 1;"
+ "if r8 < 100000 goto 1b;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 56);"
+ EXIT_AT_9);
+}
+
+/* r6 + r7 is 0 on all 1 << 24 paths */
+#define ALL_PATHS(off) \
+ "r8 = *(u32 *)(r9 + 0);" \
+ "r5 = 1;" \
+ "r6 = 0;" \
+ "r7 = 0;" \
+ ".rept 24;" \
+ "r8 >>= 1;" \
+ "r5 <<= 1;" \
+ "if r8 & 1 goto +2;" \
+ "r6 += r5;" \
+ "r7 -= r5;" \
+ ".endr;" \
+ "r6 += r7;" \
+ "if r6 == 0 goto 9f;" \
+ "r0 = *(u8 *)(r9 + " #off ");"
+
+SEC("socket")
+__description("budget: too many states are apart")
+__failure_unpriv __msg_unpriv("the limit on cost or memory ended pass")
+__naked void over_the_limit(void)
+{
+ ASM(VALUE_IN_R9 ALL_PATHS(64) EXIT_AT_9);
+}
+
+static __naked __noinline __used void long_dead_code(void)
+{
+ asm volatile ("r0 = 0;"
+ "if r1 == 0 goto 1f;"
+ ".rept 30000;"
+ "r0 += r1;"
+ ".endr;"
+"1: exit;" ::: __clobber_all);
+}
+
+/* The first load needs pass 2. Pass 1 ends while it looks for the constants of a loop. */
+SEC("tc")
+__description("budget: a pass ends before it has all thresholds of a loop")
+__success
+__naked void thresholds_over_budget(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r7 = 0;"
+ "if r6 > 7 goto 1f;"
+ "r7 = 1;"
+"1: call %[bpf_get_prandom_u32];"
+ "r2 = 0;"
+ "if r7 == 0 goto 2f;"
+ "r2 = r6;"
+"2: r1 = r9;"
+ "r1 += r2;"
+ "r0 = *(u8 *)(r1 + 0);"
+ ".rept 350;"
+ "r6 = 0;"
+"3:" MAY_GOTO(10)
+ "call %[bpf_get_prandom_u32];"
+ "if r0 & 1 goto 4f;"
+ "if r6 >= 60000 goto 4f;"
+ "r6 += 1;"
+"4: if r6 == -1 goto 9f;"
+ "if r6 == -2 goto 9f;"
+ "if r6 == -3 goto 9f;"
+ "r1 = 0;"
+ "call long_dead_code;"
+ "goto 3b;"
+ "r6 >>= 10;"
+ "r1 = r9;"
+ "r1 += r6;"
+ "r0 = *(u8 *)(r1 + 0);"
+ ".endr;"
+ EXIT_AT_9);
+}
+
+/* r6 == r7 on all 1 << 24 paths. The deps of the load keep all of them apart. */
+SEC("socket")
+__description("budget: the log has the error of the pass before the limit")
+__failure_unpriv __msg_unpriv("R6 invalid mem access 'scalar'")
+__msg_unpriv("the limit on cost or memory ended pass 1")
+__naked void deps_over_the_limit(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r8 = r0;"
+ "r5 = 1;"
+ "r6 = 0;"
+ "r7 = 0;"
+ ".rept 24;"
+ "r8 >>= 1;"
+ "r5 <<= 1;"
+ "if r8 & 1 goto +2;"
+ "r6 += r5;"
+ "r7 ^= r5;"
+ ".endr;"
+ "r6 -= r7;"
+ "if r6 == 0 goto 9f;"
+ "r0 = *(u64 *)(r6 + 0);"
+ EXIT_AT_9);
+}
+
+/* The jump leaves -48 of -48 and -24. A mask would let -24 pass. */
+SEC("socket")
+__description("speculation: no mask for a pointer to the stack")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("w13 = 48")
+__xlated_unpriv("r2 -= r13")
+#endif
+__naked void stack_offset_is_imm(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = *(u64 *)(r9 + 0);"
+ "*(u64 *)(r10 - 56) = r9;"
+ "*(u64 *)(r10 - 32) = r7;"
+ "r1 = -48;"
+ "if r7 == 0 goto 1f;"
+ "r1 = -24;"
+"1: if r1 s> -30 goto 9f;"
+ "r2 = r10;"
+ "r2 += -8;"
+ "r2 += r1;"
+ "r3 = *(u64 *)(r2 + 0);"
+ "r0 = *(u8 *)(r3 + 0);"
+ EXIT_AT_9);
+}
+
+/* The join of 0, 10, 32, 42 is cut at 38. That is no member, the bound moves by less than 24. */
+SEC("socket")
+__description("speculation: the immediate is the offset")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("if r1 > 0x26 goto")
+__xlated_unpriv("r2 = r9")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r2 += r1")
+__xlated_unpriv("r4 = 24")
+__xlated_unpriv("w13 = 24")
+#endif
+__naked void imm_is_the_offset(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = *(u64 *)(r9 + 0);"
+ "r1 = 0;"
+ "if r7 == 0 goto 1f;"
+ "r1 = 42;"
+ "if r7 == 1 goto 1f;"
+ "r1 = 32;"
+ "if r7 == 2 goto 1f;"
+ "r1 = 10;"
+"1: if r1 > 38 goto 9f;"
+ "r2 = r9;"
+ "r2 += r1;"
+ "r4 = 24;"
+ "r2 += r4;"
+ "r0 = *(u8 *)(r2 + 0);"
+ EXIT_AT_9);
+}
+
+SEC("socket")
+__description("unreadable: a pointer or not written")
+__success __failure_unpriv __msg_unpriv("invalid read from stack R10 off=-16 size=8")
+__naked void pointer_or_not_written(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = *(u64 *)(r9 + 0);"
+ "if r7 == 0 goto 1f;"
+ "*(u64 *)(r10 - 16) = r9;"
+"1: r1 = *(u64 *)(r10 - 16);"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("failing_path: the error is in trip 100000000, the trips before it are skipped")
+__failure __msg("invalid access to map value, value_size=64 off=1000 size=1") FEW_INSNS
+__naked void stored_counter(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+"1: call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 9f;"
+ "r6 += 1;"
+ "if r6 != 100000000 goto 2f;"
+ "r7 = 1000;"
+"2: r1 = r9;"
+ "r1 += r7;"
+ "*(u8 *)(r1 + 0) = r6;"
+ "goto 1b;"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("path of the join: a trap")
+__success __log_level(2) __msg("pass 1")
+__naked void trap_is_not_reached(void)
+{
+ ASM(PATH_OF_THE_JOIN
+ "call __bpf_trap;"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used int timer_cb_bad(void)
+{
+ asm volatile ("r0 = *(u64 *)(r3 + 40); r0 = 0; exit;");
+}
+
+SEC("tc")
+__description("path of the join: an async callback")
+__success __log_level(2) __msg("pass 1")
+__naked void async_cb_is_not_set(void)
+{
+ ASM(PATH_OF_THE_JOIN
+ "r1 = 0;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "r1 = %[map_timer] ll;"
+ "r2 = r10;"
+ "r2 += -8;"
+ "call %[bpf_map_lookup_elem];"
+ "if r0 == 0 goto 9f;"
+ "r1 = r0;"
+ "r2 = timer_cb_bad ll;"
+ "call %[bpf_timer_set_callback];"
+ EXIT_AT_9);
+}
+
+__noinline __used int global_bad(int *p)
+{
+ return *p;
+}
+
+SEC("tc")
+__description("path of the join: a global function")
+__success __log_level(2) __msg("pass 1")
+__naked void global_is_not_called(void)
+{
+ ASM(PATH_OF_THE_JOIN
+ "r1 = 0;"
+ "call global_bad;"
+ EXIT_AT_9);
+}
+
+struct drain_ctx {
+ struct bpf_dynptr clone;
+ struct bpf_dynptr *ptr;
+};
+
+static long keep_dynptr(struct bpf_dynptr *ptr, void *arg)
+{
+ struct drain_ctx *ctx = arg;
+
+ bpf_dynptr_clone(ptr, &ctx->clone);
+ ctx->ptr = ptr;
+ if (bpf_get_prandom_u32() & 1)
+ out[1] = 1;
+ return 0;
+}
+
+SEC("tc")
+__description("ids: the frame of the callback and a slot of the caller have one id")
+__failure __msg("R3 type=scalar expected=fp, dynptr_ptr")
+int arg_of_callback_is_gone(void *unused)
+{
+ struct drain_ctx ctx;
+
+ ctx.ptr = NULL;
+ bpf_user_ringbuf_drain(&user_rb, keep_dynptr, &ctx, 0);
+ if (ctx.ptr)
+ bpf_dynptr_read(out, sizeof(out), ctx.ptr, 0, 0);
+ return 0;
+}
+
+struct pair {
+ long a, b;
+};
+
+static __noinline void *pass_or_throw(void *p, long throw, long is_lock)
+{
+ if (!throw)
+ return p;
+ if (is_lock)
+ bpf_spin_unlock(p + 8);
+ else
+ bpf_obj_drop_impl(p, NULL);
+ asm volatile ("");
+ bpf_throw(0);
+ return NULL;
+}
+
+SEC("tc")
+__description("calls: two calls hold other things and meet in the function")
+__arch_x86_64 __arch_arm64
+__success
+int calls_hold_other_things(struct __sk_buff *skb)
+{
+ struct pair *n = bpf_obj_new(typeof(*n));
+ long is_lock = 0, throw = skb->len, val;
+ struct with_lock *e;
+ int key = 0;
+
+ if (!n)
+ return 0;
+ barrier_var(is_lock);
+ n = pass_or_throw(n, throw, is_lock);
+ bpf_obj_drop(n);
+ e = bpf_map_lookup_elem(&map_lock, &key);
+ if (!e)
+ return 0;
+ is_lock = 1;
+ barrier_var(is_lock);
+ bpf_spin_lock(&e->lock);
+ e = pass_or_throw(e, throw, is_lock);
+ val = *(volatile long *)&e->val;
+ bpf_spin_unlock(&e->lock);
+ return val;
+}
+
+/* a recursive function is verified on its own, also with an argument that can be NULL */
+static __noinline struct pair *throw_if_null(struct pair *p, long depth)
+{
+ if (!p)
+ bpf_throw(0);
+ if (depth)
+ throw_if_null(p, depth - 1);
+ barrier_var(p);
+ return p;
+}
+
+SEC("tc")
+__description("calls: the function tests a reference of the caller for NULL")
+__arch_x86_64
+__failure __msg("invalid mem access 'scalar'") __not_msg("verifier bug")
+int callee_tests_null(struct __sk_buff *skb)
+{
+ struct pair *n = bpf_obj_new(typeof(*n)), *copy;
+
+ copy = throw_if_null(n, skb->mark);
+ barrier_var(n);
+ if (!n)
+ return *(volatile long *)©->a;
+ bpf_obj_drop(n);
+ return 0;
+}
+
+static __noinline int submit(void *p)
+{
+ bpf_ringbuf_submit(p, 0);
+ return 0;
+}
+
+extern void bpf_local_irq_save(unsigned long *flags) __weak __ksym;
+extern void bpf_local_irq_restore(unsigned long *flags) __weak __ksym;
+
+SEC("tc")
+__description("calls: the function releases the first of two things that the caller holds")
+__failure __msg("invalid mem access 'scalar'")
+int callee_releases_first(void *unused)
+{
+ long *p = bpf_ringbuf_reserve(&rb, 8, 0);
+ unsigned long flags;
+
+ if (!p)
+ return 0;
+ bpf_local_irq_save(&flags);
+ submit(p);
+ *(volatile long *)p = 1;
+ bpf_local_irq_restore(&flags);
+ return 0;
+}
+
+static __naked __noinline __used void read_value(void)
+{
+ asm volatile ("r0 = *(u64 *)(r1 + 0); exit;");
+}
+
+static __naked __noinline __used void read_value_too(void)
+{
+ asm volatile ("r0 = *(u64 *)(r1 + 0); r0 += 1; exit;");
+}
+
+SEC("tc")
+__description("ids: the name of a lock that is released is in a register at callx")
+__arch_x86_64
+__success
+__naked void callx_stale_id(void)
+{
+ ASM(".pushsection .rodata,\"a\",@progbits;"
+ ".balign 8;"
+"tbl_%=: .quad read_value; .quad read_value_too;"
+ ".type tbl_%=, @object;"
+ ".size tbl_%=, .-tbl_%=;"
+ ".popsection;"
+ "call %[bpf_get_prandom_u32];"
+ "r8 = r0;"
+ "r8 &= 8;"
+ "r7 = tbl_%= ll;"
+ "r7 += r8;"
+ "r7 = *(u64 *)(r7 + 0);"
+ "r1 = 0;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "r2 = r10;"
+ "r2 += -8;"
+ "r1 = %[map_lock] ll;"
+ "call %[bpf_map_lookup_elem];"
+ "if r0 == 0 goto 9f;"
+ "r6 = r0;"
+ "r1 = r6;"
+ "r1 += 8;"
+ "call %[bpf_spin_lock];"
+ "if r8 == 0 goto 1f;"
+ "r1 = 1;"
+ "*(u64 *)(r6 + 0) = r1;"
+"1: r1 = r6;"
+ "r1 += 8;"
+ "call %[bpf_spin_unlock];"
+ "r1 = r6;"
+ "callx r7;"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void read_out(void)
+{
+ asm volatile ("r2 = %[out] ll; r2 += r1; r0 = *(u8 *)(r2 + 0); exit;" :: __imm_addr(out));
+}
+
+static __naked __noinline __used void call_read_out(void)
+{
+ asm volatile ("call read_out; exit;");
+}
+
+/* r1 is 0..7 if r7 is 0 */
+SEC("tc")
+__description("control_deps: callx leads to the unproven insn through one more call")
+__arch_x86_64
+__success
+__naked void callx_then_call(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "call %[bpf_get_prandom_u32];"
+ "r1 = r6;"
+ "r2 = call_read_out ll;"
+ "r7 = -22;"
+ "if r0 == 0 goto 1f;"
+ "r1 &= 7;"
+ "r7 = 0;"
+"1: if r7 != 0 goto 9f;"
+ "callx r2;"
+ EXIT_AT_9);
+}
+
+/* the call of a weak kfunc that is not there, as the loader leaves it */
+static const struct bpf_insn missing_kfunc = {
+ .code = 0x85, /* BPF_JMP | BPF_CALL */
+ .src_reg = 2, /* BPF_PSEUDO_KFUNC_CALL */
+};
+
+SEC("tc")
+__description("path of the join: a kfunc that is missing")
+__success __log_level(2) __msg("pass 1")
+__naked void missing_kfunc_is_not_called(void)
+{
+ asm volatile (PATH_OF_THE_JOIN
+ ".8byte %[missing_kfunc];"
+ EXIT_AT_9
+ :: OPS, __imm_insn(missing_kfunc, missing_kfunc) : __clobber_all);
+}
+
+struct big {
+ long a[64];
+};
+
+SEC("tc")
+__description("path of the join: the size of another type")
+__success __log_level(2) __msg("pass 1")
+__xlated("r1 = 512")
+__xlated("...")
+__xlated("call kernel-function")
+int size_of_other_type(void *unused)
+{
+ __u64 big = bpf_core_type_id_local(struct big), id;
+ __u64 small = bpf_core_type_id_local(struct pair);
+ __u64 rnd = bpf_get_prandom_u32(), a, b;
+ void *p;
+
+ asm volatile (
+ "%[a] = 0;"
+ "%[b] = 0;"
+ "if %[rnd] > 1000 goto +2;"
+ "%[a] = 1;"
+ "%[b] = %[rnd];"
+ "%[id] = %[big];"
+ "if %[a] != 0 goto +2;"
+ "if %[b] == 0 goto +1;"
+ "%[id] = %[small];"
+ : [id]"=&r"(id), [a]"=&r"(a), [b]"=&r"(b)
+ : [rnd]"r"(rnd), [big]"r"(big), [small]"r"(small));
+ p = bpf_obj_new_impl(id, NULL);
+ if (p)
+ bpf_obj_drop_impl(p, NULL);
+ return 0;
+}
+
+SEC("socket")
+__description("path of the join: an insn is unproven and skipped, its result is no constant")
+__success __success_unpriv
+__naked void result_of_unproven_insn(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r9 = r0;"
+ "r9 |= 1;"
+ "r9 &= 1;"
+ "r7 = 0;"
+ "if r0 > 1000 goto 1f;"
+ "r7 = 1;"
+ "r3 = r0;"
+"1: if r7 == 0 goto 9f;"
+ "r3 &= 0;"
+ "if r3 != r9 goto 9f;"
+ "r0 = r3;"
+ "exit;"
+ EXIT_AT_9);
+}
+
+/* r6 is 0..7. Pass 0 does not know that. */
+#define SMALL_R6 \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = r0;" \
+ "call %[bpf_get_prandom_u32];" \
+ "r7 = -22;" \
+ "if r6 > 7 goto +3;" \
+ "r7 = 0;" \
+ "if r0 <= 7 goto +1;" \
+ "r7 = -1;" \
+ "if r7 != 0 goto 9f;"
+
+/* If these slots are in data_deps, there are 1 << 16 states */
+#define SLOT_WITH_GAP(off) \
+ "call %[bpf_get_prandom_u32];" \
+ "r1 = r0;" \
+ "r1 &= 3;" \
+ "if r0 > 1000 goto +1;" \
+ "r1 = 100;" \
+ "*(u64 *)(r10 - " #off ") = r1;"
+
+#define EACH_SLOT(f) \
+ f(40) f(48) f(56) f(64) f(72) f(80) f(88) f(96) \
+ f(104) f(112) f(120) f(128) f(136) f(144) f(152) f(160)
+
+#define ASM_SKIP(...) \
+ asm volatile (__VA_ARGS__ :: OPS, __imm_addr(rb), \
+ __imm(bpf_dynptr_from_mem), __imm(bpf_dynptr_read), __imm(bpf_strtol), \
+ __imm(bpf_ringbuf_output), __imm(bpf_ringbuf_reserve), \
+ __imm(bpf_ringbuf_discard) : __clobber_all)
+
+#define READ_DYNPTR(off) \
+ "r1 = r10;" \
+ "r1 += -8;" \
+ "r2 = 1;" \
+ "r3 = r10;" \
+ "r3 += -24;" \
+ "r4 = *(u64 *)(r10 - " #off ");" \
+ "r5 = 0;" \
+ "call %[bpf_dynptr_read];"
+
+SEC("tc")
+__description("skip: the call does not make the dynptr, the reads are no new errors")
+__success
+__naked void skip_makes_no_dynptr(void)
+{
+ ASM_SKIP(VALUE_IN_R9
+ SMALL_R6
+ EACH_SLOT(SLOT_WITH_GAP)
+ "r1 = r9;"
+ "r2 = r6;"
+ "r3 = 0;"
+ "r4 = r10;"
+ "r4 += -24;"
+ "call %[bpf_dynptr_from_mem];"
+ EACH_SLOT(READ_DYNPTR)
+ EXIT_AT_9);
+}
+
+#define ADD_SLOT(off) "r1 = *(u64 *)(r10 - " #off "); r0 += r1;"
+
+SEC("tc")
+__description("skip: the call does not release, the exit is no new error")
+__success
+__naked void skip_does_not_release(void)
+{
+ ASM_SKIP(SMALL_R6
+ EACH_SLOT(SLOT_WITH_GAP)
+ "r1 = %[rb] ll;"
+ "r2 = 8;"
+ "r3 = 0;"
+ "call %[bpf_ringbuf_reserve];"
+ "if r0 == 0 goto 9f;"
+ "r6 >>= 3;"
+ "r1 = r0;"
+ "r1 += r6;"
+ "r2 = 0;"
+ "call %[bpf_ringbuf_discard];"
+ "r0 = 0;"
+ EACH_SLOT(ADD_SLOT)
+ "exit;"
+ EXIT_AT_9);
+}
+
+#define LOCK_VALUE_OF_KEY_R6 \
+ SMALL_R6 \
+ "r1 = %[map_lock] ll;" \
+ "r2 = r9;" \
+ "r2 += r6;" \
+ "call %[bpf_map_lookup_elem];" \
+ "if r0 == 0 goto 9f;" \
+ "r8 = r0;" \
+ "r8 += 8;" \
+ "r1 = r8;" \
+ "call %[bpf_spin_lock];" \
+ "r1 = r8;" \
+ "call %[bpf_spin_unlock];"
+
+SEC("tc")
+__description("skip: the lock is made up, the path continues to the second error")
+__success __log_level(2) __msg("pass 1") __not_msg("pass 2")
+__naked void skip_lock_is_made_up(void)
+{
+ ASM(VALUE_IN_R9 LOCK_VALUE_OF_KEY_R6 LOCK_VALUE_OF_KEY_R6 EXIT_AT_9);
+}
+
+SEC("tc")
+__description("skip: the next call can read what the call stores")
+__success
+__naked void skip_stores(void)
+{
+ ASM_SKIP(VALUE_IN_R9
+ "*(u64 *)(r10 - 32) = 0;"
+ SMALL_R6
+ EACH_SLOT(SLOT_WITH_GAP)
+ "r1 = r9;"
+ "r2 = r6;"
+ "r2 += 1;"
+ "r3 = 0;"
+ "r4 = r10;"
+ "r4 += -32;"
+ "call %[bpf_strtol];"
+ "r1 = %[rb] ll;"
+ "r2 = r10;"
+ "r2 += -160;"
+ "r3 = 136;"
+ "r4 = 0;"
+ "call %[bpf_ringbuf_output];"
+ EXIT_AT_9);
+}
+
+/* The test of r7 gives a smaller trip count, but r7 can step over r8 */
+SEC("tc")
+__description("trip count: the first guess is wrong, the second one is proven")
+__success
+__xlated("if r7 != r8 goto")
+__xlated("r0 = 0")
+__naked void second_guess(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r8 = r0;"
+ "r8 &= 0x3fffff;"
+ "r6 = 0;"
+ "r7 = 0;"
+ "r5 = 0;"
+"1: r2 = r5;"
+ "r2 >>= 20;"
+ "r1 = r9;"
+ "r1 += r2;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r5 += 3;"
+ "r6 += 1;"
+ "if r6 >= 5000000 goto 9f;"
+ "r7 += 2;"
+ "if r7 != r8 goto 1b;"
+ EXIT_AT_9);
+}
+
+/* r6 grows in some trips and stops at @bound + 1. r7 has a larger step. */
+#define APPEND(bound, shift) \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1: call %[bpf_get_prandom_u32];" \
+ "if r0 == 1 goto 3f;" \
+ "r7 += 13;" \
+ "if r0 & 1 goto 2f;" \
+ "if r6 > " #bound " goto 2f;" \
+ "r6 += 1;" \
+"2: goto 1b;" \
+"3: r6 >>= " #shift ";" \
+ "r9 += r6;" \
+ "r0 = *(u8 *)(r9 + 0);" \
+ "r0 += r7;" \
+ "exit;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("threshold: another number has a larger step")
+__success
+__naked void step_of_other_number(void)
+{
+ ASM(APPEND(0xffffe, 14));
+}
+
+SEC("tc")
+__description("threshold: the number goes one step too far")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void one_step_too_far(void)
+{
+ ASM(APPEND(63, 0));
+}
+
+static __naked __noinline __used int next_byte_cb(void)
+{
+ asm volatile (
+ "r3 = *(u64 *)(r2 + 0);"
+ "r1 = %[out] ll;"
+ "r1 += r3;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r3 += 1;"
+ "*(u64 *)(r2 + 0) = r3;"
+ "r0 = 0;"
+ "exit;"
+ :: __imm_addr(out) : __clobber_all);
+}
+
+#define LOOP_ARGS \
+ "*(u64 *)(r10 - 8) = 0;" \
+ "r1 = 3;" \
+ "r2 = next_byte_cb ll;" \
+ "r3 = r10;" \
+ "r3 += -8;" \
+ "r4 = 0;"
+
+SEC("tc")
+__description("peel: the header of a loop calls a callback")
+__success
+__naked void header_calls_callback(void)
+{
+ ASM(LOOP_ARGS
+"1: call %[bpf_loop];"
+ "call %[bpf_get_prandom_u32];"
+ "if w0 & 1 goto 9f;"
+ LOOP_ARGS
+ "goto 1b;"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("failing_path: the pointer to the context moves in a loop")
+__failure __msg("dereference of modified ctx ptr R6 off=4 disallowed")
+__naked void ctx_moves(void)
+{
+ ASM("r6 = r1;"
+"1: call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 2f;"
+ "r6 += 4;"
+ "goto 1b;"
+"2: r0 = *(u32 *)(r6 + 0);"
+ EXIT_AT_9);
+}
+
+/* The verifier can't tell that the loop ends. It adds may_goto, which leaves with r7 = 0. */
+SEC("tc")
+__description("failing_path: the path continues where may_goto leaves an infinite loop")
+__failure __msg("R7 invalid mem access 'scalar'")
+__naked void exit_of_infinite_loop(void)
+{
+ ASM(VALUE_IN_R9
+ "r1 = r9;"
+ "call set_when_done;"
+ "r7 = r0;"
+ "r0 = *(u8 *)(r7 + 0);"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void set_when_done(void)
+{
+ ASM("r6 = r1;"
+ "r7 = 0;"
+"1: r1 = *(u8 *)(r6 + 0);"
+ "if r7 != 0 goto 2f;"
+ "if r1 != 0 goto 1b;"
+ "r7 = r6;"
+ "goto 1b;"
+"2: r0 = r7;"
+ "exit;");
+}
+
+SEC("tc")
+__description("failing_path: the log is one path from the start to the error")
+__failure
+__msg("0: (b7) r1 = 0")
+__msg("r7 = 64")
+__not_msg("r7 = 1 ")
+__msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void log_is_one_path(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 1f;"
+ "r7 = 64;"
+ "goto 2f;"
+"1: r7 = 1;"
+"2: r9 += r7;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+/* The step of r7 is not a constant */
+#define NO_CONSTANT_STEP(op, trips, result) \
+ ASM(VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1: r7 " #op " r6;" \
+ "r6 += 1;" \
+ "if r6 < " #trips " goto 1b;" \
+ "r7 -= " #result ";" \
+ "r9 += r7;" \
+ "r0 = *(u8 *)(r9 + 0);" \
+ EXIT_AT_9)
+
+SEC("tc")
+__description("failing_path: a guess skips the trips of a sum")
+__success __retval(0)
+__naked void guess_skips_sum(void)
+{
+ NO_CONSTANT_STEP(+=, 1000, 499500);
+}
+
+SEC("tc")
+__description("failing_path: a guess skips the trips of xor")
+__success __retval(0)
+__naked void guess_skips_xor(void)
+{
+ NO_CONSTANT_STEP(^=, 4000, 0);
+}
+
+SEC("tc")
+__description("failing_path: a pass peels the trips of a sum, the log has its error")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void real_error_replaces_guess(void)
+{
+ NO_CONSTANT_STEP(+=, 1000, 499436);
+}
+
+/*
+ * In the first trips r7 moves by 1. From trip 4 on its range grows, so these trips cannot be
+ * skipped. r7 is 60..117 after the loop. The second loop needs more passes.
+ */
+SEC("tc")
+__description("failing_path: a number has a step only in the first trips")
+__success __retval(0)
+__naked void step_in_first_trips(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = 0;"
+ "r7 = r0;"
+ "r7 &= 1;"
+ "r8 = 0;"
+"1: call %[bpf_get_prandom_u32];"
+ ".rept 8;"
+ "r1 = 0;"
+ ".endr;"
+ "if r8 >= 4 goto 2f;"
+ "r8 += 1;"
+ "r0 = 0;"
+"2: r0 &= 1;"
+ "r7 += r0;"
+ "r7 += 1;"
+ "r6 += 1;"
+ "if r6 < 60 goto 1b;"
+ "r7 -= 60;"
+ "r1 = r9;"
+ "r1 += r7;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r6 = 0;"
+ "r7 = 0;"
+"3: r7 += r6;"
+ "r6 += 1;"
+ "if r6 < 1000 goto 3b;"
+ "r7 -= 499500;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+/* for (i = 1; i < 1000000 / i; i++); */
+SEC("tc")
+__description("failing_path: the bound of the loop changes in each trip")
+__success __retval(0)
+__naked void guess_bound_changes(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+"1: r6 += 1;"
+ "r7 = 1000000;"
+ "r7 /= r6;"
+ "if r6 < r7 goto 1b;"
+ "r6 -= 1000;"
+ "r9 += r6;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+/*
+ * The first loop goes from 600 through 1023 to 88. Preemption is off, so may_goto goes to the
+ * load. The second loop needs more passes.
+ */
+SEC("tc")
+__description("failing_path: a ring index looks like an infinite loop")
+__success __retval(0)
+__naked void guess_loop_is_infinite(void)
+{
+ asm volatile (VALUE_IN_R9
+ "call %[bpf_preempt_disable];"
+ "r6 = 600;"
+"1: r6 += 1;"
+ "r6 &= 1023;"
+ "if r6 != 88 goto 1b;"
+ "r6 -= 88;"
+ "r9 += r6;"
+ "r0 = *(u8 *)(r9 + 0);"
+ "call %[bpf_preempt_enable];"
+ "r6 = 0;"
+ "r7 = 0;"
+"2: r7 += r6;"
+ "r6 += 1;"
+ "if r6 < 5000 goto 2b;"
+ "r7 -= 12497500;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9
+ :: OPS, __imm(bpf_preempt_disable), __imm(bpf_preempt_enable) : __clobber_all);
+}
+
+/* The 0 goes from r5 to r8 in five trips. The second loop needs more passes. */
+SEC("tc")
+__description("failing_path: a copy moves one time, that is no step")
+__success __retval(0)
+__naked void guess_copy_moves_once(void)
+{
+ ASM(VALUE_IN_R9
+ "r1 = 0;"
+ "r2 = 100;"
+ "r3 = 100;"
+ "r4 = 100;"
+ "r5 = 100;"
+ "r6 = 0;"
+"1: *(u8 *)(r9 + 0) = r1;"
+ "r8 = r2;"
+ "r2 = r3;"
+ "r3 = r4;"
+ "r4 = r5;"
+ "r5 = 0;"
+ ".rept 4;"
+ "r1 = 0;"
+ ".endr;"
+ "r1 = r6;"
+ "r6 += 1;"
+ "if r6 < 60 goto 1b;"
+ "r1 = r9;"
+ "r1 += r8;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r6 = 0;"
+ "r7 = 0;"
+"3: r7 += r6;"
+ "r6 += 1;"
+ "if r6 < 5000 goto 3b;"
+ "r7 -= 12497500;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("failing_path: a copy of the iv has a constant step, fast reject")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1") FEW_INSNS
+__naked void copy_of_iv_is_no_guess(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+ "r7 = 0;"
+"1: call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 2f;"
+ "if r7 >= 8 goto 2f;"
+ "r7 += 1;"
+"2: r6 += 1;"
+ "r1 = r6;"
+ "r1 <<= 32;"
+ "r1 >>= 32;"
+ "r2 = 1000000;"
+ "if r2 > r1 goto 1b;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 56);"
+ EXIT_AT_9);
+}
+
+/*
+ * r7 is 1 in each execution. The skip of the sum is a guess, and it makes r7 = 0. On that path
+ * the loop at 2 is infinite. The next run leaves that loop with r7 = 1 and makes no skip.
+ */
+SEC("tc")
+__description("failing_path: a path with a guess finds an infinite loop")
+__success __retval(0)
+__naked void guess_before_infinite_loop(void)
+{
+ asm volatile (VALUE_IN_R9
+ "call %[bpf_preempt_disable];"
+ "call %[bpf_get_prandom_u32];"
+ "r7 = 1;"
+ "if w0 & 1 goto 3f;"
+"1: r6 = 100;"
+"2: r6 -= r7;"
+ "if r6 != 0 goto 2b;"
+ "if r7 == 0 goto 8f;"
+ "r9 += r6;"
+ "r0 = *(u8 *)(r9 + 0);"
+"8: call %[bpf_preempt_enable];"
+ "goto 9f;"
+"3: r6 = 0;"
+ "r8 = 0;"
+"4: r8 += r6;"
+ "r6 += 1;"
+ "if r6 < 100 goto 4b;"
+ "if r8 == 4950 goto 5f;"
+ "r7 = 0;"
+"5: goto 1b;"
+ EXIT_AT_9
+ :: OPS, __imm(bpf_preempt_disable), __imm(bpf_preempt_enable) : __clobber_all);
+}
+
+SEC("tc")
+__description("failing_path: a spilled iv has a constant step, fast reject")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1") FEW_INSNS
+__naked void spilled_iv_is_no_guess(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = 0;"
+ "*(u64 *)(r10 - 16) = r7;"
+"1: call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 2f;"
+ "if r7 >= 8 goto 2f;"
+ "r7 += 1;"
+"2: r1 = *(u64 *)(r10 - 16);"
+ "r1 += 1;"
+ "*(u64 *)(r10 - 16) = r1;"
+ "if r1 < 1000000 goto 1b;"
+ "r9 += r7;"
+ "r0 = *(u8 *)(r9 + 56);"
+ EXIT_AT_9);
+}
+
+/*
+ * With @bound 7 no execution disables preemption. The joined state does, and then may_goto of
+ * the loop at 3 cannot return 0. It goes to the load, with r0 = NULL.
+ * With @bound 8 an execution with r6 == 8 disables preemption, so the error is real.
+ */
+#define GUARD_OF_THE_JOIN(bound) \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = r0;" \
+ "r7 = 0;" \
+ "if r6 <= 7 goto 1f;" \
+ "r7 = 1;" \
+"1: r9 = 0;" \
+"2: call %[bpf_get_prandom_u32];" \
+ "if r0 > 9 goto 5f;" \
+"3: r1 = 0;" \
+ "*(u64 *)(r10 - 8) = r1;" \
+ "r1 = %[map64] ll;" \
+ "r2 = r10;" \
+ "r2 += -8;" \
+ "call %[bpf_map_lookup_elem];" \
+ "if r0 == 0 goto 3b;" \
+ "r9 = *(u64 *)(r0 + 0);" \
+"4: call %[bpf_get_prandom_u32];" \
+ "if r0 != 5 goto 4b;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 == 1 goto 6f;" \
+ "goto 2b;" \
+"5: if r7 == 0 goto 2b;" \
+ "if r6 > " #bound " goto 2b;" \
+ "call %[bpf_preempt_disable];" \
+ "goto 2b;" \
+"6: call %[bpf_get_prandom_u32];" \
+ "if r0 != 3 goto 1b;" \
+ "r0 = 0;" \
+ "exit;" \
+ :: OPS, __imm(bpf_preempt_disable) : __clobber_all)
+
+SEC("tc")
+__description("failing_path: a joined state selects the target of may_goto")
+__success
+__naked void guess_takes_target_of_join(void)
+{
+ GUARD_OF_THE_JOIN(7);
+}
+
+SEC("tc")
+__description("failing_path: a real state selects the target of may_goto")
+__failure __msg("R0 invalid mem access 'map_value_or_null'") __msg("this path has a guess")
+__naked void guess_takes_real_target(void)
+{
+ GUARD_OF_THE_JOIN(8);
+}
+
+SEC("tc")
+__description("failing_path: level 2 marks a path with a guess")
+__failure __log_level(2) __msg("path-sensitive run after pass 0")
+__msg("this path has a guess") __msg("pass 1")
+__naked void guess_in_level_2(void)
+{
+ GUARD_OF_THE_JOIN(8);
+}
+
+char eight[8] SEC(".data.eight");
+
+static __naked __noinline __used void read_eight(void)
+{
+ asm volatile ("r2 = %[eight] ll; r2 += r1; r0 = *(u8 *)(r2 + 0); exit;"
+ :: __imm_addr(eight) : __clobber_all);
+}
+
+/*
+ * Trip 3 reads eight[8]. At its start the run guesses that the loop is infinite. The call after
+ * the loop reads eight[0]. It fails only on the edge of may_goto.
+ */
+SEC("tc")
+__description("failing_path: the log has the path that was found")
+__failure __msg("invalid access to map value, value_size=8 off=8 size=1")
+__not_msg("this path has a guess")
+__naked void log_repeats_the_path(void)
+{
+ asm volatile (
+ "call %[bpf_preempt_disable];"
+ ".rept 10;"
+ "r0 = 0;"
+ ".endr;"
+ "r6 = 600;"
+"1: .rept 10;"
+ "r0 = 0;"
+ ".endr;"
+ "r1 = r6;"
+ "r1 += -595;"
+ "call read_eight;"
+ "r6 += 1;"
+ "r6 &= 1023;"
+ "if r6 != 88 goto 1b;"
+ "r1 = r6;"
+ "r1 += -88;"
+ "call read_eight;"
+ "call %[bpf_preempt_enable];"
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_preempt_disable), __imm(bpf_preempt_enable) : __clobber_all);
+}
+
+__noinline __used int global_reads_above_stack(int x)
+{
+ asm volatile ("if r1 > 5 goto +1; r0 = *(u64 *)(r10 + 8);" ::: "r0");
+ return x;
+}
+
+/* The path in the global function is short. The path to the call is long. */
+SEC("tc")
+__description("failing_path: the repeat for the log has the same budget")
+__failure __msg("if r1 > 0x5 goto pc+1") __msg("invalid read from stack R10 off=8 size=8")
+__naked void log_has_the_budget(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r6 = 1;"
+ "if r0 == 1 goto +1;"
+ "r6 = 3;"
+ ".rept 300;"
+ "r0 = 0;"
+ ".endr;"
+ "if r6 == 2 goto +0;"
+ ".rept 300;"
+ "r0 = 0;"
+ ".endr;"
+ "r1 = r6;"
+ "call global_reads_above_stack;"
+ "exit;");
+}
+
+/* r7 is a copy of r6. No run has an odd r6 and r7 == 0. */
+SEC("tc")
+__description("infeasible: a jump refines a copy, the copy decides the next jump")
+__success
+__naked void infeasible_path(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r6 &= 7;"
+ "r7 = r6;"
+ "r8 = 0;"
+ "if r0 == 0 goto 1f;"
+ "r8 = 8;"
+"1: if r6 & 1 goto 2f;"
+ "goto 9f;"
+"2: if r7 > 0 goto 9f;"
+ "r0 = *(u8 *)(r9 + 64);"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("monotone: a pointer far out of bounds is made, it is not used")
+__success
+__naked void far_pointer_is_not_used(void)
+{
+ ASM(VALUE_IN_R9
+ "r1 = 0x7fffffff;"
+ "r8 = r9;"
+ "r8 -= r1;"
+ "r9 += r1;"
+ EXIT_AT_9);
+}
+
+/* r1 = 48 gets to the add on the speculative path only. The barrier makes the mask unnecessary. */
+SEC("socket")
+__description("speculation: no mask fits the real paths, a speculative path puts a barrier")
+__success __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("if r1 s>= 0x7 goto")
+__xlated_unpriv("nospec")
+__xlated_unpriv("r2 += r1")
+#endif
+__naked void nospec_for_all_paths(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = *(u64 *)(r9 + 0);"
+ "r2 = r9;"
+ "r2 += 16;"
+ "r1 = 48;"
+ "if r7 == 0 goto 1f;"
+ "r1 = -16;"
+ "if r7 == 3 goto 1f;"
+ "r1 = 8;"
+ "if r7 == 7 goto 1f;"
+ "r1 = 0;"
+"1: if r1 s>= 7 goto 9f;"
+ "r2 += r1;"
+ "r0 = *(u8 *)(r2 + 0);"
+ EXIT_AT_9);
+}
+
+extern void bpf_preempt_disable(void) __weak __ksym;
+extern void bpf_preempt_enable(void) __weak __ksym;
+
+static __noinline void disable_if(long yes)
+{
+ if (yes)
+ bpf_preempt_disable();
+ asm volatile ("");
+}
+
+SEC("tc")
+__description("instances: what a function acquires goes with the arguments of the call")
+__success
+int acquires_by_argument(void *unused)
+{
+ long yes = 1, no = 0;
+
+ barrier_var(yes);
+ barrier_var(no);
+ disable_if(no);
+ disable_if(yes);
+ bpf_preempt_enable();
+ return 0;
+}
+
+SEC("tc")
+__description("peel: a loop acquires three references in any order")
+__success
+int three_refs_in_any_order(struct __sk_buff *skb)
+{
+ void *first = NULL, *second = NULL, *third = NULL;
+ int i;
+
+ bpf_for(i, 0, skb->len) {
+ if (!first)
+ first = bpf_ringbuf_reserve(&rb, 8, 0);
+ if (!second)
+ second = bpf_ringbuf_reserve(&rb, 8, 0);
+ if (!third)
+ third = bpf_ringbuf_reserve(&rb, 8, 0);
+ }
+ if (first)
+ bpf_ringbuf_discard(first, 0);
+ if (second)
+ bpf_ringbuf_discard(second, 0);
+ if (third)
+ bpf_ringbuf_discard(third, 0);
+ return 0;
+}
+
+struct list_elem {
+ struct bpf_list_node link;
+};
+
+#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
+private(list) struct bpf_spin_lock list_lock;
+private(list) struct bpf_list_head list __contains(list_elem, link);
+
+static __noinline void *pass_on(void *p)
+{
+ barrier_var(p);
+ return p;
+}
+
+SEC("tc")
+__description("ids: two calls hold objects of two kinds under one name")
+__failure __msg("R2 type=ringbuf_mem expected=ptr_")
+int two_types_one_id(void *unused)
+{
+ struct list_elem *n = bpf_obj_new(typeof(*n));
+ char *rec;
+
+ if (!n)
+ return 0;
+ bpf_obj_drop_impl(pass_on(n), NULL);
+ rec = bpf_ringbuf_reserve(&rb, 8, 0);
+ if (!rec)
+ return 0;
+ n = pass_on(rec);
+ bpf_spin_lock(&list_lock);
+ bpf_list_push_back(&list, &n->link);
+ bpf_spin_unlock(&list_lock);
+ return *(volatile char *)rec;
+}
+
+/* r9 moves with r6. Paths of other lengths come to one state. */
+SEC("tc")
+__description("peel: a loop with 'continue'")
+__success FEW_INSNS
+__naked void loop_with_continue(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+"1:" MAY_GOTO(7)
+ "if r6 >= 64 goto 9f;"
+ "*(u8 *)(r9 + 0) = 0;"
+ "call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 1b;"
+ "r9 += 1;"
+ "r6 += 1;"
+ "goto 1b;"
+ EXIT_AT_9);
+}
+
+/* r1 is the low 16 bits of r8. r8 is above 32 and r1 is not: no run goes around again. */
+SEC("tc")
+__description("failing_path: a linked register has no value left")
+__success
+__naked void linked_is_empty(void)
+{
+ ASM(VALUE_IN_R9
+ "r8 = 0;"
+ "r7 = 0;"
+ "r6 = 20;"
+"1: if r7 != 0 goto 2f;"
+ "r1 = r9;"
+ "r1 += r8;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "call %[bpf_get_prandom_u32];"
+ "w0 &= 7;"
+ "w8 += w0;"
+ "w1 = w8;"
+ "w1 &= 65535;"
+ "r7 = 1;"
+ "if w1 > 32 goto 2f;"
+ "r7 = 0;"
+"2: r6 += -1;"
+ "if r6 != 0 goto 1b;"
+ EXIT_AT_9);
+}
+
+/* BTF FUNC records are not generated for kfuncs that only inline assembly calls */
+void __kfunc_btf_root(void)
+{
+ bpf_iter_num_new(0, 0, 0);
+ bpf_iter_num_next(0);
+ bpf_iter_num_destroy(0);
+ __bpf_trap();
+}
+
+/* 0 on each run. The join has 0 or 1. */
+static __always_inline long never(void)
+{
+ long rnd = bpf_get_prandom_u32(), x, y, ret;
+
+ asm volatile (
+ "%[x] = 0;"
+ "%[y] = 0;"
+ "if %[rnd] & 1 goto +2;"
+ "%[x] = 1;"
+ "%[y] = %[rnd];"
+ "%[ret] = 0;"
+ "if %[x] != 0 goto +2;"
+ "if %[y] == 0 goto +1;"
+ "%[ret] = 1;"
+ : [x]"=&r"(x), [y]"=&r"(y), [ret]"=&r"(ret) : [rnd]"r"(rnd));
+ return ret;
+}
+
+#define CALLS_TWICE(name, next) \
+ static __noinline long name(long n) \
+ { \
+ long a = next(n); \
+ \
+ barrier_var(n); \
+ return a + next(n); \
+ }
+
+static __noinline long leaf(long n)
+{
+ barrier_var(n);
+ return n & 3;
+}
+
+CALLS_TWICE(twice_1, leaf) CALLS_TWICE(twice_2, twice_1) CALLS_TWICE(twice_3, twice_2)
+CALLS_TWICE(twice_4, twice_3) CALLS_TWICE(twice_5, twice_4) CALLS_TWICE(twice_6, twice_5)
+CALLS_TWICE(twice_7, twice_6) CALLS_TWICE(twice_8, twice_7) CALLS_TWICE(twice_9, twice_8)
+CALLS_TWICE(twice_a, twice_9) CALLS_TWICE(twice_b, twice_a) CALLS_TWICE(twice_c, twice_b)
+CALLS_TWICE(twice_d, twice_c) CALLS_TWICE(twice_e, twice_d) CALLS_TWICE(twice_f, twice_e)
+CALLS_TWICE(twice_g, twice_f) CALLS_TWICE(twice_h, twice_g) CALLS_TWICE(twice_i, twice_h)
+
+SEC("tc")
+__description("instances: equal values share one, 1 << 18 chains of calls")
+__success FEW_INSNS
+int calls_share_instance(struct __sk_buff *skb)
+{
+ long yes = 1, no = 0, ret = twice_i(skb->len);
+
+ barrier_var(yes);
+ barrier_var(no);
+ disable_if(no);
+ disable_if(yes);
+ bpf_preempt_enable();
+ return ret;
+}
+
+static __noinline void throw_if(long yes)
+{
+ if (yes)
+ bpf_throw(0);
+ asm volatile ("");
+}
+
+SEC("tc")
+__description("instances: the second call does not return")
+__arch_x86_64 __arch_arm64
+__success
+__naked void call_does_not_return(void)
+{
+ asm volatile (
+ "r1 = 0;"
+ "call %[throw_if];"
+ "r1 = 1;"
+ "call %[throw_if];"
+ "r0 = *(u64 *)(r9 + 0);"
+ "exit;"
+ :: __imm(throw_if) : __clobber_all);
+}
+
+static __noinline long next_of(long n)
+{
+ barrier_var(n);
+ return n + 1;
+}
+
+SEC("tc")
+__description("instances: the result of a call goes into the next call")
+__success FEW_INSNS
+int result_is_argument(void *unused)
+{
+ return out[next_of(next_of(next_of(0)))];
+}
+
+char bytes[1 << 16];
+
+static __noinline long byte_at(unsigned long off)
+{
+ return bytes[off];
+}
+
+#define CALLS_HALVES(name, next, half) \
+ static __noinline long name(unsigned long off) \
+ { \
+ long a = next(off); \
+ \
+ barrier_var(off); \
+ return a + next(off + (half)); \
+ }
+
+CALLS_HALVES(half_0, byte_at, 1 << 0) CALLS_HALVES(half_1, half_0, 1 << 1)
+CALLS_HALVES(half_2, half_1, 1 << 2) CALLS_HALVES(half_3, half_2, 1 << 3)
+CALLS_HALVES(half_4, half_3, 1 << 4) CALLS_HALVES(half_5, half_4, 1 << 5)
+CALLS_HALVES(half_6, half_5, 1 << 6) CALLS_HALVES(half_7, half_6, 1 << 7)
+CALLS_HALVES(half_8, half_7, 1 << 8) CALLS_HALVES(half_9, half_8, 1 << 9)
+CALLS_HALVES(half_a, half_9, 1 << 10) CALLS_HALVES(half_b, half_a, 1 << 11)
+CALLS_HALVES(half_c, half_b, 1 << 12) CALLS_HALVES(half_d, half_c, 1 << 13)
+CALLS_HALVES(half_e, half_d, 1 << 14) CALLS_HALVES(half_f, half_e, 1 << 15)
+
+SEC("tc")
+__description("instances: a call continues with the old exits, 1 << 16 values are joined")
+__success INSNS(3)
+int calls_are_joined(void *unused)
+{
+ return half_f(0);
+}
+
+SEC("tc")
+__description("instances: the last value is out of bounds")
+__failure __msg("invalid access to map value")
+int calls_are_joined_bad(void *unused)
+{
+ long off = sizeof(bytes) - 8;
+
+ barrier_var(off);
+ return half_3(off);
+}
+
+SEC("tc")
+__description("instances: 1 << 15 values in data_deps are joined over the budget")
+__success FEW_INSNS
+int calls_joined_over_budget(void *unused)
+{
+ unsigned long off = 0;
+ long sum = 0;
+ int i;
+
+ for (i = 0; i < 8; i++) {
+ off += i;
+ barrier_var(i);
+ barrier_var(off);
+ sum += half_e(off);
+ }
+ return sum;
+}
+
+static __noinline long byte_slowly(unsigned long off, long slow)
+{
+ long ret = bytes[off];
+
+ if (slow)
+ asm volatile (".rept 30000; r3 = 0; .endr;" ::: "r3");
+ return ret;
+}
+
+SEC("tc")
+__description("instances: the join starts a new instance, it counts for the budget")
+__success
+int grown_instance_has_cost(void *unused)
+{
+ long sum = 0, no = 0, yes = 1;
+ unsigned long off = 0;
+ int i;
+
+ for (i = 0; i < 300; i++) {
+ barrier_var(no);
+ barrier_var(yes);
+ sum += byte_slowly(off, no);
+ sum += byte_slowly(off, yes);
+ off += i;
+ }
+ return sum;
+}
+
+static __noinline unsigned long twice_of(unsigned long n)
+{
+ barrier_var(n);
+ return 2 * n;
+}
+
+SEC("tc")
+__description("instances: the new exits have more than the old ones")
+__failure __msg("invalid access to map value")
+int exits_grow(void *unused)
+{
+ unsigned long off = twice_of(1);
+
+ barrier_var(off);
+ return bytes[twice_of(40000)];
+}
+
+static __noinline void *same_object(void *p)
+{
+ barrier_var(p);
+ return p;
+}
+
+#define RELEASES_BY_RESULT(second) \
+ void *a = bpf_ringbuf_reserve(&rb, 8, 0), *b; \
+ \
+ if (!a) \
+ return 0; \
+ b = bpf_ringbuf_reserve(&rb, 8, 0); \
+ if (!b) { \
+ bpf_ringbuf_discard(a, 0); \
+ return 0; \
+ } \
+ same_object(a); \
+ bpf_ringbuf_discard(same_object(b), 0); \
+ bpf_ringbuf_discard(second, 0); \
+ return 0
+
+SEC("tc")
+__description("instances: the old exits name another object")
+__success
+int exits_name_another_object(void *unused)
+{
+ RELEASES_BY_RESULT(a);
+}
+
+SEC("tc")
+__description("instances: the old exits name another object, double release")
+__failure __msg("R1 type=scalar expected=ringbuf_mem")
+int exits_name_another_object_bad(void *unused)
+{
+ RELEASES_BY_RESULT(b);
+}
+
+static __noinline void unlock_if_large(long n)
+{
+ if (n > 4)
+ bpf_preempt_enable();
+}
+
+SEC("tc")
+__description("instances: only the new instance releases")
+__failure __msg("unmatched attempt to enable preemption")
+int only_new_instance_releases(void *unused)
+{
+ bpf_preempt_disable();
+ unlock_if_large(1);
+ unlock_if_large(2);
+ unlock_if_large(7);
+ bpf_preempt_enable();
+ return 0;
+}
+
+/* r7 is 0 if r6 is 1. r9 has 1 << 24 values, no check depends on it. */
+SEC("tc")
+__description("instances: an error after a call does not make the call unproven")
+__success FEW_INSNS
+__naked void error_after_call(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r8 = r0;"
+ "r1 = r0;"
+ "call same;"
+ "r6 = 0; r7 = 5;"
+ "if r8 == 9 goto 1f;"
+ "r6 = 2; r7 = 7;"
+ "if r8 == 11 goto 1f;"
+ "r6 = 1; r7 = 0;"
+"1: r9 = 0;"
+ "r5 = 3;"
+ ".rept 24;"
+ "r8 >>= 1;"
+ "r5 *= 3;"
+ "r5 += 1;"
+ "if r8 & 1 goto +1;"
+ "r9 += r5;"
+ ".endr;"
+ "if r6 != 1 goto 9f;"
+ "if r7 == 0 goto 9f;"
+ "r1 = r9;"
+ "call same;"
+"2: goto 2b;"
+ EXIT_AT_9);
+}
+
+SEC("tc")
+__description("instances: the header of a loop stays while a call waits")
+__success FEW_INSNS
+int call_in_counted_loop(void *unused)
+{
+ unsigned long off = 0;
+ long sum = 0;
+ int i;
+
+ for (i = 0; i < 100000; i++) {
+ off = (off + 1) & 0xffff;
+ barrier_var(off);
+ sum += byte_at(off);
+ }
+ return sum;
+}
+
+static __noinline long byte_behind(unsigned long off)
+{
+ return bytes[off + 1];
+}
+
+SEC("tc")
+__description("instances: a call that waits holds back the next trip")
+__success FEW_INSNS
+int calls_in_peeled_trips(void *unused)
+{
+ unsigned long off = 0;
+ long sum = 0;
+ int i;
+
+ for (i = 0; i < 300; i++) {
+ if (bpf_get_prandom_u32() & 1)
+ sum += byte_at(off);
+ else
+ sum -= byte_behind(off);
+ off += i;
+ }
+ return sum;
+}
+
+#define LEVEL_OF(name, bound) \
+ static __noinline unsigned long name##_of(unsigned long n) \
+ { \
+ barrier_var(n); \
+ return n > bound ? 2 : 0; \
+ } \
+ static __noinline unsigned long name(unsigned long n) \
+ { \
+ unsigned long ret = name##_of(n + 1); \
+ \
+ barrier_var(ret); \
+ return ret; \
+ }
+
+LEVEL_OF(level_a, 50) LEVEL_OF(level_b, 60) LEVEL_OF(level_c, 70) LEVEL_OF(level_d, 80)
+
+SEC("tc")
+__description("instances: no late exit in a trip")
+__success FEW_INSNS
+int exits_grow_in_peeled_trips(void *unused)
+{
+ unsigned long off = 0;
+ long sum = 0;
+ int i;
+
+ for (i = 0; i < 500; i++) {
+ off += level_a(i) + level_b(i) + level_c(i) + level_d(i);
+ barrier_var(off);
+ sum += bytes[off];
+ }
+ return sum;
+}
+
+static __noinline void record_if(long want)
+{
+ long mode = want ? 2 : 0;
+ void *p = NULL;
+
+ barrier_var(mode);
+ if (want)
+ p = bpf_ringbuf_reserve(&rb, 8, 0);
+ barrier_var(p);
+ if (mode && p)
+ bpf_ringbuf_discard(p, 0);
+}
+
+SEC("tc")
+__description("held: a join in the callee makes up a leak")
+__success
+int leak_made_up_in_callee(struct __sk_buff *skb)
+{
+ record_if(skb->len & 1);
+ return 0;
+}
+
+SEC("tc")
+__description("held: a join in a loop makes up a leak")
+__success FEW_INSNS
+int leak_made_up_in_loop(struct __sk_buff *skb)
+{
+ while (can_loop)
+ record_if(bpf_get_prandom_u32() & 1);
+ return 0;
+}
+
+SEC("tc")
+__description("held: a loop acquires and does not release")
+__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable")
+int loop_only_acquires(void *unused)
+{
+ while (can_loop)
+ bpf_preempt_disable();
+ return 0;
+}
+
+#define KEEPS_TWO(release_last) \
+ void *prev = NULL, *cur = NULL; \
+ \
+ while (can_loop) { \
+ if (prev) \
+ bpf_ringbuf_discard(prev, 0); \
+ prev = cur; \
+ cur = bpf_ringbuf_reserve(&rb, 8, 0); \
+ } \
+ if (prev) \
+ bpf_ringbuf_discard(prev, 0); \
+ if (cur && release_last) \
+ bpf_ringbuf_discard(cur, 0); \
+ return 0
+
+SEC("tc")
+__description("held: one insn acquires, two xregs hold")
+__success FEW_INSNS
+int loop_keeps_two(void *unused)
+{
+ KEEPS_TWO(true);
+}
+
+SEC("tc")
+__description("held: one insn acquires, two xregs hold, one leaks")
+__failure __msg("Unreleased reference")
+int loop_keeps_two_bad(void *unused)
+{
+ KEEPS_TWO(false);
+}
+
+SEC("tc")
+__description("held: a reference that is in no xreg")
+__failure __msg("Unreleased reference") FEW_INSNS
+int loop_drops_the_pointer(void *unused)
+{
+ void *cur = NULL;
+
+ while (can_loop)
+ cur = bpf_ringbuf_reserve(&rb, 8, 0);
+ if (cur)
+ bpf_ringbuf_discard(cur, 0);
+ return 0;
+}
+
+static __noinline void acquire_on_the_way_back(long n)
+{
+ if (!n)
+ return;
+ acquire_on_the_way_back(n - 1);
+ bpf_preempt_disable();
+}
+
+SEC("tc")
+__description("held: a recursion acquires after the inner call")
+__arch_x86_64
+__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable")
+int recursion_acquires(struct __sk_buff *skb)
+{
+ acquire_on_the_way_back(skb->len);
+ return 0;
+}
+
+static __noinline long pong(long depth);
+
+static __noinline long ping(long depth)
+{
+ long ret;
+
+ bpf_preempt_disable();
+ ret = pong(depth);
+ bpf_preempt_enable();
+ return ret;
+}
+
+static __noinline long pong(long depth)
+{
+ long ret = depth ? ping(depth - 1) : 0;
+
+ barrier_var(ret);
+ return ret + 1;
+}
+
+SEC("tc")
+__description("held: two functions are one recursion")
+__arch_x86_64
+__success
+int recursion_of_two(void *unused)
+{
+ return ping(3);
+}
+
+static __noinline long count_down(long n)
+{
+ long ret = n ? count_down(n - 1) : 0;
+
+ barrier_var(ret);
+ return ret ^ n;
+}
+
+static __noinline long walk_down(long depth)
+{
+ long ret;
+
+ bpf_preempt_disable();
+ ret = count_down(bpf_get_prandom_u32());
+ if (depth)
+ ret += walk_down(depth - 1);
+ bpf_preempt_enable();
+ return ret;
+}
+
+SEC("tc")
+__description("held: a recursion calls another one")
+__arch_x86_64
+__success
+int recursion_in_recursion(void *unused)
+{
+ return walk_down(3);
+}
+
+#define CALLS_OTHER(name, other) \
+ static __naked __noinline __used void name(void) \
+ { \
+ ASM("r0 = 0;" \
+ "if r1 == 0 goto +2;" \
+ "r1 += -1;" \
+ "call " #other ";" \
+ "exit;"); \
+ }
+
+CALLS_OTHER(even, odd) CALLS_OTHER(odd, even)
+
+SEC("tc")
+__description("instances: two instances wait for each other and end")
+__arch_x86_64
+__success __log_level(4) __msg("peak_states {{[0-9]?[0-9]}} mark_read")
+__naked void recursion_of_two_ends(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r1 = r0;"
+ "call even;"
+ ".rept 200;"
+ "if r6 & 1 goto +1;"
+ "r0 = 0;"
+ ".endr;"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void calls_even(void)
+{
+ ASM("call even; exit;");
+}
+
+SEC("tc")
+__description("instances: a function that waits for a recursion did not end")
+__arch_x86_64
+__failure __msg("R0 invalid mem access 'scalar'")
+__naked void waits_for_recursion_of_two(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r1 = r0;"
+ "call calls_even;"
+ "r0 = *(u8 *)(r0 + 0);"
+ "exit;");
+}
+
+struct task_first {
+ struct task_struct __kptr *task;
+ long val;
+};
+
+struct task_first_too {
+ struct task_struct __kptr *task;
+ long val;
+};
+
+struct task_last {
+ long val;
+ struct task_struct __kptr *task;
+};
+
+#define NEW(type) ({ \
+ struct type *p_ = bpf_obj_new(struct type); \
+ \
+ if (!p_) \
+ return 1; \
+ p_->val = -1; \
+ (void *)p_; \
+})
+
+SEC("tc")
+__description("fixups: two types with the same fields, the third one is on a path of the join")
+__success __retval(0) __log_level(2) __msg("pass 1")
+int drop_same_fields(void *unused)
+{
+ void *p;
+
+ if (never())
+ p = NEW(task_last);
+ else if (bpf_get_prandom_u32() & 1)
+ p = NEW(task_first);
+ else
+ p = NEW(task_first_too);
+ barrier_var(p);
+ bpf_obj_drop_impl(p, NULL);
+ return 0;
+}
+
+SEC("tc")
+__description("fixups: two types with other fields")
+__failure __msg("same call cannot be used with different types")
+int drop_other_fields(void *unused)
+{
+ void *p;
+
+ if (bpf_get_prandom_u32() & 1)
+ p = NEW(task_last);
+ else
+ p = NEW(task_first);
+ barrier_var(p);
+ bpf_obj_drop_impl(p, NULL);
+ return 0;
+}
+
+struct no_fields {
+ long val;
+};
+
+SEC("tc")
+__description("fixups: two types, one has no fields")
+__failure __msg("same call cannot be used with different types")
+int drop_fields_or_not(void *unused)
+{
+ void *p;
+
+ if (bpf_get_prandom_u32() & 1)
+ p = NEW(no_fields);
+ else
+ p = NEW(task_first);
+ barrier_var(p);
+ bpf_obj_drop_impl(p, NULL);
+ return 0;
+}
+
+struct with_percpu {
+ struct no_fields __percpu_kptr *p;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, struct with_percpu);
+} map_percpu SEC(".maps");
+
+extern const int bpf_prog_active __ksym;
+
+static __noinline void *this_cpu(const void *p)
+{
+ return bpf_this_cpu_ptr(p);
+}
+
+SEC("tc")
+__description("fixups: a percpu object of the program and a percpu variable of the kernel")
+__failure __msg("same call cannot be used with different types")
+int two_kinds_of_percpu(void *unused)
+{
+ struct no_fields *p, *mine;
+ struct with_percpu *v;
+ int key = 0;
+
+ v = bpf_map_lookup_elem(&map_percpu, &key);
+ if (!v)
+ return 0;
+ p = v->p;
+ if (!p)
+ return 0;
+ mine = this_cpu(p);
+ return mine->val + *(int *)this_cpu(&bpf_prog_active);
+}
+
+struct two_links {
+ long val;
+ struct bpf_refcount ref;
+ struct bpf_list_node link;
+ struct bpf_list_node link2;
+};
+
+private(two) struct bpf_spin_lock two_lock;
+private(two) struct bpf_list_head two_list __contains(two_links, link);
+private(two) struct bpf_list_head two_list2 __contains(two_links, link2);
+
+static __noinline void push(struct bpf_list_head *head, struct bpf_list_node *link)
+{
+ bpf_list_push_back(head, link);
+}
+
+SEC("tc")
+__description("fixups: one push with two offsets of the node")
+__failure __msg("same call cannot be used with different types")
+int push_two_links(void *unused)
+{
+ struct two_links *p = NEW(two_links), *q = bpf_refcount_acquire(p);
+
+ bpf_spin_lock(&two_lock);
+ push(&two_list, &p->link);
+ push(&two_list2, &q->link2);
+ bpf_spin_unlock(&two_lock);
+ return 0;
+}
+
+static __noinline void *acquire(void *p)
+{
+ return bpf_refcount_acquire_impl(p, NULL);
+}
+
+struct ref_first {
+ struct bpf_refcount ref;
+ long val;
+};
+
+SEC("tc")
+__description("fixups: one acquire with two offsets of the refcount")
+__failure __msg("same call cannot be used with different types")
+int acquire_two_types(void *unused)
+{
+ struct two_links *p = NEW(two_links);
+ struct ref_first *q;
+
+ bpf_obj_drop(acquire(p));
+ bpf_obj_drop(p);
+ q = NEW(ref_first);
+ bpf_obj_drop(acquire(q));
+ bpf_obj_drop(q);
+ return 0;
+}
+
+struct ref_first_with_link {
+ struct bpf_refcount ref;
+ long val;
+ struct bpf_list_node link;
+};
+
+SEC("tc")
+__description("fixups: one acquire with one offset of the refcount, the other fields differ")
+__success __retval(0)
+int acquire_same_offset(void *unused)
+{
+ struct ref_first_with_link *p = NEW(ref_first_with_link);
+ struct ref_first *q;
+
+ bpf_obj_drop(acquire(p));
+ bpf_obj_drop(p);
+ q = NEW(ref_first);
+ bpf_obj_drop(acquire(q));
+ bpf_obj_drop(q);
+ return 0;
+}
+
+static __noinline int new_if(long yes, __u64 type)
+{
+ void *p;
+
+ if (!yes)
+ return 0;
+ p = bpf_obj_new_impl(type, NULL);
+ if (p)
+ bpf_obj_drop_impl(p, NULL);
+ return 0;
+}
+
+SEC("tc")
+__description("fixups: a speculative state was first")
+__success __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __success_unpriv
+#ifdef SPEC_V1
+__xlated_unpriv("nospec")
+#endif
+int speculative_first(void *unused)
+{
+ long yes = 1, no = 0;
+
+ barrier_var(yes);
+ barrier_var(no);
+ return new_if(no, bpf_core_type_id_local(struct pair)) +
+ new_if(yes, bpf_core_type_id_local(struct bytes64));
+}
+
+/* r7 is 21 at most. The leftover gets to the load first, with the pointer to the context. */
+SEC("tc")
+__description("fixups: the leftover of a loop made up the state that was first")
+__success
+__naked void first_writer_is_wrong(void)
+{
+ ASM("r6 = r1;"
+ VALUE_IN_R9
+ "r7 = 0;"
+ "r8 = 0;"
+"1: call %[bpf_get_prandom_u32];"
+ "if w0 & 1 goto 2f;"
+ "r7 += 3;"
+"2: r8 += 1;"
+ "if r8 < 7 goto 1b;"
+ "if r7 <= 30 goto 3f;"
+ "r9 = r6;"
+"3: r0 = *(u32 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void deep_stack(void)
+{
+ asm volatile ("r0 = 0; *(u64 *)(r10 - 512) = r0; exit;" ::: __clobber_all);
+}
+
+static __naked __noinline __used void deeper_1(void)
+{
+ asm volatile ("r0 = 0; *(u64 *)(r10 - 512) = r0; call deep_stack; exit;" ::: __clobber_all);
+}
+
+static __naked __noinline __used void deeper_2(void)
+{
+ asm volatile ("r0 = 0; *(u64 *)(r10 - 512) = r0; call deeper_1; exit;" ::: __clobber_all);
+}
+
+static __naked __noinline __used void deeper_3(void)
+{
+ asm volatile ("r0 = 0; *(u64 *)(r10 - 512) = r0; call deeper_2; exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("fixups: the leftover of a loop made up a call")
+__success
+__naked void call_made_up_by_leftover(void)
+{
+ ASM("r7 = 0;"
+ "r8 = 0;"
+ "*(u64 *)(r10 - 512) = r8;"
+"1: call %[bpf_get_prandom_u32];"
+ "if w0 & 1 goto 2f;"
+ "r7 += 3;"
+"2: r8 += 1;"
+ "if r8 < 7 goto 1b;"
+ "if r7 <= 30 goto 9f;"
+ "call deeper_3;"
+ EXIT_AT_9);
+}
+
+/* r6 counts and is tested later, so its loop is a suspect in each pass */
+SEC("tc")
+__description("partition: a loop that counts does not hold up the other ones")
+__success FEW_INSNS
+__naked void peel_does_not_starve(void)
+{
+ ASM(VALUE_IN_R9
+ "r6 = 0;"
+"1:" MAY_GOTO(2)
+ "r6 += 1;"
+ "goto 1b;"
+ "call %[bpf_get_prandom_u32];"
+ "r7 = 0;"
+ "if r0 < 64 goto 2f;"
+ "r7 = 1;"
+"2: r8 = 0;"
+ "if r7 == 0 goto 3f;"
+ "r8 = 1;"
+"3: r1 = 0;"
+ "if r8 == 0 goto 4f;"
+ "r1 = 1;"
+"4: if r6 == 12345 goto 9f;"
+ "if r1 != 0 goto 9f;"
+ "r9 += r0;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+/*
+ * 'r6 u>= 3' cuts -4..3 in two. It makes 3..max of the join. The load is unproven in pass 0,
+ * because of r7.
+ */
+SEC("tc")
+__description("failing_path: no check fails on a path where the joined state is proven")
+__success __log_level(2) __msg("path-sensitive run after pass 0")
+__not_msg("invalid access to map value") __msg("pass 1")
+__naked void false_failing_path(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = *(u64 *)(r9 + 0);"
+ "if w0 & 2 goto 1f;"
+ "r6 &= 7;"
+ "r6 += -4;"
+"1: call %[bpf_get_prandom_u32];"
+ "r7 = 0;"
+ "r8 = 0;"
+ "if w0 & 1 goto 2f;"
+ "r7 = 100;"
+ "r8 = 1;"
+ "if w0 & 4 goto 2f;"
+ "r7 = 50;"
+ "r8 = 2;"
+"2: if r8 != 0 goto 9f;"
+ "if r6 < 3 goto 9f;"
+ "if r6 > 10 goto 9f;"
+ "r6 += r7;"
+ "r9 += r6;"
+ "r0 = *(u8 *)(r9 - 3);"
+ EXIT_AT_9);
+}
+
+/* r6 + r7 is 0 on all paths. States with all slots of the stack wait after 1800 jumps. */
+SEC("socket")
+__description("budget: memory")
+__failure_unpriv __msg_unpriv("the limit on cost or memory ended pass")
+__naked void over_the_memory_limit(void)
+{
+ ASM(VALUE_IN_R9
+ "r1 = *(u64 *)(r9 + 8);"
+ "r2 = *(u64 *)(r9 + 16);"
+ "r8 = *(u64 *)(r9 + 0);"
+ "r5 = 1;"
+ "r6 = 0;"
+ "r7 = 0;"
+ "*(u64 *)(r10 - 512) = r7;"
+ ".rept 12;"
+ "r8 >>= 1;"
+ "r5 <<= 1;"
+ "if r8 & 1 goto +2;"
+ "r6 += r5;"
+ "r7 -= r5;"
+ ".rept 150;"
+ "if r1 > r2 goto +1860;"
+ ".endr;"
+ ".endr;"
+ ".rept 1861;"
+ "r0 = 0;"
+ ".endr;"
+ "r0 = *(u64 *)(r10 - 512);"
+ "r6 += r7;"
+ "if r6 == 0 goto 9f;"
+ "r0 = *(u8 *)(r9 + 64);"
+ EXIT_AT_9);
+}
+
+/* r7 - r6 is 1..7 at the start. The ranges of r6 and r7 overlap at the jump. */
+SEC("tc")
+__description("peel: the distance of an iv from its bound")
+__success __xlated("r6 += 1") __xlated("if r6 != r7 goto")
+__naked void distance_in_peeled_trips(void)
+{
+ ASM(VALUE_IN_R9
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r6 &= 3;"
+ "r7 = r0;"
+ "r7 >>= 8;"
+ "r7 &= 3;"
+ "r7 += 4;"
+ "r8 = 1;"
+"1: r1 = r9;"
+ "r1 += r8;"
+ "r0 = *(u8 *)(r1 - 1);"
+ "r8 <<= 1;"
+ "r6 += 1;"
+ "if r6 != r7 goto 1b;"
+ EXIT_AT_9);
+}
+
+/* The inner loop continues where it stopped. Each of its trips starts with r8 = r9 + r7 - 1. */
+SEC("tc")
+__description("peel: states that enter a loop share the peeled trips")
+__success FEW_INSNS
+__naked void entries_share_peeled_trips(void)
+{
+ ASM(VALUE_IN_R9
+ "r7 = 0;"
+"1: r8 = r9;"
+ "r8 += r7;"
+ "r7 += 1;"
+"2: call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 3f;"
+ "*(u8 *)(r8 + 0) = 0;"
+ "r8 += 1;"
+ "r7 += 1;"
+ "if r7 == 65 goto 9f;"
+ "goto 2b;"
+"3: *(u8 *)(r8 + 0) = 1;"
+ "if r7 != 64 goto 1b;"
+ EXIT_AT_9);
+}
+
+/* (off, len) is (0, 64) or (56, 8) */
+SEC("syscall")
+__description("pessimized: the join raises the size of the context")
+__success __log_level(2) __msg("pass 1")
+__naked void join_raises_ctx_size(void)
+{
+ asm volatile (
+ "r6 = r1;"
+ "call %[bpf_get_prandom_u32];"
+ "r1 = 0;"
+ "r2 = 64;"
+ "if w0 & 1 goto 1f;"
+ "r1 = 56;"
+ "r2 = 8;"
+"1: r1 += r6;"
+ "r3 = 0;"
+ "call %[bpf_probe_read_kernel];"
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_get_prandom_u32), __imm(bpf_probe_read_kernel) : __clobber_all);
+}
+
+SEC("raw_tp")
+__description("pessimized: a join does not change a constant offset")
+__success __log_level(2) __not_msg("pass 1")
+__naked void join_before_ctx_read(void)
+{
+ ASM("r6 = r1;"
+ "call %[bpf_get_prandom_u32];"
+ "r7 = 1;"
+ "if w0 & 1 goto 1f;"
+ "r7 = 9;"
+"1: r0 = *(u64 *)(r6 + 8);"
+ "r0 += r7;"
+ "exit;");
+}
+
+/* The last one has no zero */
+static const char strings[] SEC(".rodata.strings") = {
+ 'a', '%', 'd', 0, [4] = 'b', 'c', 'd', '%', 'd', 0, [10] = '%', 'z', 0, [13] = 'e',
+};
+
+/* r3 = strings + @a or strings + @b */
+#define ONE_OF_TWO_STRINGS(a, b, call) \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "r3 = %[strings] ll;" \
+ "if w0 & 1 goto 1f;" \
+ "r3 += " #a ";" \
+ "goto 2f;" \
+"1: r3 += " #b ";" \
+"2: r1 = 0;" \
+ "*(u64 *)(r10 - 8) = r1;" \
+ "r1 = %[out] ll;" \
+ "r2 = 8;" \
+ "r4 = r10;" \
+ "r4 += -8;" \
+ "r5 = 8;" \
+ "call %[" #call "];" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_get_prandom_u32), __imm(call), __imm_addr(strings),\
+ __imm_addr(out) \
+ : __clobber_all)
+
+SEC("raw_tp")
+__description("strings: one call of bpf_strncmp() gets one of two strings")
+__success __log_level(2) __not_msg("pass 1")
+__naked void strncmp_two_strings(void)
+{
+ ONE_OF_TWO_STRINGS(0, 10, bpf_strncmp);
+}
+
+SEC("raw_tp")
+__description("strings: one of two strings has no zero")
+__failure __msg("string is not zero-terminated")
+__naked void strncmp_two_strings_bad(void)
+{
+ ONE_OF_TWO_STRINGS(0, 13, bpf_strncmp);
+}
+
+SEC("raw_tp")
+__description("strings: one call of bpf_snprintf() gets one of two formats")
+__success __log_level(2) __msg("R3 is not a constant address") __msg("pass 1")
+__naked void snprintf_two_formats(void)
+{
+ ONE_OF_TWO_STRINGS(0, 4, bpf_snprintf);
+}
+
+SEC("raw_tp")
+__description("strings: one of two formats is not valid")
+__failure __msg("Invalid format string")
+__naked void snprintf_two_formats_bad(void)
+{
+ ONE_OF_TWO_STRINGS(0, 10, bpf_snprintf);
+}
+
+#define STRING_AT_INDEX(max) \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 > " #max " goto 1f;" \
+ "r3 = %[strings] ll;" \
+ "r3 += r0;" \
+ "r1 = %[out] ll;" \
+ "r2 = 8;" \
+ "call %[bpf_strncmp];" \
+"1: r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_get_prandom_u32), __imm(bpf_strncmp), \
+ __imm_addr(strings), __imm_addr(out) \
+ : __clobber_all)
+
+SEC("socket")
+__description("strings: variable offset")
+__success
+#ifdef SPEC_V1
+__caps_unpriv(CAP_BPF) __failure_unpriv __msg_unpriv("R3 is not a constant address")
+#endif
+__naked void string_at_index(void)
+{
+ STRING_AT_INDEX(12);
+}
+
+SEC("socket")
+__description("strings: variable offset, the map ends")
+__failure __msg("R3 max value is outside of the allowed memory range")
+__naked void string_at_index_bad(void)
+{
+ STRING_AT_INDEX(14);
+}
+
+#define TWO_OFFSETS(op) \
+ VALUE_IN_R9 \
+ "r8 = r9;" \
+ "r8 += 32;" \
+ "if r9 " #op " r8 goto 9f;" \
+ "r0 = *(u8 *)(r9 + 64);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("infeasible: the offsets of two pointers to one object decide a jump")
+__success
+__naked void offsets_decide(void)
+{
+ ASM(TWO_OFFSETS(<));
+}
+
+SEC("tc")
+__description("infeasible: the offsets of two pointers decide for the other way")
+__failure __msg("invalid access to map value, value_size=64 off=64 size=1")
+__naked void offsets_decide_bad(void)
+{
+ ASM(TWO_OFFSETS(>));
+}
+
+__naked __noinline __used
+static long third_arg(void)
+{
+ ASM("r0 = r3;"
+ "exit;");
+}
+
+SEC("tc")
+__description("control_deps: one way of the jump throws")
+__arch_x86_64
+__success
+__naked void jump_to_throw(void)
+{
+ asm volatile (
+ "r3 = 5;"
+ "call third_arg;"
+ "r3 = 0;"
+ "call third_arg;"
+ "if r0 > 0 goto 1f;"
+ "r1 = 0;"
+ "call %[bpf_throw];"
+ "r0 = 0;"
+"1: r0 = r9;"
+ "exit;"
+ :: __imm(bpf_throw) : __clobber_all);
+}
+
+__naked __noinline __used
+static long first_arg_or_deeper(void)
+{
+ ASM("r6 = r1;"
+ "if r2 == 0 goto 1f;"
+ "r1 = 1;"
+ "call first_arg_or_deeper;"
+"1: r0 = r6;"
+ "exit;");
+}
+
+/* r7 is not 0 if r8 holds an entry */
+SEC("tc")
+__description("instances: a late exit finds the header of its loop")
+__arch_x86_64
+__success
+__naked void late_exit_in_counted_loop(void)
+{
+ asm volatile (
+ "r8 = 0;"
+"1:" MAY_GOTO(28)
+ "r6 = 0;"
+"2: r1 = 5;"
+ "r2 = 0;"
+ "r3 = r8;"
+ "call first_arg_or_deeper;"
+ "r7 = r0;"
+ "r1 = r7;"
+ "r2 = 2;"
+ "call first_arg_or_deeper;"
+ "r8 = 0;"
+ "r6 += 1;"
+ "if r6 < 3 goto 2b;"
+ "call %[bpf_get_prandom_u32];"
+ "if r0 == 0 goto 4f;"
+ "r7 = 1;"
+"4: if r0 == 0 goto 3f;"
+ "r1 = %[rb] ll;"
+ "r2 = 8;"
+ "r3 = 0;"
+ "call %[bpf_ringbuf_reserve];"
+ "r8 = r0;"
+"3: if r7 == 0 goto 1b;"
+ "if r8 == 0 goto 1b;"
+ "r1 = r8;"
+ "r2 = 0;"
+ "call %[bpf_ringbuf_discard];"
+ "goto 1b;"
+ "r0 = 0;"
+ "exit;"
+ :: __imm_addr(rb), __imm(bpf_get_prandom_u32), __imm(bpf_ringbuf_reserve),
+ __imm(bpf_ringbuf_discard)
+ : __clobber_all);
+}
+
+/* The loop has two entries. With the iterator may_goto goes to where the loop is left. */
+SEC("tc")
+__description("guards: the code after the loop reads a slot that each trip writes")
+__success
+__naked void guard_keeps_slot(void)
+{
+ ASM("r1 = r10;"
+ "r1 += -8;"
+ "r2 = 0;"
+ "r3 = 9;"
+ "call %[bpf_iter_num_new];"
+ "r6 = 0;"
+ "*(u64 *)(r10 - 16) = r6;"
+ "goto 2f;"
+"1: r1 = r10;"
+ "r1 += -8;"
+ "call %[bpf_iter_num_next];"
+ "if r0 == 0 goto 3f;"
+ "r6 = 0;"
+"2: call %[bpf_get_prandom_u32];"
+ "*(u64 *)(r10 - 16) = r0;"
+ "r6 += 1;"
+ "if r6 != 32 goto 2b;"
+ "if r0 != 7 goto 1b;"
+"3: r1 = r10;"
+ "r1 += -8;"
+ "call %[bpf_iter_num_destroy];"
+ "r0 = *(u64 *)(r10 - 16);"
+ "exit;");
+}
+
+/* r7 is 5 if r6 is an offset in the value. r8 is 0, 3, 6, ... */
+SEC("tc")
+__description("join point: a join with a step has no gap")
+__success
+__naked void step_is_no_gap(void)
+{
+ ASM("r6 = 0;"
+ "r7 = 5;"
+ "r8 = 0;"
+ "*(u64 *)(r10 - 16) = r8;"
+ VALUE_IN_R9
+"1: if r7 == 5 goto 2f;"
+ "r8 += 3;"
+ "goto 3f;"
+"2: r1 = r9;"
+ "r1 += r6;"
+ "r0 = *(u8 *)(r1 + 0);"
+"3: call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r7 = 5;"
+ "if r6 < 64 goto 4f;"
+ "r7 = 2;"
+"4: r1 = *(u64 *)(r10 - 16);"
+ "r1 += 1;"
+ "*(u64 *)(r10 - 16) = r1;"
+ "if r1 < 300 goto 1b;"
+ "r8 >>= 4;"
+ "r9 += r8;"
+ "r0 = *(u8 *)(r9 + 0);"
+ EXIT_AT_9);
+}
+
+/* r6 + r7 is the same on each of 2^14 paths */
+#define MANY_PAIRS(sum) \
+ VALUE_IN_R9 \
+ "r6 = 0;" \
+ "r7 = 16383;" \
+ ".irp bit, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13\n" \
+ "call %[bpf_get_prandom_u32];" \
+ "if w0 & 1 goto 1f;" \
+ "r6 += 1 << \\bit;" \
+ "r7 -= 1 << \\bit;" \
+ "1:\n" \
+ ".endr\n" \
+ "r6 += r7;" \
+ "r6 -= " #sum ";" \
+ "r9 += r6;" \
+ "r0 = *(u8 *)(r9 + 0);" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("join point: many states at one insn")
+__success __retval(0)
+__naked void many_pairs(void)
+{
+ ASM(MANY_PAIRS(16383));
+}
+
+SEC("tc")
+__description("join point: many states at one insn, out of bounds")
+__failure __msg("invalid access to map value")
+__naked void many_pairs_bad(void)
+{
+ ASM(MANY_PAIRS(16319));
+}
+
+/* r2 is r1 + 8. No run has r1 <= 32 and r2 > 40. */
+SEC("tc")
+__description("infeasible: an equation decides a jump")
+__success __retval(0)
+__naked void equation_decides(void)
+{
+ ASM("call %[bpf_get_prandom_u32];"
+ "r1 = r0;"
+ "r1 &= 255;"
+ "r2 = r1;"
+ "r2 += 8;"
+ "if r1 > 32 goto 9f;"
+ "if r2 <= 40 goto 9f;"
+ "if r1 > 16 goto 1f;"
+ "r0 = 1;"
+ "exit;"
+"1: r0 = 2;"
+ "exit;"
+ EXIT_AT_9);
+}
+
+static __naked __noinline __used void takes_number(long x)
+{
+ asm volatile ("r0 = 0; exit;" ::: __clobber_all);
+}
+
+/* The argument does not fit the BTF of a static function. That is no error. */
+SEC("tc")
+__description("log: no report for a call that is proven")
+__failure __msg("R1 type=fp expected=scalar") __not_msg("Verification failed")
+__msg("invalid access to map value") __msg("Verification failed")
+__naked void log_of_proven_call(void)
+{
+ ASM("r1 = r10;"
+ "call takes_number;"
+ VALUE_IN_R9
+ "r0 = *(u8 *)(r9 + 64);"
+ EXIT_AT_9);
+}
+
+/* clang ignores the arguments of a naked function */
+#if defined(__clang__) || defined(__BPF_FEATURE_STACK_ARGUMENT)
+static __naked __noinline __used long loads_6th(long a, long b, long c, long d, long e, long f)
+{
+ /* r0 = *(u64 *)(r11 + 8) */
+ asm volatile (".8byte 0x8b079; exit;" ::: __clobber_all);
+}
+
+SEC("tc")
+__description("log: one report for a stack argument that is not there")
+__failure __msg("callee expects 6 args")
+__msg("Verification failed") __not_msg("Verification failed")
+__load_if_JITed()
+__arch_x86_64 __arch_arm64
+__naked void log_of_missing_stack_arg(void)
+{
+ asm volatile ("r1 = 1; r2 = 2; r3 = 3; r4 = 4; r5 = 5; call loads_6th; exit;"
+ ::: __clobber_all);
+}
+#endif
+
+/* *(u64 *)(r11 - 8) = imm */
+#define STORE_6TH(imm) ".byte 0x7a; .byte 0xb; .short -8; .long " #imm ";"
+#define ARGS_1_TO_5 "r1 = 1; r2 = 2; r3 = 3; r4 = 4; r5 = 5;"
+
+/* r7 is 0..15 with the argument, and add..add + 15 without it */
+#define STORE_ON_ONE_PATH(add) \
+ "call %[bpf_get_prandom_u32];" \
+ "r7 = r0;" \
+ "r7 &= 15;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 == 0 goto 1f;" \
+ STORE_6TH(1) \
+ "goto 2f;" \
+"1: r7 += " #add ";" \
+"2: if r7 > 7 goto 9f;" \
+ ARGS_1_TO_5 \
+ "call loads_6th;" \
+ EXIT_AT_9
+
+SEC("tc")
+__description("deps: only one state has the stack argument")
+__success
+__load_if_JITed()
+__arch_x86_64 __arch_arm64
+__naked void stack_arg_on_one_path(void)
+{
+ ASM(STORE_ON_ONE_PATH(8));
+}
+
+SEC("tc")
+__description("deps: the state without the stack argument gets to the call")
+__failure __msg("callee expects 6 args")
+__load_if_JITed()
+__arch_x86_64 __arch_arm64
+__naked void stack_arg_on_one_path_bad(void)
+{
+ ASM(STORE_ON_ONE_PATH(7));
+}
+
+/* x * x - 21 * x + 110 is 0 for 10 and for 11 */
+static __naked __noinline __used long roots_of_6th(long a, long b, long c, long d, long e, long f)
+{
+ /* r6 = *(u64 *)(r11 + 8) */
+ ASM(".8byte 0x8b679;"
+ VALUE_IN_R9
+ "r1 = r6;"
+ "r1 *= r6;"
+ "r6 *= 21;"
+ "r1 -= r6;"
+ "r1 += 110;"
+ "r9 += r1;"
+ "r0 = *(u8 *)(r9 + 63);"
+ EXIT_AT_9);
+}
+
+#define ROOTS_ON_STACK(second) \
+ "call %[bpf_get_prandom_u32];" \
+ STORE_6TH(10) \
+ "if r0 == 0 goto 1f;" \
+ STORE_6TH(second) \
+"1:" ARGS_1_TO_5 \
+ "call roots_of_6th;" \
+ "exit;"
+
+SEC("tc")
+__description("keep_disjoint_apart: a stack argument")
+__success
+__load_if_JITed()
+__arch_x86_64 __arch_arm64
+__naked void disjoint_stack_arg(void)
+{
+ ASM(ROOTS_ON_STACK(11));
+}
+
+SEC("tc")
+__description("keep_disjoint_apart: a stack argument is no root")
+__failure __msg("invalid access to map value, value_size=64 off=65 size=1")
+__load_if_JITed()
+__arch_x86_64 __arch_arm64
+__naked void disjoint_stack_arg_bad(void)
+{
+ ASM(ROOTS_ON_STACK(9));
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c
index e7636b86cc2..e08f0501a25 100644
--- a/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c
+++ b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c
@@ -27,6 +27,20 @@ __naked int stats_main_only(void)
);
}
+/* The pass checks 3 insns, the path-sensitive run 2. The repeat for the log does not count. */
+SEC("?raw_tp")
+__failure __log_level(4)
+__msg("subprog 0 (stats_failure) main insns_self 5 insns_total 5 stack 0")
+__msg("processed 5 insns")
+__naked int stats_failure(void)
+{
+ asm volatile (
+ "r0 = 0;"
+ "r0 = *(u64 *)(r10 + 8);"
+ "exit;"
+ );
+}
+
__naked __noinline __used
static int stats_chain_leaf(void)
{
diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c
index 4cfdedca889..bc45adafca3 100644
--- a/tools/testing/selftests/bpf/progs/verifier_var_off.c
+++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c
@@ -759,4 +759,50 @@ __naked void variable_offset_stack_access_ok(void)
: __clobber_all);
}
+/* The next two need bpf_jit_bypass_spec_v1() */
+SEC("socket")
+__description("variable-offset stack write of a pointer")
+__arch_loongarch
+__success __caps_unpriv(CAP_BPF) __failure_unpriv
+__msg_unpriv("R1 leaks addr into stack")
+__naked void stack_write_ptr_var_off(void)
+{
+ asm volatile (" \
+ r0 = 0; \
+ *(u64*)(r10 - 8) = r0; \
+ *(u64*)(r10 - 16) = r0; \
+ lock *(u64*)(r10 - 8) += r0; \
+ lock *(u64*)(r10 - 16) += r0; \
+ r2 = *(u32*)(r1 + 0); \
+ r2 &= 8; \
+ r2 -= 16; \
+ r2 += r10; \
+ *(u64*)(r2 + 0) = r1; \
+ r0 = *(u64*)(r10 - 8); \
+ exit; \
+" ::: __clobber_all);
+}
+
+SEC("socket")
+__description("variable-offset stack pointer at the header of a loop")
+__arch_loongarch
+__success __caps_unpriv(CAP_BPF) __success_unpriv
+__naked void stack_ptr_var_off_in_loop(void)
+{
+ asm volatile (" \
+ r1 = 0; \
+ *(u64*)(r10 - 8) = r1; \
+ r2 = r10; \
+ r2 += -8; \
+1: r0 = *(u8*)(r2 + 0); \
+ r1 += 1; \
+ r2 = r1; \
+ r2 &= 7; \
+ r2 -= 8; \
+ r2 += r10; \
+ if r1 < 0x10000000 goto 1b; \
+ exit; \
+" ::: __clobber_all);
+}
+
char _license[] SEC("license") = "GPL";
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* Re: [PATCH bpf-next 47/49] selftests/bpf: Test passes, instances, precision and the failing path
2026-11-20 7:42 ` [PATCH bpf-next 47/49] selftests/bpf: Test passes, instances, precision and the failing path Alexei Starovoitov
@ 2026-11-20 8:30 ` sashiko-bot
0 siblings, 0 replies; 79+ messages in thread
From: sashiko-bot @ 2026-11-20 8:30 UTC (permalink / raw)
To: Alexei Starovoitov, bpf
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Medium] selftests/bpf: premature #endif around loads_6th() in
verifier_passes.c
--
commit ef0b2c0eaafb6e699dfff5dd0c5556a322cf2fa1
Author: Alexei Starovoitov <ast@kernel.org>
selftests/bpf: Test passes, instances, precision and the failing path
This commit adds BPF verifier selftests covering verification passes,
partitions, loop peeling, call instances, precision tracking, and
failing-path diagnostics.
> diff --git a/tools/testing/selftests/bpf/progs/verifier_passes.c b/tools/testing/selftests/bpf/progs/verifier_passes.c
> new file mode 100644
> index 0000000000000..a6edb33d0014b
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/progs/verifier_passes.c
> @@ -0,0 +1,3636 @@
[ ... ]
> +/* clang ignores the arguments of a naked function */
> +#if defined(__clang__) || defined(__BPF_FEATURE_STACK_ARGUMENT)
> +static __naked __noinline __used long loads_6th(long a, long b, long c, long d, long e, long f)
> +{
> + /* r0 = *(u64 *)(r11 + 8) */
> + asm volatile (".8byte 0x8b079; exit;" ::: __clobber_all);
> +}
> +
> +SEC("tc")
> +__description("log: one report for a stack argument that is not there")
> +__failure __msg("callee expects 6 args")
> +__msg("Verification failed") __not_msg("Verification failed")
> +__load_if_JITed()
> +__arch_x86_64 __arch_arm64
> +__naked void log_of_missing_stack_arg(void)
> +{
> + asm volatile ("r1 = 1; r2 = 2; r3 = 3; r4 = 4; r5 = 5; call loads_6th; exit;"
> + ::: __clobber_all);
> +}
> +#endif
> +
> +/* *(u64 *)(r11 - 8) = imm */
> +#define STORE_6TH(imm) ".byte 0x7a; .byte 0xb; .short -8; .long " #imm ";"
> +#define ARGS_1_TO_5 "r1 = 1; r2 = 2; r3 = 3; r4 = 4; r5 = 5;"
> +
> +/* r7 is 0..15 with the argument, and add..add + 15 without it */
> +#define STORE_ON_ONE_PATH(add) \
> + "call %[bpf_get_prandom_u32];" \
> + "r7 = r0;" \
> + "r7 &= 15;" \
> + "call %[bpf_get_prandom_u32];" \
> + "if r0 == 0 goto 1f;" \
> + STORE_6TH(1) \
> + "goto 2f;" \
> +"1: r7 += " #add ";" \
> +"2: if r7 > 7 goto 9f;" \
> + ARGS_1_TO_5 \
> + "call loads_6th;" \
> + EXIT_AT_9
> +
> +SEC("tc")
> +__description("deps: only one state has the stack argument")
> +__success
> +__load_if_JITed()
> +__arch_x86_64 __arch_arm64
> +__naked void stack_arg_on_one_path(void)
> +{
> + ASM(STORE_ON_ONE_PATH(8));
> +}
> +
> +SEC("tc")
> +__description("deps: the state without the stack argument gets to the call")
> +__failure __msg("callee expects 6 args")
> +__load_if_JITed()
> +__arch_x86_64 __arch_arm64
> +__naked void stack_arg_on_one_path_bad(void)
> +{
> + ASM(STORE_ON_ONE_PATH(7));
> +}
> +
> +/* x * x - 21 * x + 110 is 0 for 10 and for 11 */
> +static __naked __noinline __used long roots_of_6th(long a, long b, long c, long d, long e, long f)
> +{
> + /* r6 = *(u64 *)(r11 + 8) */
> + ASM(".8byte 0x8b679;"
> + VALUE_IN_R9
> + "r1 = r6;"
> + "r1 *= r6;"
> + "r6 *= 21;"
> + "r1 -= r6;"
> + "r1 += 110;"
> + "r9 += r1;"
> + "r0 = *(u8 *)(r9 + 63);"
> + EXIT_AT_9);
> +}
> +
> +#define ROOTS_ON_STACK(second) \
> + "call %[bpf_get_prandom_u32];" \
> + STORE_6TH(10) \
> + "if r0 == 0 goto 1f;" \
> + STORE_6TH(second) \
> +"1:" ARGS_1_TO_5 \
> + "call roots_of_6th;" \
> + "exit;"
> +
> +SEC("tc")
> +__description("keep_disjoint_apart: a stack argument")
> +__success
> +__load_if_JITed()
> +__arch_x86_64 __arch_arm64
> +__naked void disjoint_stack_arg(void)
> +{
> + ASM(ROOTS_ON_STACK(11));
> +}
> +
> +SEC("tc")
> +__description("keep_disjoint_apart: a stack argument is no root")
> +__failure __msg("invalid access to map value, value_size=64 off=65 size=1")
> +__load_if_JITed()
> +__arch_x86_64 __arch_arm64
> +__naked void disjoint_stack_arg_bad(void)
> +{
> + ASM(ROOTS_ON_STACK(9));
> +}
[Severity: Medium]
Should the #endif after log_of_missing_stack_arg() be moved below
disjoint_stack_arg_bad()? When building with bpf-gcc without
__BPF_FEATURE_STACK_ARGUMENT, loads_6th() is not defined for
STORE_ON_ONE_PATH, and roots_of_6th() declares six parameters outside the
guard, breaking compilation.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261120074430.2704488-1-alexei.starovoitov@gmail.com?part=47
^ permalink raw reply [flat|nested] 79+ messages in thread
* [PATCH bpf-next 48/49] selftests/bpf: Test equations
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (46 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 47/49] selftests/bpf: Test passes, instances, precision and the failing path Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
2026-11-20 7:42 ` [PATCH bpf-next 49/49] bpf, docs: Rewrite verifier.rst Alexei Starovoitov
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
verifier_equations.c has one group for each kind of insn: truncation,
AND, op, shift, division, spill. It also tests joins, rings such as
'(i + 1) & 63', irreducible loops, the join rule, steps, an int that
goes through 0, sign extensions, the distance of two pointers, packet
pointers, far pointers, new ids, speculation past a stored condition, a
number that grows in two loops, right and wrong steps from the code of a
loop, the offset and the size of a memory argument, a test against the
rest of a buffer, and a test of the carry.
INDEX_TEST() makes two programs: the array is large enough for the
index, and the array is too small.
The other files get tests for the trip index, thresholds, atomic insns,
packet pointers that share an id, hidden registers, and division.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
.../selftests/bpf/prog_tests/verifier.c | 2 +
.../selftests/bpf/progs/verifier_and.c | 41 +
.../bpf/progs/verifier_arena_stack.c | 50 +
.../bpf/progs/verifier_bounds_step.c | 53 +
.../bpf/progs/verifier_bpf_fastcall.c | 33 +
.../bpf/progs/verifier_direct_packet_access.c | 98 +
.../bpf/progs/verifier_div_mod_bounds.c | 28 +
.../selftests/bpf/progs/verifier_equations.c | 1583 +++++++++++++++++
.../bpf/progs/verifier_linked_scalars.c | 144 ++
.../selftests/bpf/progs/verifier_movsx.c | 20 +
.../selftests/bpf/progs/verifier_spill_fill.c | 26 +
.../selftests/bpf/progs/verifier_trip_count.c | 474 +++++
12 files changed, 2552 insertions(+)
create mode 100644 tools/testing/selftests/bpf/progs/verifier_equations.c
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index da112954a74..22e948cf66e 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -60,6 +60,7 @@
#include "verifier_div_const.skel.h"
#include "verifier_div_mod_bounds.skel.h"
#include "verifier_div_overflow.skel.h"
+#include "verifier_equations.skel.h"
#include "verifier_flow_keys.skel.h"
#include "verifier_global_subprogs.skel.h"
#include "verifier_global_ptr_args.skel.h"
@@ -274,6 +275,7 @@ void test_verifier_div0(void) { RUN(verifier_div0); }
void test_verifier_div_const(void) { RUN(verifier_div_const); }
void test_verifier_div_mod_bounds(void) { RUN(verifier_div_mod_bounds); }
void test_verifier_div_overflow(void) { RUN(verifier_div_overflow); }
+void test_verifier_equations(void) { RUN(verifier_equations); }
void test_verifier_flow_keys(void) { RUN(verifier_flow_keys); }
void test_verifier_global_subprogs(void) { RUN(verifier_global_subprogs); }
void test_verifier_global_ptr_args(void) { RUN(verifier_global_ptr_args); }
diff --git a/tools/testing/selftests/bpf/progs/verifier_and.c b/tools/testing/selftests/bpf/progs/verifier_and.c
index 8ef5f87a976..22980b0c536 100644
--- a/tools/testing/selftests/bpf/progs/verifier_and.c
+++ b/tools/testing/selftests/bpf/progs/verifier_and.c
@@ -110,4 +110,45 @@ l0_%=: r0 = 0; \
: __clobber_all);
}
+SEC("socket")
+__description("and that clears no bit")
+__success __log_level(2)
+__msg("(54) w0 &= 255 {{.*}}R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=16")
+__naked void and_clears_no_bit(void)
+{
+ asm volatile (" \
+ call %[bpf_get_prandom_u32]; \
+ r0 &= 15; \
+ r0 += 1; \
+ w0 &= 255; \
+ exit; \
+" :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
+SEC("socket")
+__description("and that clears no bit of the low half")
+__success __retval(0)
+__naked void and_clears_no_low_bit(void)
+{
+ asm volatile (" \
+ call %[bpf_ktime_get_ns]; \
+ if w0 < 1 goto l0_%=; \
+ if w0 > 16 goto l0_%=; \
+ w0 &= 255; \
+ if w0 == 0 goto l1_%=; \
+ call %[bpf_ktime_get_ns]; \
+ if w0 < 1 goto l0_%=; \
+ if w0 > 16 goto l0_%=; \
+ r0 &= 255; \
+ if r0 != 0 goto l0_%=; \
+l1_%=: r0 /= 0; \
+l0_%=: r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_ktime_get_ns)
+ : __clobber_all);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_stack.c b/tools/testing/selftests/bpf/progs/verifier_arena_stack.c
index 4feaebf0d72..b90127d1f36 100644
--- a/tools/testing/selftests/bpf/progs/verifier_arena_stack.c
+++ b/tools/testing/selftests/bpf/progs/verifier_arena_stack.c
@@ -1362,6 +1362,56 @@ __naked void packet_write_prologue(void)
: __clobber_all);
}
+/* r12 is live after the first move. The number in fp-16 is another number. */
+SEC("syscall")
+__description("arena stack: r12 is not a slot")
+__success __retval(1)
+__naked void not_a_slot(void)
+{
+ asm volatile (
+ ".8byte %[adj];"
+ "r1 = 0;"
+ "*(u64 *)(r10 - 16) = r1;"
+ ".8byte %[mov1];" /* r1 = r12 */
+ ".8byte %[mov2];" /* r2 = r12 */
+ "r0 = 0;"
+ "if r1 == 0 goto 1f;"
+ "r0 = 1;"
+"1:"
+ "exit;"
+ :
+ : __imm_insn(adj, ADJ(16)),
+ __imm_insn(mov1, MOV(BPF_REG_1)),
+ __imm_insn(mov2, MOV(BPF_REG_2))
+ : __clobber_all);
+}
+
+SEC("syscall")
+__description("arena stack: r12 has no bound from a slot")
+__failure __msg("math between fp pointer and register with unbounded min value")
+__naked void no_bound_from_slot(void)
+{
+ asm volatile (
+ ".8byte %[adj];"
+ "call %[bpf_get_prandom_u32];"
+ "*(u64 *)(r10 - 16) = r0;"
+ ".8byte %[mov1];" /* r1 = r12 */
+ ".8byte %[mov2];" /* r2 = r12 */
+ "if r0 > 7 goto 1f;"
+ "r3 = r10;"
+ "r3 += -16;"
+ "r3 += r1;"
+ "r0 = *(u8 *)(r3 + 0);"
+"1:"
+ "exit;"
+ :
+ : __imm(bpf_get_prandom_u32),
+ __imm_insn(adj, ADJ(16)),
+ __imm_insn(mov1, MOV(BPF_REG_1)),
+ __imm_insn(mov2, MOV(BPF_REG_2))
+ : __clobber_all);
+}
+
#endif
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
index 28bdc63f48d..d1b3639a13c 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c
@@ -545,4 +545,57 @@ l_out_%=: \
: __clobber_all);
}
+/*
+ * Three states join: r6 is a multiple of 3 and has no bounds. r7 = r6 + 1 is not a multiple of 3.
+ * r7 <= 10 leaves r6 = 0, 3, 6, 9.
+ */
+SEC("socket")
+__failure __log_level(2)
+__msg("(bf) r7 = r6 {{.*}}R6=scalar(step=0+3)")
+__msg("if r7 > 0xa {{.*}}R6=scalar(smin=smin32=0,smax=umax=smax32=umax32=9,")
+__msg("div by zero")
+__naked void step_with_no_bounds(void)
+{
+ asm volatile (" \
+ call %[bpf_get_prandom_u32]; \
+ r8 = r0; \
+ call %[bpf_get_prandom_u32]; \
+ r6 = r0; \
+ r6 <<= 32; \
+ r6 s>>= 2; \
+ r1 = 0x2800000000000000 ll; \
+ r2 = -0x2800000000000000 ll; \
+ if r6 s> r1 goto l_out_%=; \
+ if r6 s< r2 goto l_out_%=; \
+ r6 |= r8; \
+ if r6 s> r1 goto l_out_%=; \
+ if r6 s< r2 goto l_out_%=; \
+ if r8 < 100 goto l_all_%=; \
+ r1 s>>= 1; \
+ r2 s>>= 1; \
+ if r6 s> r1 goto l_high_%=; \
+ if r6 s> r2 goto l_out_%=; \
+ r6 *= 3; \
+ goto l_ends_%=; \
+l_high_%=: \
+ r6 *= 3; \
+l_ends_%=: \
+ r0 = 0; \
+ goto l_join_%=; \
+l_all_%=: \
+ r6 *= 3; \
+l_join_%=: \
+ r7 = r6; \
+ r7 += 1; \
+ if r7 > 10 goto l_out_%=; \
+ if r6 != 9 goto l_out_%=; \
+ r0 /= 0; \
+l_out_%=: \
+ r0 = 0; \
+ exit; \
+" :
+ : __imm(bpf_get_prandom_u32)
+ : __clobber_all);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
index 314652a0006..2120f4e71fb 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
@@ -671,6 +671,39 @@ __naked void helper_zero_size_buffer(void)
: __clobber_all);
}
+/* The size is 0 only at the largest offset. The other offsets touch the stack. */
+SEC("raw_tp")
+__arch_x86_64
+__log_level(4)
+__msg("subprog 0 (helper_zero_size_at_max_offset) main {{.*}} stack 128")
+__success
+__naked void helper_zero_size_at_max_offset(void)
+{
+ asm volatile (
+ "call %[bpf_ktime_get_ns];"
+ "r6 = r0;"
+ "if r6 > 128 goto 1f;"
+ "r1 = 1;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "call %[bpf_get_smp_processor_id];"
+ "r1 = *(u64 *)(r10 - 8);"
+ "r1 = r10;"
+ "r1 += -128;"
+ "r1 += r6;"
+ "r2 = 128;"
+ "r2 -= r6;"
+ "r3 = 0;"
+ "call %[bpf_probe_read_kernel];"
+"1:"
+ "r0 = 0;"
+ "exit;"
+ :
+ : __imm(bpf_get_smp_processor_id),
+ __imm(bpf_ktime_get_ns),
+ __imm(bpf_probe_read_kernel)
+ : __clobber_all);
+}
+
SEC("raw_tp")
__arch_x86_64
/* main, not patched */
diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
index 7c29ffed743..bbc773fd8e7 100644
--- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
+++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c
@@ -1092,6 +1092,26 @@ __naked void pkt_add_negative_above(void)
PKT_ADD_NEGATIVE(50);
}
+#define NOT_CHECKED __failure __msg("invalid access to packet")
+#define PKT_MOVES(name, verdict, insns) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ PKT_54_BYTES("if r4 > 8 goto l0_%=;" \
+ "r4 += -4;" \
+ insns \
+ "r0 = *(u8*)(r2 + 0);"); \
+}
+
+/* r4 is -4..4. The largest offset is 53 in the good programs and 54 in the bad ones. */
+PKT_MOVES(pkt_sub_negative, __success, "r2 += 49; r2 -= r4;")
+PKT_MOVES(pkt_sub_negative_above, NOT_CHECKED, "r2 += 50; r2 -= r4;")
+PKT_MOVES(pkt_comes_back, __success, "r2 += 60; r2 += r4; r2 += -11;")
+PKT_MOVES(pkt_comes_back_above, NOT_CHECKED, "r2 += 60; r2 += r4; r2 += -10;")
+PKT_MOVES(pkt_comes_back_from_far, __success, "r2 += 100000; r2 -= 99947;")
+PKT_MOVES(pkt_comes_back_from_far_above, NOT_CHECKED, "r2 += 100000; r2 -= 99946;")
+
/* The byte is in the packet, but a range that umax gets to is not kept */
SEC("tc")
__description("direct packet access: checked pointer plus unknown that can get to the range")
@@ -1218,4 +1238,82 @@ l0_%=: r0 = 0; \
: __clobber_all);
}
+/* r2 is in front of the packet. Its smax goes from S64_MAX to @max. The division is reached. */
+#define PKT_SMAX_DROPS(max, op) \
+ asm volatile (" \
+ r6 = r1; \
+ call %[bpf_ktime_get_ns]; \
+ r2 = *(u32*)(r6 + %[__sk_buff_data]); \
+ r3 = *(u32*)(r6 + %[__sk_buff_data_end]); \
+ if r0 s< -20 goto l0_%=; \
+ r2 += r0; \
+ if r0 s> " #max " goto l0_%=; \
+ if r2 " #op " r3 goto l0_%=; \
+ r0 /= 0; \
+l0_%=: r0 = 0; \
+ exit; \
+" : \
+ : __imm(bpf_ktime_get_ns), \
+ __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), \
+ __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) \
+ : __clobber_all)
+
+SEC("tc")
+__description("direct packet access: a jump refines the pointer, it is not beyond the end")
+__failure __msg("div by zero")
+__naked void pkt_refined_is_not_beyond_end(void)
+{
+ PKT_SMAX_DROPS(-3, >);
+}
+
+SEC("tc")
+__description("direct packet access: a jump refines the pointer, it is not at the end")
+__failure __msg("div by zero")
+__naked void pkt_refined_is_not_at_end(void)
+{
+ PKT_SMAX_DROPS(-2, >=);
+}
+
+/*
+ * r4 is 20, 24, 28 or 48.., its range is 20..40. r2 = pkt + r4 and r5 = r2 - 16 share an id.
+ * The known bits of r5 tell 4..12. The check of r2 gives r=40 to both: 16 bytes from r5.
+ */
+#define PKT_TWO_OF_AN_ID(off) \
+ asm volatile (" \
+ r2 = *(u32*)(r1 + %[__sk_buff_data]); \
+ r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \
+ r4 = *(u32*)(r1 + %[__sk_buff_mark]); \
+ r4 &= 0x2c; \
+ r4 |= 0x10; \
+ r2 += r4; \
+ r5 = r2; \
+ r5 += -16; \
+ if r4 < 20 goto l0_%=; \
+ if r4 > 40 goto l0_%=; \
+ if r2 > r3 goto l0_%=; \
+ r0 = *(u8*)(r5 + " #off "); \
+l0_%=: r0 = 0; \
+ exit; \
+" : \
+ : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), \
+ __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) \
+ : __clobber_all)
+
+SEC("tc")
+__description("direct packet access: a jump refines two pointers of an id, last byte")
+__success __retval(0)
+__naked void pkt_refined_id_last_byte(void)
+{
+ PKT_TWO_OF_AN_ID(15);
+}
+
+SEC("tc")
+__description("direct packet access: a jump refines two pointers of an id, after the end")
+__failure __msg("invalid access to packet, off=40 size=1, R5(id={{[0-9]+}},off=40,r=40)")
+__naked void pkt_refined_id_after_end(void)
+{
+ PKT_TWO_OF_AN_ID(16);
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c b/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c
index e814a054d69..4040e71c2d3 100644
--- a/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c
+++ b/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c
@@ -1149,3 +1149,31 @@ l0_%=: r0 = *(u64 *)(r1 + 0); \
: __imm_const(llong_min, LLONG_MIN)
: __clobber_all);
}
+
+/* r6 is computed from r7, then r7 is tested: 0..200. The division is reached if r6 > @max. */
+#define OP_THEN_TEST(op, max) \
+ asm volatile (" \
+ call %[bpf_ktime_get_ns]; \
+ r7 = r0; \
+ r6 = r0; \
+ " op "; \
+ if r7 > 200 goto l0_%=; \
+ if r6 <= " #max " goto l0_%=; \
+ r0 /= 0; \
+l0_%=: r0 = 0; \
+ exit; \
+" : \
+ : __imm(bpf_ktime_get_ns) \
+ : __clobber_all)
+
+#define TEST_OP(name, op, max, below) \
+SEC("socket") __success \
+__naked void name(void) { OP_THEN_TEST(op, max); } \
+SEC("socket") __failure __msg("div by zero") \
+__naked void name##_bad(void) { OP_THEN_TEST(op, below); }
+
+TEST_OP(rsh32_of_wide, "w6 >>= 2", 50, 49)
+TEST_OP(div32_of_wide, "w6 /= 5", 40, 39)
+TEST_OP(mod32_of_wide, "w6 %%= 1000", 200, 199)
+TEST_OP(rsh_by_0, "r6 >>= 0", 200, 199)
+TEST_OP(div_by_1, "r6 /= 1", 200, 199)
diff --git a/tools/testing/selftests/bpf/progs/verifier_equations.c b/tools/testing/selftests/bpf/progs/verifier_equations.c
new file mode 100644
index 00000000000..e88d6ff0ca8
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_equations.c
@@ -0,0 +1,1583 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+/*
+ * The log has the first source line of a macro call.
+ * If __msg() is in that line, the log always matches it.
+ */
+#define BAD_INDEX __failure __msg("invalid variable-offset write to stack R1")
+#define BAD_WRITE __failure __msg("write to stack R1")
+#define BAD_PKT __failure __msg("invalid access to packet")
+
+#define ONE_TEST(name, verdict, size, body) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_ktime_get_ns];" \
+ "r6 = r0;" \
+ body \
+ "r1 = r10;" \
+ "r1 += -" #size ";" \
+ "r1 += r7;" \
+ "*(u8 *)(r1 + 0) = 0;" \
+"9:" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns) : __clobber_all); \
+}
+
+/* r6 is unknown. @body leaves an index in r7. It fits into @size bytes and not into @less. */
+#define INDEX_TEST(name, size, less, body) \
+ ONE_TEST(name, __success, size, body) \
+ ONE_TEST(name##_bad, BAD_INDEX, less, body)
+
+/* Another path is checked first. Its empty table does not make the chain of r7 shorter. */
+INDEX_TEST(chain_after_other_path, 8, 7,
+ "w6 = w6; r7 = r6; r7 >>= 1; r7 >>= 1; r7 >>= 1; r7 >>= 1;"
+ "call %[bpf_ktime_get_ns];"
+ "if r0 != 0 goto 2f;"
+ "r7 >>= 1; r7 >>= 1; r7 >>= 1;"
+ "if r6 > 1023 goto 9f;"
+ "goto 1f;"
+"2: goto +0;"
+ "r0 = 0;"
+ "exit;"
+"1:")
+
+/* The low 32 bits of the mask are -8 */
+ONE_TEST(and_mask_with_high_hole, BAD_WRITE, 8,
+ "r6 >>= 20; r2 = 0xfffffffefffffff8 ll;"
+ "r7 = r6; r7 &= r2;"
+ "if r6 > 400 goto 9f;")
+
+/*
+ * Another path is checked first. It ends with a refined sym0 that no jump propagated.
+ * The test of r9 refines r6, then r7.
+ */
+#define AFTER_OTHER_PATH(max, use) \
+ "w6 = w6; r8 = r6; r8 &= 3;" \
+ "r7 = r6; r7 += r8; w9 = w6; w9 += -1;" \
+ "call %[bpf_ktime_get_ns];" \
+ "if r0 != 0 goto 1f;" \
+ "goto 2f;" \
+"1:" \
+ "call %[bpf_ktime_get_ns];" \
+ "r2 = r0; r1 = r0; r1 *= 3;" \
+ "if r1 < 6 goto 9f;" \
+ "if r1 > 100 goto 9f;" \
+ "if r2 > " #max " goto 9f;" \
+ use \
+ "r0 = r1;" \
+ "exit;" \
+"2:" \
+ "if r9 > 58 goto 9f;" \
+ "*(u64 *)(r10 - 72) = r6;" \
+ "*(u64 *)(r10 - 80) = r8;"
+INDEX_TEST(after_constant, 63, 62, AFTER_OTHER_PATH(2, ""))
+INDEX_TEST(after_division, 63, 62, AFTER_OTHER_PATH(50, "r1 >>= 1; r1 += r2;"))
+
+/* a - b, b - c and c - a cannot all be 20 or more. The last jump shows that on both edges. */
+SEC("socket")
+__success __retval(0)
+__xlated("if r3 s< 0x14 goto pc+1")
+__xlated("goto pc-1")
+__xlated("r0 = 0")
+__naked void both_edges_are_infeasible(void)
+{
+ asm volatile (
+ "call %[bpf_ktime_get_ns];"
+ "r6 = r0; r6 &= 255;"
+ "r7 = r0; r7 >>= 8; r7 &= 255;"
+ "r8 = r0; r8 >>= 56;"
+ "r1 = r6; r1 -= r7;"
+ "r2 = r7; r2 -= r8;"
+ "r3 = r8; r3 -= r6;"
+ "if r1 s< 20 goto 9f;"
+ "if r2 s< 20 goto 9f;"
+ "if r3 s< 20 goto 9f;"
+ "if r6 > 128 goto 1f;"
+ "r1 += 1;"
+"1:"
+ "r0 = r1; r0 += r2; r0 += r3; r0 += r7; r0 += r8;"
+ "exit;"
+"9:"
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_ktime_get_ns) : __clobber_all);
+}
+
+__u64 words[8];
+
+/* e = p + 8 * n; while (e > p) s += *--e; */
+#define WALK_DOWN(name, verdict, off) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_ktime_get_ns];" \
+ "r0 &= 0x38;" \
+ "if r0 == 0 goto 9f;" \
+ "r1 = %[words] ll;" \
+ "r2 = r1;" \
+ "r2 += r0;" \
+"1:" \
+ "r0 = *(u64 *)(r2 " #off ");" \
+ "r2 += -8;" \
+ "if r2 > r1 goto 1b;" \
+"9:" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns), __imm_addr(words) : __clobber_all); \
+}
+
+WALK_DOWN(walk_down, __success, - 8)
+WALK_DOWN(walk_down_bad,
+ __failure __msg("invalid access to map value, value_size=64 off=-8"), - 16)
+
+/*
+ * Code with no 32-bit insns can test the carry of a sum: x != (u32)x.
+ * r7 is 2^32 - 8 .. 2^32 + 55. The index is 2^32 + 55 - r7.
+ */
+#define CARRY(change_low, test) \
+ "r8 = 0x100000000 ll; r7 = r6; r7 &= 63; r7 += r8; r7 += -8;" \
+ "r2 = r7; r2 <<= 32; r2 >>= 32;" change_low \
+ "if " test " goto 9f;" \
+ "r7 -= r8; r7 = -r7; r7 += 55;"
+INDEX_TEST(carry, 56, 55, CARRY("", "r2 == r7"))
+INDEX_TEST(carry_swapped, 56, 55, CARRY("w2 = w7;", "r7 == r2"))
+INDEX_TEST(no_carry, 64, 63, CARRY("", "r2 != r7"))
+ONE_TEST(carry_of_next_number_bad, BAD_INDEX, 56, CARRY("w2 += 1;", "r2 == r7"))
+ONE_TEST(carry_of_double_bad, BAD_INDEX, 56, CARRY("w2 += w2;", "r2 == r7"))
+ONE_TEST(carry_sign_extended_bad, BAD_INDEX, 56, CARRY("r2 <<= 32; r2 s>>= 32;", "r2 == r7"))
+ONE_TEST(carry_16_bits_bad, BAD_INDEX, 56, CARRY("r2 &= 0xffff;", "r2 == r7"))
+ONE_TEST(carry_31_bits_bad, BAD_INDEX, 56, CARRY("r2 &= 0x7fffffff;", "r2 == r7"))
+ONE_TEST(carry_32_bit_jump_bad, BAD_INDEX, 56, CARRY("w2 += 1;", "w2 == w7"))
+
+/* A negative number is above U32_MAX too. r7 is -8..7, then -8..-1. */
+INDEX_TEST(borrow, 8, 7,
+ "r7 = r6; r7 &= 15; r7 += -8; w2 = w7;"
+ "if r2 == r7 goto 9f;"
+ "r7 += 8;")
+
+/* r6 = 1 << 31 gives r3 = -(1 << 31), r2 = 1 << 31 and the index 8. r6 has no carry. */
+ONE_TEST(carry_of_extended_copy_bad, BAD_INDEX, 8,
+ "r3 = r6; r3 <<= 32; r3 s>>= 32; w2 = w6;"
+ "if r2 == r3 goto 9f;"
+ "r7 = r6; r7 >>= 32; if r7 != 0 goto 9f;"
+ "r7 = r6; r7 >>= 28;")
+
+/* r6 = 0x80001 gives r2 = 0x10003, r3 = 0x190003 and the index 8 */
+ONE_TEST(carry_of_16_bit_product_bad, BAD_INDEX, 8,
+ "r2 = r6; r2 &= 0xffff; r2 *= 0x10003;"
+ "w3 = w6; w3 *= 0x10003;"
+ "if r3 == r2 goto 9f;"
+ "if r2 != 0x10003 goto 9f;"
+ "r7 = r6; r7 >>= 16; r7 &= 15;")
+
+/* u8 n; if ((u8)(n - 1) > 99) return; a[n] */
+INDEX_TEST(low_8_bits_of_sum, 101, 100,
+ "r7 = r6; r7 &= 255;"
+ "w2 = w7; w2 += -1; w2 &= 255;"
+ "if w2 > 99 goto 9f;")
+
+INDEX_TEST(low_16_bits_of_sum, 101, 100,
+ "r7 = r6; r7 &= 0xffff;"
+ "w2 = w7; w2 += -1; w2 &= 0xffff;"
+ "if w2 > 99 goto 9f;")
+
+/* One number is truncated twice */
+INDEX_TEST(same_truncation_twice, 101, 100,
+ "w7 = w6; w7 &= 0xffff;"
+ "r6 &= 0xffff;"
+ "if r6 > 100 goto 9f;")
+
+/* The test of the low 16 bits tells the low 8 bits */
+INDEX_TEST(low_16_bits_tell_low_8_bits, 101, 100,
+ "r7 = r6; r7 &= 255;"
+ "r6 &= 0xffff;"
+ "if r6 > 100 goto 9f;")
+
+/* It does not work the other way */
+ONE_TEST(low_8_bits_tell_low_16_bits, BAD_INDEX, 256,
+ "r7 = r6; r7 &= 0xffff;"
+ "r6 &= 255;"
+ "if r6 > 100 goto 9f;")
+
+/* r8 has the same low 8 bits as r6. It is not a truncation. */
+INDEX_TEST(truncation_next_to_sum, 51, 50,
+ "call %[bpf_ktime_get_ns];"
+ "r0 <<= 8; r8 = r6; r8 += r0;"
+ "r7 = r6; r7 &= 255;"
+ "if r6 > 50 goto 9f;"
+ "if r8 == 7 goto 9f;")
+
+/* (u16)x and (s16)x */
+INDEX_TEST(truncation_with_sign, 101, 100,
+ "r7 = r6; r7 &= 0xffff;"
+ "r6 <<= 48; r6 s>>= 48;"
+ "if r6 s< 0 goto 9f;"
+ "if r6 s> 100 goto 9f;")
+
+/* (x + 7) & ~7 of a u16 */
+INDEX_TEST(and_rounds_up, 57, 56,
+ "w6 &= 0xffff;"
+ "w7 = w6; w7 += 7; w7 &= 0x1fff8;"
+ "if w6 > 50 goto 9f;")
+
+/* (x & 0xfff) * 4 */
+INDEX_TEST(and_after_shift, 61, 60,
+ "w7 = w6; w7 <<= 2; w7 &= 0x3ffc;"
+ "if w6 > 15 goto 9f;")
+
+INDEX_TEST(and_with_hole, 62, 61,
+ "w7 = w6; w7 &= 0xfd;"
+ "if w6 > 63 goto 9f;")
+
+INDEX_TEST(or_const, 64, 63,
+ "w7 = w6; w7 |= 1;"
+ "if w6 > 62 goto 9f;")
+
+INDEX_TEST(xor_const, 64, 63,
+ "r7 = r6; r7 ^= 1;"
+ "if r6 > 62 goto 9f;")
+
+/* The insn runs again after the test of its operand */
+INDEX_TEST(op_square, 101, 100,
+ "r7 = r6; r7 *= r6;"
+ "if r6 > 10 goto 9f;")
+
+INDEX_TEST(op_mod, 21, 20,
+ "r7 = r6; r7 %%= 100;"
+ "if r6 > 20 goto 9f;")
+
+INDEX_TEST(op_div_of_unknown, 34, 33,
+ "r7 = r6; r7 /= 3;"
+ "if r6 > 100 goto 9f;")
+
+INDEX_TEST(op_shift_by_number, 33, 32,
+ "r6 &= 7; r7 = 1; r7 <<= r6;"
+ "if r6 > 5 goto 9f;")
+
+INDEX_TEST(op_chain, 64, 27,
+ "r7 = r6; r7 ^= 1; r7 *= r7; r7 /= 16; r7 |= 1;"
+ "if r6 > 20 goto 9f;")
+
+/* Two insns make one number */
+INDEX_TEST(op_twice, 11, 10,
+ "r7 = r6; r7 ^= 0x55; r8 = r6; r8 ^= 0x55;"
+ "if r8 > 10 goto 9f;")
+
+ONE_TEST(op_other_constant, BAD_INDEX, 11,
+ "r6 &= 0xff; r7 = r6; r7 ^= 0x55; r8 = r6; r8 ^= 0x15;"
+ "if r8 > 10 goto 9f;")
+
+ONE_TEST(op_other_width, BAD_INDEX, 11,
+ "r6 &= 0xff; r6 <<= 28; r7 = r6; r7 /= 3; r8 = r6; w8 /= 3; r7 >>= 20;"
+ "if r8 > 10 goto 9f;")
+
+ONE_TEST(op_operand_is_written, BAD_INDEX, 101,
+ "r6 &= 0xf; r7 = r6; r7 *= r6; r6 = r0; r6 >>= 4;"
+ "if r6 > 10 goto 9f;")
+
+/* The insn reads the low bits only. A truncation that has them is tested. */
+INDEX_TEST(op_reads_truncation_made_later, 9, 8,
+ "r7 = r6; r7 &= 0xf8; r6 &= 0xff;"
+ "if r6 > 8 goto 9f;")
+
+INDEX_TEST(op_reads_truncation_made_before, 9, 8,
+ "r8 = r6; r8 &= 0xff; r7 = r6; r7 &= 0xf8;"
+ "if r8 > 8 goto 9f;")
+
+INDEX_TEST(op_reads_truncation_tested_before, 14, 13,
+ "r8 = r6; r8 &= 0xff;"
+ "if r8 > 0xdf goto 9f;"
+ "r7 = r6; r7 &= 0xf0; r7 >>= 4;")
+
+ONE_TEST(op_reads_more_than_was_tested, BAD_INDEX, 14,
+ "r8 = r6; r8 &= 0xff;"
+ "if r8 > 0xdf goto 9f;"
+ "r7 = r6; r7 &= 0x1f0; r7 >>= 4;")
+
+ONE_TEST(op_reads_more_than_truncation, BAD_INDEX, 9,
+ "r7 = r6; r7 &= 0x1f8; r6 &= 0xff;"
+ "if r6 > 8 goto 9f;")
+
+INDEX_TEST(mod_pow2, 51, 50,
+ "r7 = r6; r7 %%= 256;"
+ "if r6 > 50 goto 9f;")
+
+/* The low 32 bits of the divisor are 4 */
+ONE_TEST(mod_not_pow2, BAD_WRITE, 8,
+ "r2 = 0x100000004 ll;"
+ "r7 = r6; r7 %%= r2;"
+ "if w6 != 0 goto 9f;"
+ "w7 = w7;")
+
+INDEX_TEST(arsh32, 51, 50,
+ "w7 = w6; w7 s>>= 1;"
+ "if w6 s< 1 goto 9f;"
+ "if w6 s> 100 goto 9f;")
+
+INDEX_TEST(rsh32_of_wide_number, 51, 50,
+ "w7 = w6; w7 >>= 1;"
+ "if w6 > 100 goto 9f;")
+
+/* (u64)(x + 7) / 8 of an int. The sum is dead at the test. */
+INDEX_TEST(truncate_then_shift, 64, 63,
+ "w7 = w6; w7 += 7; r7 <<= 32; r7 s>>= 32; r7 >>= 3;"
+ "if w6 s< 1 goto 9f;"
+ "if w6 s> 500 goto 9f;")
+
+/* The same with a join point before the test */
+INDEX_TEST(truncate_then_shift_join, 64, 63,
+ "call %[bpf_ktime_get_ns];"
+ "if r0 > 5 goto 1f;"
+ "w7 = w6; w7 += 7; w7 >>= 3;"
+ "goto 2f;"
+"1:"
+ "r0 = 0;"
+ "w7 = w6; w7 += 7; w7 >>= 3;"
+"2:"
+ "if w6 > 500 goto 9f;")
+
+#define JOIN_POINT "call %[bpf_ktime_get_ns]; r8 = 1; if r0 != 0 goto 1f; r8 = 0; 1:"
+
+/* The join keeps the truncation below the shift */
+INDEX_TEST(and_then_join, 66, 65,
+ "w7 = w6; w7 &= 0xf8;"
+ JOIN_POINT
+ "if w6 > 64 goto 9f;"
+ "r7 += r8;")
+
+/* The join keeps the equation of an insn that both paths have */
+INDEX_TEST(op_then_join, 102, 101,
+ "r7 = r6; r7 *= r6;"
+ JOIN_POINT
+ "if r6 > 10 goto 9f;"
+ "r7 += r8;")
+
+ONE_TEST(op_on_one_path, BAD_WRITE, 101,
+ "r6 &= 0xf; r7 = r6; r7 *= r6;"
+ "call %[bpf_ktime_get_ns]; if r0 != 0 goto 1f; r7 = 200; 1:"
+ "if r6 > 10 goto 9f;")
+
+/* x - x / 4. The join keeps the range of the quotient. */
+INDEX_TEST(sub_quotient_then_join, 39, 31,
+ "r6 &= 0xffff;"
+ "r2 = r6; r2 >>= 2; r7 = r6; r7 -= r2;"
+ JOIN_POINT
+ "if r6 > 30 goto 9f;"
+ "r7 += r8; r7 += 7;")
+
+/* One path shifts another number */
+ONE_TEST(truncate_then_shift_other, BAD_INDEX, 64,
+ "call %[bpf_ktime_get_ns];"
+ "if r0 > 5 goto 1f;"
+ "w7 = w6; w7 += 7; w7 >>= 3;"
+ "goto 2f;"
+"1:"
+ "call %[bpf_ktime_get_ns];"
+ "w7 = w0; w7 += 7; w7 >>= 3;"
+"2:"
+ "if w6 > 500 goto 9f;")
+
+/* x / 3 of a u32 */
+INDEX_TEST(magic_division, 101, 100,
+ "w7 = w6; r1 = 0xaaaaaaab ll; r7 *= r1; r7 >>= 33;"
+ "if w6 > 300 goto 9f;")
+
+/* for (i = 0; i < 10000; i++) { a[h] = 0; h = next(h); } */
+#define RING_TEST(name, verdict, size, next) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1:" \
+ "r1 = r10;" \
+ "r1 += -" #size ";" \
+ "r1 += r7;" \
+ "*(u8 *)(r1 + 0) = 0;" \
+ next \
+ "r6 += 1;" \
+ "if r6 != 10000 goto 1b;" \
+ "r0 = 0;" \
+ "exit;" \
+ ::: __clobber_all); \
+}
+
+RING_TEST(ring_mask, __success, 64, "r7 += 1; r7 &= 63;")
+RING_TEST(ring_mask_bad, BAD_WRITE, 63, "r7 += 1; r7 &= 63;")
+RING_TEST(ring_step, __success, 57, "w7 += 8; w7 &= 63;")
+RING_TEST(ring_step_bad, BAD_WRITE, 56, "w7 += 8; w7 &= 63;")
+RING_TEST(ring_down, __success, 64, "w7 += -1; w7 &= 63;")
+RING_TEST(ring_does_not_wrap, BAD_WRITE, 64, "r7 += 1; r7 &= 0xfffff;")
+
+/* h = (h + 1) % 60 by a product. A path-sensitive run must not take h for an iv. */
+#define MOD_60 "w7 += 1; r2 = r7; r3 = 0x88888889 ll; r2 *= r3; r2 >>= 37; w2 *= 60; w7 -= w2;"
+
+RING_TEST(ring_modulo, __success, 60, MOD_60)
+RING_TEST(ring_modulo_bad, BAD_WRITE, 59, MOD_60)
+
+/* The range wraps in 64 bits. The sign extension keeps the low 32 bits. */
+INDEX_TEST(movsx_keeps_range32, 201, 200,
+ "if w6 s< -100 goto 9f;"
+ "if w6 s> 100 goto 9f;"
+ "r7 = (s32)r6;"
+ "r7 += 100;")
+
+/* The test of w6 - 1 goes back to r6, on to its low half and to the sum */
+#define SUM_OF_LOW_HALF \
+ "r6 <<= 51; r6 s>>= 51;" \
+ "if r6 s> 256 goto 9f;" \
+ "call %[bpf_ktime_get_ns];" \
+ "r0 &= 7;" \
+ "r7 = r6; r7 <<= 32; r7 >>= 32;" \
+ "r7 += r0;" \
+ "w6 += -1;" \
+ "if w6 > 63 goto 9f;"
+
+ONE_TEST(sum_of_low_half, __success, 72, SUM_OF_LOW_HALF)
+ONE_TEST(sum_of_low_half_bad, BAD_WRITE, 71, SUM_OF_LOW_HALF)
+
+#define INC4 "w6 += 1; w6 += 1; w6 += 1; w6 += 1;"
+
+/* The numbers in the middle of the chain are gone */
+INDEX_TEST(chain_of_adds32, 64, 63,
+ "w6 = w6; r7 = r6;"
+ INC4 INC4 INC4 INC4 INC4
+ "if w6 < 20 goto 9f;"
+ "if w6 > 83 goto 9f;")
+
+/* What enters an inner loop for the second time is joined and not widened */
+SEC("socket")
+__success __log_level(2) __not_msg("pass 1")
+__naked void enters_inner_loop_again(void)
+{
+ asm volatile (
+ "call %[bpf_ktime_get_ns];"
+ "r6 = r0;"
+ "call %[bpf_ktime_get_ns];"
+ "if r6 > 64 goto 9f;"
+ "if r6 == 0 goto 9f;"
+ "r7 = 0;"
+"1:"
+ "r1 = %[words] ll;"
+ "r1 += r7;"
+ "r1 = *(u8 *)(r1 + 0);"
+ "r2 = 0;"
+"2:"
+ "r2 += 1;"
+ "r3 = 0;"
+"3:"
+ "r3 += 1;"
+ "if r3 < r0 goto 3b;"
+ "if r2 < r0 goto 2b;"
+ "r7 += 1;"
+ "if r7 != r6 goto 1b;"
+"9:"
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_ktime_get_ns), __imm_addr(words) : __clobber_all);
+}
+
+/* while (!*p) { *p = n >> 4; n *= 3; } return *p; */
+__naked __noinline __used
+static void find_level(void)
+{
+ asm volatile (
+ "r2 = *(u64 *)(r1 + 0);"
+ "if r2 != 0 goto 2f;"
+ "r2 = 1;"
+"1:"
+ "r3 = r2;"
+ "r3 >>= 4;"
+ "*(u64 *)(r1 + 0) = r3;"
+ "r2 *= 3;"
+ "r3 = *(u64 *)(r1 + 0);"
+ "if r3 == 0 goto 1b;"
+"2:"
+ "r0 = *(u64 *)(r1 + 0);"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/* The leftover has no trip count. The loop ends in the peeled trips and needs no may_goto. */
+SEC("socket")
+__success
+__naked void leftover_without_trip_count(void)
+{
+ asm volatile (
+ "*(u64 *)(r10 - 8) = 0;"
+ "r1 = r10;"
+ "r1 += -8;"
+ "call find_level;"
+ "r1 = %[words] ll;"
+ "r1 += r0;"
+ "r0 = *(u8 *)(r1 - 1);"
+ "exit;"
+ :: __imm_addr(words) : __clobber_all);
+}
+
+/* r9 has any offset after the loop. The test of r7 tells where it is. r5 is 16 bytes further. */
+#define FAR_PKT(name, verdict, tested) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r9 = *(u32 *)(r1 + %[data]);" \
+ "r8 = *(u32 *)(r1 + %[data_end]);" \
+ "r7 = 0;" \
+"1:" \
+ "r7 += 1;" \
+ "r9 += 8;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 != 0 goto 1b;" \
+ "r5 = r9;" \
+ "r5 += 16;" \
+ "if r7 > 10 goto 9f;" \
+ "r4 = " #tested ";" \
+ "r4 += 8;" \
+ "if r4 > r8 goto 9f;" \
+ "r0 = *(u64 *)(r5 + 0);" \
+"9:" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm(bpf_get_prandom_u32) \
+ : __clobber_all); \
+}
+
+FAR_PKT(far_pkt_comes_back, __success, r5)
+FAR_PKT(far_pkt_other_is_tested, BAD_PKT, r9)
+
+/* n = (len + 7) & -8; if (n == 0 || len < -7) return; a[64 - n] */
+#define ROUND_UP(not) \
+ "r6 &= 127; r6 += -63;" \
+ "r2 = r6; r2 += 7; r2 &= -8;" \
+ "if r2 == " #not " goto 9f;" \
+ "if r6 s< -7 goto 9f;" \
+ "if r2 s> 64 goto 9f;" \
+ "r7 = 64; r7 -= r2;"
+
+INDEX_TEST(not_zero_then_not_negative, 57, 56, ROUND_UP(0))
+ONE_TEST(not_8_then_not_negative, BAD_INDEX, 64, ROUND_UP(8))
+
+/* struct { u32 i; u32 flag; } */
+INDEX_TEST(write_next_to_spill, 8, 7,
+ "*(u32 *)(r10 - 128) = r6;"
+ "*(u32 *)(r10 - 124) = 1;"
+ "r2 = *(u32 *)(r10 - 128);"
+ "if r2 > 7 goto 9f;"
+ "r7 = *(u32 *)(r10 - 128);")
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+INDEX_TEST(write_over_high_half_of_spill, 8, 7,
+ "r6 &= 7;"
+ "*(u64 *)(r10 - 128) = r6;"
+ "*(u32 *)(r10 - 124) = -1;"
+ "r7 = *(u32 *)(r10 - 128);")
+#endif
+
+ONE_TEST(write_over_high_half_read_all,
+ __failure __msg("math between fp pointer"), 8,
+ "r6 &= 7;"
+ "*(u64 *)(r10 - 128) = r6;"
+ "*(u32 *)(r10 - 124) = -1;"
+ "r7 = *(u64 *)(r10 - 128);")
+
+ONE_TEST(write_into_low_half_of_spill, BAD_WRITE, 8,
+ "r6 &= 7;"
+ "*(u64 *)(r10 - 128) = r6;"
+ "*(u8 *)(r10 - 125) = 1;"
+ "r7 = *(u32 *)(r10 - 128);")
+
+/* Records in a packet. Each of three groups has short ones, then long ones with a size. */
+#define RECORDS(name, verdict, ahead) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r7 = *(u32 *)(r1 + %[data]);" \
+ "r8 = *(u32 *)(r1 + %[data_end]);" \
+ "r9 = *(u32 *)(r1 + %[mark]);" \
+ "r6 = 0;" \
+"1:" "r2 = 0;" \
+"2:" "if r9 == 1 goto 3f;" \
+ "if r9 != 2 goto 4f;" \
+"3:" "r1 = r7; r1 += r2;" \
+ "r3 = r1; r3 += 4;" \
+ "if r3 > r8 goto 4f;" \
+ "r9 = *(u8 *)(r1 + 3);" \
+ "r2 += 4;" \
+ "if r2 != 64 goto 2b;" \
+"4:" "r7 += r2;" \
+ "r1 = r7; r1 += 2;" \
+ "if r1 > r8 goto 7f;" \
+ "r2 = 0;" \
+"5:" "r1 = r7;" \
+ "if r9 == 3 goto 6f;" \
+ "if r9 != 4 goto 7f;" \
+ "r3 = *(u8 *)(r7 + 1); r3 <<= 3;" \
+ "r1 += r3;" \
+"6:" "r9 = *(u8 *)(r7 + 0);" \
+ "r7 = r1; r7 += 8;" \
+ "if r2 > 4 goto 7f;" \
+ "r2 += 1;" \
+ "r1 += " #ahead ";" \
+ "if r1 <= r8 goto 5b;" \
+"7:" "if r9 != 5 goto 9f;" \
+ "r6 += 1;" \
+ "if r6 != 3 goto 1b;" \
+"9:" "r0 = 0;" \
+ "exit;" \
+ :: __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm_const(mark, offsetof(struct __sk_buff, mark)) \
+ : __clobber_all); \
+}
+
+RECORDS(records_in_groups, __success, 10)
+RECORDS(records_in_groups_bad, BAD_PKT, 9)
+
+/* for (s16 i = -4; i < n; i++) a[i + 4]. The counter is not truncated to 16 bits. */
+#define S16_COUNTER \
+ "w6 &= 255; w6 += -43;" \
+ "w3 = 0xfffc; w2 = -4; r7 = 0;" \
+ "if w6 s<= w2 goto 9f;" \
+"1:" "w1 = w2; w1 += 4;" \
+ "r7 = (s32)r1;" \
+ "r4 = r10; r4 += -256; r4 += r7; *(u8 *)(r4 + 0) = 0;" \
+ "w3 += 1; w2 = (s16)w3;" \
+ "if w6 s> w2 goto 1b;"
+
+ONE_TEST(s16_counter_through_zero,
+ __success __log_level(2) __not_msg("pass 1") __msg("has trip count 216")
+ __not_msg("pass 1"),
+ 216, S16_COUNTER)
+ONE_TEST(s16_counter_through_zero_bad, BAD_WRITE, 215, S16_COUNTER)
+
+#define MAY_GOTO(off) ".byte 0xe5, 0; .short " #off "; .long 0;"
+
+/* s16 at = -1; for (i = 0; i < 400; i++) if (x > best) best = x, at = i; if (at >= 0) a[at] */
+#define INDEX_OF_BEST \
+ "w7 = 0xffff; w6 = 0; r8 = 0;" \
+"1:" MAY_GOTO(11) \
+ "call %[bpf_ktime_get_ns];" \
+ "r0 &= 255;" \
+ "w5 = w8;" \
+ "if w0 > w6 goto 2f;" \
+ "w5 = w7;" \
+"2:" "if w0 > w6 goto 3f;" \
+ "w0 = w6;" \
+"3:" "w7 = w5; w6 = w0;" \
+ "r8 += 1;" \
+ "if r8 != 400 goto 1b;" \
+ "w2 = w7; w2 <<= 16; w2 s>>= 16;" \
+ "if w2 s< 0 goto 9f;" \
+ "r7 &= 0xffff;"
+
+ONE_TEST(index_of_best, __success, 400, INDEX_OF_BEST)
+ONE_TEST(index_of_best_bad, BAD_WRITE, 399, INDEX_OF_BEST)
+
+/* The slot gets a copy of r7, or r7 moves. These numbers decide a jump and nothing else. */
+#define COPY_OR_ADD(slot) \
+ "call %[bpf_ktime_get_ns];" \
+ "r1 = 0;" \
+ "if r0 == 1 goto +1;" \
+ "r1 = r7;" \
+ "*(u64 *)(r10 - " #slot ") = r1;" \
+ "if r0 != 1 goto +1;" \
+ "r7 += r8;"
+#define XOR_SLOT(slot) "r1 = *(u64 *)(r10 - " #slot "); r7 ^= r1;"
+#define SLOTS2(f, n) f(0x##n##0) f(0x##n##8)
+#define SLOTS8(f, a, b, c, d) SLOTS2(f, a) SLOTS2(f, b) SLOTS2(f, c) SLOTS2(f, d)
+#define SLOTS(fn) \
+ SLOTS8(fn, 4, 5, 6, 7) SLOTS8(fn, 8, 9, a, b) SLOTS8(fn, c, d, e, f) \
+ SLOTS8(fn, 10, 11, 12, 13) SLOTS8(fn, 14, 15, 16, 17) SLOTS8(fn, 18, 19, 1a, 1b)
+#define COPIES_IN_CONTROL_DEPS \
+ "call %[bpf_ktime_get_ns];" \
+ "r7 = r0; r8 = r0; r8 &= 255;" \
+ SLOTS(COPY_OR_ADD) SLOTS(XOR_SLOT) \
+ "r1 = 0;" \
+ "if r6 < 16 goto +1;" \
+ "r1 = 1;" \
+ "if r7 == 3 goto 9f;" \
+ "if r1 != 0 goto 9f;" \
+ "r7 = r6;"
+
+ONE_TEST(copies_in_control_deps,
+ __success __log_level(4) __msg("processed {{[0-9][0-9][0-9][0-9]}} insns"),
+ 16, COPIES_IN_CONTROL_DEPS)
+ONE_TEST(copies_in_control_deps_bad, BAD_INDEX, 15, COPIES_IN_CONTROL_DEPS)
+
+/* Three ways into a nest of loops. The sum of r7 and r8 stays as it is. */
+#define IRREDUCIBLE_SUM \
+ "r8 = r6; r8 &= 255; r7 = 0;" \
+ "if r8 == 0 goto 9f;" \
+ "r8 += -1;" \
+ "if r6 s< 0 goto 5f;" \
+"1:" "call %[bpf_ktime_get_ns];" \
+ "r9 = 0;" \
+ "if r0 == 1 goto 6f;" \
+"2:" "call %[bpf_ktime_get_ns];" \
+"3:" "call %[bpf_ktime_get_ns];" \
+ "if r0 == 2 goto 4f;" \
+ "r9 = r0; r9 &= 7; r9 += 4;" \
+"4:" "if r9 > r0 goto 7f;" \
+ "if r8 == 0 goto 7f;" \
+ "r1 = r10; r1 += -256; r1 += r7; *(u8 *)(r1 + 1) = 0;" \
+ "r8 += -1; r7 += 1;" \
+ "if r0 > 3 goto 1b;" \
+"5:" "call %[bpf_ktime_get_ns];" \
+ "r9 = 7;" \
+ "if r0 != 4 goto 2b;" \
+"6:" "goto 3b;" \
+"7:"
+
+ONE_TEST(sum_in_irreducible_loops,
+ __success __log_level(4) __msg("processed {{[0-9][0-9][0-9]}} insns"),
+ 255, IRREDUCIBLE_SUM)
+ONE_TEST(sum_in_irreducible_loops_bad, BAD_WRITE, 254, IRREDUCIBLE_SUM)
+
+__u8 bytes[65536] SEC(".bss.bytes");
+
+/* for (o = 0; o < 3; o++) for (i = a; i != b; i++) bytes[k++] */
+#define WALK_TO_RANGE(name, verdict, end) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_ktime_get_ns];" \
+ "r1 = r0; r1 &= 31;" \
+ "r2 = r0; r2 >>= 63; r2 += " #end ";" \
+ "r3 = %[bytes] ll; r4 = 0;" \
+"1:" "if r1 == r2 goto 3f;" \
+ "r5 = r1;" \
+"2:" "r0 = *(u8 *)(r3 + 0);" \
+ "r3 += 1; r5 += 1;" \
+ "if r5 != r2 goto 2b;" \
+"3:" "r4 += 1;" \
+ "if r4 != 3 goto 1b;" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns), __imm_addr(bytes) : __clobber_all); \
+}
+
+WALK_TO_RANGE(walk_to_range, __success, 21844)
+WALK_TO_RANGE(walk_to_range_bad,
+ __failure __msg("invalid access to map value"), 21845)
+
+/* c = x != 2 tells nothing about x that can replace x != 0 */
+#define COND_THEN_ZERO(k) \
+ "r2 = 0;" \
+ "if r6 == 2 goto 1f;" \
+ "r2 = 1;" \
+"1:" "if r2 != 0 goto 9f;" \
+ "r7 = 0;" \
+ "if r6 == 0 goto 9f;" \
+ "if r6 != " #k " goto 2f;" \
+ "r7 = 8;" \
+"2:"
+
+ONE_TEST(cond_then_zero,
+ __success __log_level(2) __not_msg("pass 1"),
+ 8, COND_THEN_ZERO(0))
+ONE_TEST(cond_then_zero_bad, BAD_WRITE, 8, COND_THEN_ZERO(2))
+
+/* idx = (int)x < 0 ? 100 : 0; if ((int)x < @k) return; Only the low 32 bits tell the path. */
+#define COND_OF_LOW_HALF(k) \
+ "r7 = 0;" \
+ "if w6 s> -1 goto 1f;" \
+ "r7 = 100;" \
+"1:" "call %[bpf_ktime_get_ns];" \
+ "if w6 s< " #k " goto 9f;"
+
+ONE_TEST(cond_of_low_half,
+ __success __log_level(2) __not_msg("pass 1"),
+ 8, COND_OF_LOW_HALF(0))
+ONE_TEST(cond_of_low_half_bad, BAD_WRITE, 8, COND_OF_LOW_HALF(-1))
+
+/* r6 is 0 or -1. The first AND is checked for each value. */
+#define TWO_ANDS(k) \
+ "w6 &= 1; w6 += -1;" \
+ "w2 = w6; w2 &= 255;" \
+ "if w2 > " #k " goto 9f;" \
+ "r7 = r6; r7 &= 255;"
+
+ONE_TEST(and_of_two_values, __success, 8, TWO_ANDS(7))
+ONE_TEST(and_of_two_values_bad, BAD_WRITE, 8, TWO_ANDS(255))
+
+/* The jump compares a number with itself. It gives no trip count. */
+ONE_TEST(jump_that_does_not_count,
+ __failure __msg("invalid write to stack R8 off=0 size=1"), 8,
+ "r7 = 0; r8 = r10; r8 += -24; r9 = r10; r9 += -256;"
+"1:" "r0 = 0;"
+"2:" MAY_GOTO(2)
+ "r9 += 1;"
+ "if r9 == r9 goto 2b;"
+ "*(u8 *)(r8 + 0) = r7;"
+ "r8 += 8;"
+ "if r7 s> r9 goto 1b;")
+
+/* struct { u32 a; u8 b, c; } */
+#define ZERO_BYTE \
+ "*(u32 *)(r10 - 128) = 5;" \
+ "*(u8 *)(r10 - 124) = 0;" \
+ "*(u8 *)(r10 - 123) = r6;" \
+ "*(u16 *)(r10 - 126) = r6;" \
+ "r7 = *(u8 *)(r10 - 124);"
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ONE_TEST(zero_byte_next_to_spill, __success, 1, ZERO_BYTE)
+#endif
+ONE_TEST(zero_byte_next_to_spill_bad, BAD_WRITE, 0, ZERO_BYTE)
+
+ONE_TEST(other_byte_next_to_spill, BAD_INDEX, 1,
+ "*(u32 *)(r10 - 128) = 5;"
+ "*(u8 *)(r10 - 124) = 0;"
+ "*(u8 *)(r10 - 123) = r6;"
+ "r7 = *(u8 *)(r10 - 123);")
+
+/* s8 j = x << 4; a[j + 150]: j is -128..112. 0..0xf0 << 56 crosses S64_MAX. */
+INDEX_TEST(arsh_of_range_over_smax, 263, 262,
+ "if r6 > 15 goto 9f;"
+ "r7 = r6; r7 <<= 4; r7 <<= 56; r7 s>>= 56; r7 += 150;"
+ "*(u64 *)(r10 - 272) = r6;")
+
+INDEX_TEST(arsh32_of_range_over_smax, 263, 262,
+ "if r6 > 15 goto 9f;"
+ "w7 = w6; w7 <<= 4; w7 <<= 24; w7 s>>= 24; w7 += 150;")
+
+/* for (i = 5 * a; i != 5 * b; i += 5) k++; a is 0..31, b is 40..71 */
+#define STEP_5 \
+ "r2 = r6; r2 &= 31; r2 *= 5;" \
+ "r3 = r6; r3 >>= 8; r3 &= 31; r3 *= 5; r3 += 200;" \
+ "r7 = 0;" \
+ "1: r7 += 1; r2 += 5; if r2 != r3 goto 1b;"
+
+ONE_TEST(distance_has_step, __success, 72, STEP_5)
+ONE_TEST(distance_has_step_bad, BAD_WRITE, 71, STEP_5)
+
+/* 3 * a + 6 * b is a multiple of 3, 3 * a + 2 * b is not */
+#define SUM_OF_STEPS(mul) \
+ "r2 = r6; r2 &= 7; r2 *= 3;" \
+ "r7 = r6; r7 >>= 8; r7 &= 7; r7 *= " #mul ";" \
+ "r7 += r2; if r7 > 20 goto 9f;"
+
+INDEX_TEST(sum_has_step, 19, 18, SUM_OF_STEPS(6))
+ONE_TEST(sum_has_no_step, BAD_INDEX, 20, SUM_OF_STEPS(2))
+
+/* -3 * (3 * a + 1) is 6 + 9 * k */
+INDEX_TEST(negative_product_has_step, 59, 58,
+ "r7 = r6; r7 &= 7; r7 *= 3; r7 += 1; r7 *= -3;"
+ "r7 += 70; if r7 > 66 goto 9f;")
+
+/* Both are multiples of 3. The sum is truncated to 32 bits: 2, 5, .. 20. */
+ONE_TEST(sum_of_steps_wraps, BAD_INDEX, 20,
+ "r2 = r6; r2 &= 7; r2 *= 3;"
+ "r7 = r6; r7 >>= 8; r7 &= 1; r7 *= 3; r3 = 0xfffffffc ll; r7 += r3;"
+ "w7 += w2; if r7 > 20 goto 9f;")
+
+/* for (o = 0; o < 3; o++) { int i = -6 + a; do k++; while (++i != b); } b is -1 or 0 */
+#define TO_ZERO \
+ "w2 = w6; w2 &= 1; w2 += -6;" \
+ "w3 = w6; w3 >>= 1; w3 &= 1; w3 += -1;" \
+ "r7 = 0; w4 = 0;" \
+ "1: w5 = w2;" \
+ "2: r7 += 1; w5 += 1; if w5 != w3 goto 2b;" \
+ "w4 += 1; if w4 != 3 goto 1b;"
+
+ONE_TEST(int_goes_to_zero, __success, 19, TO_ZERO)
+ONE_TEST(int_goes_to_zero_bad, BAD_WRITE, 18, TO_ZERO)
+
+/* int i = -9 + a; do k++; while (++i != b); b is 2000000 or one more */
+#define THROUGH_ZERO \
+ "w2 = w6; w2 &= 1; w2 += -9;" \
+ "w3 = w6; w3 >>= 1; w3 &= 1; w3 += 2000000;" \
+ "r7 = 0;" \
+ "1: r7 += 1; w2 += 1; if w2 != w3 goto 1b;" \
+ "r7 >>= 15;"
+
+ONE_TEST(int_goes_through_zero, __success, 62, THROUGH_ZERO)
+ONE_TEST(int_goes_through_zero_bad, BAD_WRITE, 61, THROUGH_ZERO)
+
+/* r2 <= r3 <= r4 <= r5. The test of r5 goes down the chain, also after a join point. */
+#define ADD_BYTE(to, from) \
+ "r6 >>= 8; r1 = r6; r1 &= 255;" #to " = " #from ";" #to " += r1;"
+INDEX_TEST(chain_of_sums_then_join, 41, 40,
+ "r2 = r6; r2 &= 255;" ADD_BYTE(r3, r2) ADD_BYTE(r4, r3) ADD_BYTE(r5, r4)
+ "r8 = 0; if r3 > 100 goto 1f; r8 = 1; 1:"
+ "if r5 > 40 goto 9f;"
+ "r7 = r2; if r3 > r4 goto 9f; if r8 > r4 goto 9f;")
+
+/*
+ * The range of r1 is lost, an equation tells it at the header of the loop. There r0 gets a new
+ * id. It is another pointer, so the test of r0 tells nothing about r1.
+ */
+#define NEW_ID_TEST(name, verdict, use) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r6 = r1;" \
+ "call %[bpf_ktime_get_ns];" \
+ "r9 = r0; r9 >>= 2;" \
+ "r3 = *(u32 *)(r6 + %[mark]); r8 = r3;" \
+ "r3 >>= 26; r8 &= 0x7fff;" \
+ "r5 = *(u32 *)(r6 + %[data]);" \
+ "r4 = *(u32 *)(r6 + %[data_end]);" \
+ "r2 = r9; r2 += r3;" \
+ "r1 = r5; r1 += r2; r1 -= r9; r1 += r9; r1 -= r9;" \
+ "r7 = 0;" \
+"1:" "r0 = r5; r0 += r8;" \
+ "if r0 > r4 goto 9f;" \
+ "if r7 == 0 goto 2f;" \
+ use \
+"2:" "r7 += 1;" \
+ "if r7 < 2 goto 1b;" \
+ "r0 = r3;" \
+ "exit;" \
+"9:" "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns), \
+ __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm_const(mark, offsetof(struct __sk_buff, mark)) \
+ : __clobber_all); \
+}
+
+NEW_ID_TEST(new_id_at_header, __success,
+ "r0 = r1; r0 += 8; if r0 > r4 goto 9f; r0 = *(u8 *)(r1 + 7);")
+NEW_ID_TEST(new_id_at_header_bad, BAD_PKT __log_level(2), "r0 = *(u8 *)(r1 + 7);")
+
+/*
+ * r7 is 12 * r6. Both edges of the jump before the header go to the header. One state is
+ * widened and loses the checked bytes of r2. Then it is joined with the other state.
+ */
+SEC("tc")
+__success __log_level(2) __not_msg("pass 1")
+__naked void join_of_widened_state_is_refined(void)
+{
+ asm volatile (
+ "r2 = *(u32 *)(r1 + %[data]);"
+ "r3 = *(u32 *)(r1 + %[data_end]);"
+ "r4 = r2; r4 += 16;"
+ "if r4 > r3 goto 9f;"
+ "r5 = 0; r6 = 0; r7 = 0;"
+ "goto 3f;"
+"1:" "r2 += 1;"
+ "goto 3f;"
+"2:" "r2 += 16;"
+ "r4 = r2; r4 += 16;"
+ "if r4 > r3 goto +0;"
+"3:" "r1 = r10; r1 += -480; r1 += r7;"
+ "r0 = *(u8 *)(r1 + 0);"
+ "r5 += r0;"
+ "r6 += 1; r7 += 12;"
+ "if r6 >= 40 goto 8f;"
+ "if r6 >= 2 goto 2b;"
+ "goto 1b;"
+"8:" "if r2 > r3 goto 9f;"
+ "r0 = r5;"
+ "exit;"
+"9:" "r0 = 0;"
+ "exit;"
+ :: __imm_const(data, offsetof(struct __sk_buff, data)),
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end))
+ : __clobber_all);
+}
+
+/* r8 goes away by @far and comes back. The test of r2 refines the offsets in between. */
+#define AWAY_AND_BACK(name, verdict, far, off) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r2 = *(u32 *)(r1 + %[mark]); r2 &= 255;" \
+ "r5 = *(u32 *)(r1 + %[data_end]);" \
+ "r9 = *(u32 *)(r1 + %[data]);" \
+ "r9 += r2;" \
+ "r8 = r9; r8 += 8;" \
+ "if r8 > r5 goto 9f;" \
+ "r4 = " #far " ll;" \
+ "r8 = r9; r8 += r4;" \
+ "if r2 > 100 goto 9f;" \
+ "r8 -= r4;" \
+ "r0 = *(u8 *)(r8 + " #off ");" \
+"9:" "r0 = 0;" \
+ "exit;" \
+ :: __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm_const(mark, offsetof(struct __sk_buff, mark)) \
+ : __clobber_all); \
+}
+
+AWAY_AND_BACK(near_pointer_keeps_range, __success, 1000, 7)
+AWAY_AND_BACK(near_pointer_keeps_range_bad, BAD_PKT, 1000, 8)
+AWAY_AND_BACK(past_64k_keeps_range, __success, 70000, 7)
+AWAY_AND_BACK(past_64k_keeps_range_bad, __failure __msg("invalid access to packet"), 70000, 8)
+AWAY_AND_BACK(before_64k_keeps_range, __success, -70000, 7)
+AWAY_AND_BACK(far_pointer_loses_range, BAD_PKT, 0x7fffffffffffffc0, 100)
+
+/* r9 comes back and is tested, its far copy in r8 comes back later */
+SEC("tc")
+__failure __msg("invalid access to packet")
+__naked void far_copy_gets_no_range(void)
+{
+ asm volatile (
+ "r2 = *(u32 *)(r1 + %[mark]); r2 &= 255;"
+ "r5 = *(u32 *)(r1 + %[data_end]);"
+ "r9 = *(u32 *)(r1 + %[data]);"
+ "r9 += r2;"
+ "r4 = 0x7fffffffffffffc0 ll;"
+ "r9 += r4;"
+ "r8 = r9;"
+ "r9 -= r4;"
+ "r9 += 8;"
+ "if r9 > r5 goto 9f;"
+ "if r2 > 100 goto 9f;"
+ "*(u64 *)(r10 - 8) = r9;"
+ "r8 -= r4;"
+ "r0 = *(u8 *)(r8 + 100);"
+"9:" "r0 = 0;"
+ "exit;"
+ :: __imm_const(data, offsetof(struct __sk_buff, data)),
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)),
+ __imm_const(mark, offsetof(struct __sk_buff, mark))
+ : __clobber_all);
+}
+
+/* The two sides of the last jump can know another largest offset for r7 */
+#define RANGE_OF_BRANCH(name, verdict, off) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r9 = r1;" \
+ "call %[bpf_ktime_get_ns];" \
+ "r2 = r0;" \
+ "r7 = *(u32 *)(r9 + %[data]);" \
+ "r8 = *(u32 *)(r9 + %[data_end]);" \
+ "r3 = r2; r3 *= 3;" \
+ "*(u64 *)(r10 - 40) = r2;" \
+ "if r3 > 20000 goto 9f;" \
+ "r4 = r3; r4 *= 2;" \
+ "r2 = r3; r2 *= 3;" \
+ "if r4 > 2000 goto 9f;" \
+ "if r2 < 30 goto 9f;" \
+ "r5 = *(u8 *)(r9 + %[cb]);" \
+ "r6 = *(u8 *)(r9 + %[cb] + 1);" \
+ "r0 = r3; r0 += r5; *(u64 *)(r10 - 8) = r0;" \
+ "r1 = r3; r1 += r6; *(u64 *)(r10 - 16) = r1;" \
+ "r3 = r1; r3 /= 2; *(u64 *)(r10 - 24) = r3;" \
+ "r1 -= r0;" \
+ "if r1 s> 0 goto 9f;" \
+ "r0 -= r3;" \
+ "if r0 s> 20 goto 9f;" \
+ "r5 *= 2; r5 += r4;" \
+ "r7 += r5;" \
+ "r6 *= 3; r6 += r2;" \
+ "r6 /= 5;" \
+ "if r7 < r8 goto 1f;" \
+ "goto 9f;" \
+"1:" "r0 = *(u8 *)(r7 + " #off ");" \
+ "r1 = *(u64 *)(r10 - 8); r0 += r1;" \
+ "r1 = *(u64 *)(r10 - 16); r0 += r1;" \
+ "r1 = *(u64 *)(r10 - 24); r0 += r1;" \
+ "r1 = *(u64 *)(r10 - 40); r0 += r1;" \
+ "r0 += r6; r0 += r4; r0 += r2;" \
+ "exit;" \
+"9:" "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns), \
+ __imm_const(cb, offsetof(struct __sk_buff, cb)), \
+ __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)) \
+ : __clobber_all); \
+}
+
+RANGE_OF_BRANCH(range_for_taken_branch, __success, 0)
+RANGE_OF_BRANCH(range_for_taken_branch_bad, BAD_PKT, 500)
+
+/* r8 = r9 - 200. The join widens the offsets of r9 only. The loop makes more than two trips. */
+SEC("tc")
+__failure __msg("write to stack R1")
+__naked void far_pointers_share_no_id(void)
+{
+#define FAR_PAIR \
+ "call %[bpf_ktime_get_ns];" \
+ "r4 = 0x4000000000000010 ll;" \
+ "if r0 > r4 goto 9f;" \
+ "r1 = *(u64 *)(r10 - 32);" \
+ "r0 &= r1;" \
+ "*(u64 *)(r10 - 24) = r0;" \
+ "r9 = *(u32 *)(r6 + %[data]);" \
+ "r9 += 100; r9 += r0; r9 += r0;" \
+ "r8 = r9; r8 += -200;"
+ asm volatile (
+ "r6 = r1; r7 = 0;"
+ "*(u64 *)(r10 - 8) = 0;"
+ "*(u64 *)(r10 - 16) = 0;"
+ "*(u64 *)(r10 - 32) = -16;"
+ FAR_PAIR
+"1:" "r0 = *(u32 *)(r6 + %[mark]);"
+ "if r0 == 0 goto 2f;"
+ "r2 = *(u64 *)(r10 - 24);"
+ "if r2 != 48 goto 9f;"
+ "*(u64 *)(r10 - 40) = r8;"
+ "*(u64 *)(r10 - 40) = r9;"
+ "r7 = *(u64 *)(r10 - 8);"
+ "r0 = *(u64 *)(r10 - 16);"
+ "*(u64 *)(r10 - 8) = r0;"
+ "*(u64 *)(r10 - 16) = 1000;"
+ "r1 = *(u64 *)(r10 - 32);"
+ "r1 s>>= 1;"
+ "*(u64 *)(r10 - 32) = r1;"
+ FAR_PAIR
+ "goto 1b;"
+"2:" "r1 = r10; r1 += -48; r1 += r7;"
+ "*(u8 *)(r1 + 0) = 0;"
+"9:" "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_ktime_get_ns),
+ __imm_const(data, offsetof(struct __sk_buff, data)),
+ __imm_const(mark, offsetof(struct __sk_buff, mark))
+ : __clobber_all);
+#undef FAR_PAIR
+}
+
+/* The stored condition tells how much of the packet is there */
+#define CHECKED_TEST(name, verdict, last) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r5 = *(u32 *)(r1 + %[data]);" \
+ "r4 = *(u32 *)(r1 + %[data_end]);" \
+ "r8 = *(u32 *)(r1 + %[mark]); r3 = r8; r3 &= 127;" \
+ "if r8 > 1000 goto 1f;" \
+ "r0 = r5; r0 += 54; if r0 > r4 goto 9f;" \
+ "r7 = 1; r5 += r3;" \
+ "goto 2f;" \
+"1:" "r0 = r5; r0 += 14; if r0 > r4 goto 9f;" \
+ "r7 = 0; r5 += r3;" \
+"2:" "if r7 == 0 goto 9f;" \
+ "if r3 > " #last " goto 9f;" \
+ "r0 = *(u8 *)(r5 + 0);" \
+"9:" "r0 = 0;" \
+ "exit;" \
+ :: __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm_const(mark, offsetof(struct __sk_buff, mark)) \
+ : __clobber_all); \
+}
+
+CHECKED_TEST(cond_tells_what_is_checked, __success, 53)
+CHECKED_TEST(cond_tells_what_is_checked_bad, BAD_PKT, 54)
+
+/* The two paths have the same ranges. The one that has no equation is the failing path. */
+#define COPY_OR_NOT(other) \
+ "r6 &= 255; call %[bpf_ktime_get_ns]; r8 = r0; r8 &= 255;" \
+ "call %[bpf_ktime_get_ns];" \
+ "if r0 > 5 goto 1f;" \
+ "r7 = " other "; goto 2f;" \
+"1:" "r7 = r6;" \
+"2:" "if r6 > 7 goto 9f;"
+
+ONE_TEST(paths_have_an_equation, __success, 8, COPY_OR_NOT("r6"))
+ONE_TEST(one_path_has_no_equation,
+ __failure __log_level(2) __msg("path-sensitive run after pass 0")
+ __msg("write to stack R1"),
+ 8, COPY_OR_NOT("r8"))
+
+/* The test is on one extension of a number, the index is another one */
+INDEX_TEST(shifts_extend_twice, 101, 100,
+ "w2 = w6; w2 <<= 24; w2 s>>= 24;"
+ "w7 = w6; w7 <<= 24; w7 s>>= 24;"
+ "if r2 > 100 goto 9f;")
+
+INDEX_TEST(shifts_extend_what_is_tested, 101, 100,
+ "w7 = w6; w7 <<= 16; w7 s>>= 16;"
+ "w2 = w6;"
+ "if w2 s> 100 goto 9f;"
+ "if w2 s< 3 goto 9f;")
+
+/* The shifts are not the same: bit 7 is the sign in one of them */
+ONE_TEST(shifts_differ, BAD_WRITE, 128,
+ "w2 = w6; w2 <<= 16; w2 s>>= 16;"
+ "w7 = w6; w7 <<= 24; w7 s>>= 24;"
+ "if r2 > 200 goto 9f;")
+
+INDEX_TEST(extend_to_64_and_32, 132, 131,
+ "r7 = (s8)r6; w2 = (s8)w6;"
+ "if w2 s> 3 goto 9f;"
+ "r7 += 128;")
+
+/* The sign is one of the bits that the test tells */
+INDEX_TEST(extend_known_sign, 128, 100,
+ "r2 = r6; r2 &= 65535; r7 = (s16)r6;"
+ "if r2 s> 100 goto 9f;")
+
+/* Headers of 8 bytes, or with a size. Each one tells the type of the next one. */
+#define TWO_STRIDES(name, verdict, off) \
+SEC("tc") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r2 = *(u32 *)(r1 + %[data_end]);" \
+ "r5 = *(u32 *)(r1 + %[mark]);" \
+ "r1 = *(u32 *)(r1 + %[data]);" \
+ "r3 = 50;" \
+"1:" "r4 = r1; r4 += 8;" \
+ "if r4 > r2 goto 9f;" \
+ "if r5 == 7 goto 2f;" \
+ "r5 = *(u8 *)(r1 + 1); r5 &= 31; r5 += 1; r5 <<= 3;" \
+ "r4 = r1; r4 += r5;" \
+"2:" "r5 = *(u8 *)(r1 + 0);" \
+ "r1 = r4;" \
+ "r3 += -1;" \
+ "if r3 != 0 goto 1b;" \
+ "r1 += 16;" \
+ "if r1 > r2 goto 9f;" \
+ "r0 = *(u8 *)(r4 + " #off ");" \
+"9:" "r0 = 0;" \
+ "exit;" \
+ :: __imm_const(data, offsetof(struct __sk_buff, data)), \
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \
+ __imm_const(mark, offsetof(struct __sk_buff, mark)) \
+ : __clobber_all); \
+}
+
+TWO_STRIDES(two_strides, __success __log_level(4) __msg("processed {{[0-9]{2} }}insns"), 15)
+TWO_STRIDES(two_strides_bad, BAD_PKT, 16)
+
+#ifdef SPEC_V1
+#define BARRIER_AT_CALL __xlated_unpriv("nospec") __xlated_unpriv("call")
+#else
+#define BARRIER_AT_CALL
+#endif
+
+/* The size is correlated with a stored condition. The cpu can speculate past its test. */
+#define SIZE_GOES_WITH_COND(name, other) \
+SEC("tc") \
+__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) \
+__success __success_unpriv BARRIER_AT_CALL \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r2 = *(u32 *)(r1 + %[mark]);" \
+ "r4 = 8;" \
+ "r5 = 0;" \
+ "if r2 > 9 goto 1f;" \
+ other \
+"1:" \
+ "if r5 != 0 goto 9f;" \
+ "r2 = 0;" \
+ "r3 = %[words] ll;" \
+ "call %[bpf_skb_load_bytes];" \
+"9:" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm_const(mark, offsetof(struct __sk_buff, mark)), __imm_addr(words), \
+ __imm(bpf_skb_load_bytes) \
+ : __clobber_all); \
+}
+
+SIZE_GOES_WITH_COND(size_goes_with_cond, "r4 = 4096; r5 = 1;")
+SIZE_GOES_WITH_COND(size_goes_with_range, "r5 = r2; r5 += 1; r4 = r5; r4 *= 4096; r4 += 8;")
+
+static __naked __noinline __used void live_callee(void)
+{
+ asm volatile ("r0 = 7; exit;" ::: __clobber_all);
+}
+
+static __naked __noinline __used void callee_of_no_run(void)
+{
+ asm volatile ("r0 = *(u64 *)(r1 + 0); exit;" ::: __clobber_all);
+}
+
+/* r6 > r7 > r8 > r6 does not hold. An insn that is not a jump finds that out. */
+SEC("socket")
+__success __retval(0)
+__naked void call_that_no_run_gets_to(void)
+{
+ asm volatile (
+ "call live_callee;"
+ "call %[bpf_ktime_get_ns];"
+ "r6 = r0; r6 &= 15;"
+ "r7 = r0; r7 >>= 8; r7 &= 15;"
+ "r8 = r0; r8 >>= 16; r8 &= 15;"
+ "r0 = 0;"
+ "if r6 > 9 goto 9f;"
+ "if r7 > 9 goto 9f;"
+ "if r8 > 9 goto 9f;"
+ "r1 = r6; r1 -= r7; if r1 s< 1 goto 9f;"
+ "r2 = r7; r2 -= r8; if r2 s< 1 goto 9f;"
+ "r3 = r8; r3 -= r6; if r3 s< 1 goto 9f;"
+ "r4 = r8; r4 -= r6;"
+ "r9 = r1; r9 += r2; r9 += r3; r9 += r4;"
+ "r9 += r6; r9 += r7; r9 += r8;"
+ "*(u64 *)(r10 - 8) = r9;"
+ "r1 = r10;"
+ "r1 += -8;"
+ "call callee_of_no_run;"
+ "r0 = 0;"
+"9:"
+ "exit;"
+ :: __imm(bpf_ktime_get_ns) : __clobber_all);
+}
+
+/* Only the distance of the two pointers ends the loop */
+#define WALK(first, end) \
+ "r2 = r6; r2 &= " #first "; r2 += r10; r2 += -128;" \
+ "r6 >>= 8; r6 &= 3; r6 += 4;" \
+ "r3 = " end "; r3 += r6;" \
+ "r7 = 0;" \
+"1:" \
+ "if r2 == r3 goto 2f;" \
+ "*(u8 *)(r2 + 0) = 0;" \
+ "r2 += 1;" \
+ "r7 += 1;" \
+ "goto 1b;" \
+"2:"
+
+ONE_TEST(pointer_walks_to_pointer, __success, 8, WALK(3, "r10; r3 += -128"))
+ONE_TEST(pointer_walks_to_pointer_bad, BAD_WRITE, 7, WALK(3, "r10; r3 += -128"))
+ONE_TEST(pointer_starts_after_end,
+ __failure __msg("write to stack R2"), 8,
+ WALK(7, "r10; r3 += -128"))
+
+/* The offsets meet, the pointers do not */
+SEC("socket")
+__failure __msg("write to stack R2")
+__naked void pointer_walks_to_other_object(void)
+{
+ asm volatile (
+ "call %[bpf_ktime_get_ns];"
+ "r6 = r0;"
+ WALK(3, "%[words] ll; r3 += -128")
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_ktime_get_ns), __imm_addr(words) : __clobber_all);
+}
+
+/* off += 0..1 after the inner loop, off += 0..2 in it: 13 * (1 + 20 * 2) is 533 */
+#define TWO_TRIPS(name, verdict, max) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r6 = 0; r8 = 0;" \
+"1:" "r7 = 0;" \
+"2:" "call %[bpf_get_prandom_u32];" \
+ "if r0 > 2 goto 3f;" \
+ "r8 += r0;" \
+ "r7 += 1;" \
+ "if r7 != 20 goto 2b;" \
+"3:" "call %[bpf_get_prandom_u32];" \
+ "if r0 > 1 goto 4f;" \
+ "r8 += r0;" \
+"4:" "r6 += 1;" \
+ "if r6 != 13 goto 1b;" \
+ "r0 = 0;" \
+ "if r8 <= " #max " goto 9f;" \
+ "r0 = *(u64 *)(r10 + 0);" \
+"9:" "exit;" \
+ :: __imm(bpf_get_prandom_u32) : __clobber_all); \
+}
+
+TWO_TRIPS(two_trips, __success __log_level(4) __msg("processed {{[0-9]{2} }}insns"), 533)
+TWO_TRIPS(two_trips_bad,
+ __failure __msg("invalid read from stack"), 532)
+
+/* r6 is 0, 8, .. 56 at the header. @side runs in some trips. */
+#define STEPS(name, verdict, side) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r6 = 0;" \
+"1:" "r1 = r10; r1 += -64; r1 += r6;" \
+ "*(u64 *)(r1 + 0) = 0;" \
+ "*(u64 *)(r10 - 72) = r6;" \
+ "call %[bpf_get_prandom_u32];" \
+ "if r0 == 0 goto 3f;" \
+ side \
+ "r6 = r0;" \
+ "r6 = *(u64 *)(r10 - 72);" \
+"3:" "r6 += 8;" \
+ "if r6 != 64 goto 1b;" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_get_prandom_u32) : __clobber_all); \
+}
+
+STEPS(steps_one_walk, __success __log_level(4) __msg("processed 15 insns"), "")
+/* the last write gives r6 its value again, and one path goes around that write */
+STEPS(steps_wrong_guess,
+ __failure __msg("invalid write to stack"),
+ "r6 += 16; if r0 == 1 goto 3f;")
+STEPS(steps_by_alias,
+ __failure __msg("math between fp pointer and register"),
+ "r1 = r10; r1 += -72; r2 = 56; lock *(u64 *)(r1 + 0) += r2;")
+
+/* r6 is 0 in all trips. The verifier knows that before pass 0. */
+SEC("socket")
+__success __retval(0)
+__naked void steps_of_a_constant(void)
+{
+ asm volatile (
+ "r6 = 0; r7 = 0;"
+"1:" "if r7 != 1 goto 2f;"
+ "r0 = r6;"
+"2:" "r6 += 256;"
+ "r6 = (s8)r6;"
+ "r7 += 1;"
+ "if r7 < 10 goto 1b;"
+ "r0 = r6;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/* One path leaves the inner loop between two writes. The code seems to show @seems. */
+#define WRONG_STEP(name, verdict, init, test, reg, seems, tail) \
+SEC("socket") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r6 = 0;" init \
+"1:" test \
+ reg " += " #seems ";" \
+ "r7 = 0;" \
+"2:" reg " += -1;" \
+ "r7 += 1;" \
+ "if r7 > 3 goto 3f;" \
+ reg " += 1;" \
+ "goto 2b;" \
+"3:" tail \
+ "if r6 < 100 goto 1b;" \
+ "r0 = 0;" \
+ "exit;" \
+ ::: __clobber_all); \
+}
+
+WRONG_STEP(wrong_step_of_a_constant, __success __retval(0), "r1 = -100;",
+ "if r6 != 1 goto +2; w3 = w1; r0 = r3;", "r1", 1, "r6 += 1;")
+/* the loop gets may_goto */
+WRONG_STEP(wrong_step_of_the_counter, __success __retval(0), "", "", "r6", 2, "")
+
+/* r6 and r7 are unknown. @body leaves a pointer in r1 and a size in r2. */
+#define MEM_SIZE_TEST(name, verdict, body) \
+SEC("raw_tp") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_ktime_get_ns];" \
+ "r6 = r0;" \
+ "call %[bpf_ktime_get_ns];" \
+ "r7 = r0;" \
+ body \
+ "r3 = 0;" \
+ "call %[bpf_probe_read_kernel];" \
+"9:" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns), __imm(bpf_probe_read_kernel), \
+ __imm_addr(words) : __clobber_all); \
+}
+
+#define BAD_PAIR __failure __msg("memory, len pair leads to invalid memory access")
+#define WORDS "r1 = %[words] ll;"
+#define STACK "r1 = r10; r1 += -64;"
+
+/* read(buf + off, size - off) */
+#define REST(buf, size) "r6 &= 63;" buf "r1 += r6; r2 = " #size "; r2 -= r6;"
+MEM_SIZE_TEST(rest_of_map_value, __success, REST(WORDS, 64))
+MEM_SIZE_TEST(rest_of_map_value_bad, BAD_PAIR, REST(WORDS, 65))
+MEM_SIZE_TEST(rest_of_stack, __success, REST(STACK, 64))
+MEM_SIZE_TEST(rest_of_stack_bad, BAD_PAIR, REST(STACK, 65))
+MEM_SIZE_TEST(rest_is_changed_bad, BAD_PAIR, REST(WORDS, 64) "r7 &= 1; r2 += r7;")
+MEM_SIZE_TEST(rest_on_one_path_bad, BAD_PAIR,
+ REST(WORDS, 64) "if r7 & 64 goto +3; r2 = r7; r2 &= 63; r2 += 1;")
+
+/* if (off + len > size) return; read(buf + off, len) */
+#define END_IS_TESTED(size) \
+ "r6 &= 255; r7 &= 255; r2 = r6; r2 += r7;" \
+ "if r2 > " #size " goto 9f;" \
+ WORDS "r1 += r6; r2 = r7;"
+MEM_SIZE_TEST(end_is_tested, __success, END_IS_TESTED(64))
+MEM_SIZE_TEST(end_is_tested_bad, BAD_PAIR, END_IS_TESTED(65))
+
+/* if (len > size - off) return; read(buf + off, len) */
+#define REST_IS_TESTED(mask, len, size, test) \
+ "r6 &= " #mask ";" len "r2 = " #size "; r2 -= r6;" \
+ "if " test " goto 9f;" \
+ WORDS "r1 += r6; r2 = r7;"
+#define LEN_8 "r7 &= 255;"
+MEM_SIZE_TEST(rest_is_tested, __success, REST_IS_TESTED(63, LEN_8, 64, "r7 > r2"))
+MEM_SIZE_TEST(rest_is_tested_bad, BAD_PAIR, REST_IS_TESTED(63, LEN_8, 65, "r7 > r2"))
+MEM_SIZE_TEST(rest_is_tested_32, __success, REST_IS_TESTED(63, LEN_8, 64, "w2 < w7"))
+MEM_SIZE_TEST(rest_bounds_len, __success, REST_IS_TESTED(63, "", 64, "r7 > r2"))
+MEM_SIZE_TEST(rest_bounds_len_bad, BAD_PAIR, REST_IS_TESTED(63, "", 65, "r7 > r2"))
+/* 64 - off is negative for off > 64 */
+MEM_SIZE_TEST(rest_wraps_bad, BAD_PAIR, REST_IS_TESTED(127, LEN_8, 64, "r7 > r2"))
+MEM_SIZE_TEST(rest_wraps_32_bad, BAD_PAIR, REST_IS_TESTED(127, LEN_8, 64, "w7 > w2"))
+MEM_SIZE_TEST(rest_is_signed, __success, REST_IS_TESTED(127, LEN_8, 64, "r7 s> r2"))
+/* the opcode of s<= is the same as BPF_END */
+MEM_SIZE_TEST(rest_is_signed_le, __success, REST_IS_TESTED(127, LEN_8, 65, "r2 s<= r7"))
+MEM_SIZE_TEST(rest_is_signed_le_bad, BAD_PAIR, REST_IS_TESTED(127, LEN_8, 66, "r2 s<= r7"))
+
+/* One byte is checked behind the pointer. 64 bytes are read at offset 0. */
+SEC("tc")
+__failure __msg("invalid access to packet")
+__naked void rest_of_packet_bad(void)
+{
+ asm volatile (
+ "r6 = *(u32 *)(r1 + %[len]);"
+ "r7 = *(u32 *)(r1 + %[data_end]);"
+ "r1 = *(u32 *)(r1 + %[data]);"
+ "r6 &= 63;"
+ "r1 += r6;"
+ "r2 = r1;"
+ "r2 += 1;"
+ "if r2 > r7 goto 1f;"
+ "r2 = 64;"
+ "r2 -= r6;"
+ "r3 = 0;"
+ "r4 = 0;"
+ "r5 = 0;"
+ "call %[bpf_csum_diff];"
+"1:"
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_csum_diff),
+ __imm_const(len, offsetof(struct __sk_buff, len)),
+ __imm_const(data, offsetof(struct __sk_buff, data)),
+ __imm_const(data_end, offsetof(struct __sk_buff, data_end))
+ : __clobber_all);
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c
index dcb8f981f47..f61a0742483 100644
--- a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c
+++ b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c
@@ -3,6 +3,7 @@
#include <linux/bpf.h>
#include <limits.h>
#include <bpf/bpf_helpers.h>
+#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
SEC("socket")
@@ -716,4 +717,147 @@ l_exit_%=: \
: __clobber_all);
}
+/* r6 is an index in 8 bytes of the stack if it is 0..7 */
+#define LINK_TEST(name, verdict, body) \
+SEC("raw_tp") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = r0;" \
+ "r1 = 0;" \
+ "*(u64 *)(r10 - 8) = r1;" \
+ "*(u64 *)(r10 - 16) = r1;" \
+ body \
+ "r1 = r10;" \
+ "r1 += -8;" \
+ "r1 += r6;" \
+ "r0 = *(u8 *)(r1 + 0);" \
+"1:" \
+ "r0 = 0;" \
+ "exit;" \
+ : \
+ : __imm(bpf_get_prandom_u32) \
+ : __clobber_all); \
+}
+
+#define NO_LINK __failure __msg("math between fp pointer and register with unbounded min value")
+
+/* An atomic insn writes the register that it fetches to, and no other */
+LINK_TEST(atomic_keeps_src, __success,
+ "r7 = r6; lock *(u64 *)(r10 - 16) += r7; if r7 > 7 goto 1f;")
+LINK_TEST(atomic_keeps_r0, __success,
+ "r7 = 1; lock *(u64 *)(r10 - 16) += r7; if r0 > 7 goto 1f;")
+LINK_TEST(fetch_keeps_r0, __success,
+ "r7 = 1; r7 = atomic_fetch_add((u64 *)(r10 - 16), r7); if r0 > 7 goto 1f;")
+LINK_TEST(cmpxchg_keeps_src, __success,
+ "r7 = r6; r0 = cmpxchg_64(r10 - 16, r0, r7); if r7 > 7 goto 1f;")
+LINK_TEST(fetch_writes_src, NO_LINK,
+ "r7 = r6; r7 = atomic_fetch_add((u64 *)(r10 - 16), r7); if r7 > 7 goto 1f;")
+LINK_TEST(xchg_writes_src, NO_LINK,
+ "r7 = r6; r7 = xchg_64(r10 - 16, r7); if r7 > 7 goto 1f;")
+LINK_TEST(cmpxchg_writes_r0, NO_LINK,
+ "r7 = 1; r0 = cmpxchg_64(r10 - 16, r0, r7); if r0 > 7 goto 1f;")
+
+/* r1 moves with the trip index, r1 >> 14 is 0..6 */
+SEC("raw_tp")
+__success __log_level(2)
+__msg("has trip count 100000") __not_msg("pass 1")
+__naked void atomic_keeps_counter(void)
+{
+ asm volatile (
+ "r1 = 0;"
+ "r2 = 0;"
+ "*(u64 *)(r10 - 8) = r1;"
+ "*(u64 *)(r10 - 16) = r1;"
+"1:"
+ "r3 = r1;"
+ "r3 >>= 14;"
+ "r3 += r10;"
+ "r0 = *(u8 *)(r3 - 8);"
+ "lock *(u64 *)(r10 - 16) += r1;"
+ "r1 += 1;"
+ "r2 += 1;"
+ "if r2 < 100000 goto 1b;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+#define TOO_FAR __failure __msg("invalid variable-offset read from stack")
+
+/* AND that clears no bit is a move */
+LINK_TEST(and_clears_no_bit, __success,
+ "r6 &= 0xc6; r7 = r6; r7 &= 0x1c6; if r7 > 7 goto 1f;")
+LINK_TEST(and32_clears_no_bit, __success,
+ "r6 &= 0xc6; r7 = r6; w7 &= 0x1c6; if w7 > 7 goto 1f;")
+LINK_TEST(and_clears_a_bit, TOO_FAR,
+ "r6 &= 0xc6; r7 = r6; r7 &= 0x146; if r7 > 7 goto 1f;")
+LINK_TEST(and_reg_may_clear_a_bit, TOO_FAR,
+ "r6 &= 0xc6; r7 = r6; r8 = r0; r8 |= 0x46; r7 &= r8; if r7 > 7 goto 1f;")
+/* the high bits are cleared */
+LINK_TEST(and32_of_wide, NO_LINK,
+ "r6 <<= 8; r7 = r6; w7 &= -256; if r7 > 7 goto 1f;")
+LINK_TEST(and_mask_comes_first_far, TOO_FAR,
+ "r6 &= 0xc6; r7 = 0x146; r7 &= r6; if r7 > 7 goto 1f;")
+
+/* The compiler clears the bits of the mask that are 0 in the number */
+LINK_TEST(and_low_bits, __success,
+ "r6 <<= 2; r7 = r6; r6 &= 0xfffc; if r7 > 7 goto 1f;")
+LINK_TEST(and_rounds_down, __success,
+ "r6 &= 0xff; r7 = r6; r6 &= 0xfc; if r7 > 7 goto 1f;")
+LINK_TEST(and_cuts_in_the_middle, TOO_FAR,
+ "r7 = r6; r7 &= 0xff; r6 = r7; r7 &= 0xef; if r7 > 7 goto 1f;")
+LINK_TEST(and_rounds_down_far, TOO_FAR,
+ "r6 &= 0xff; r7 = r6; r7 &= 0xf0; if r7 > 7 goto 1f;")
+
+/* r7 is 4 * trip + 8, r8 is 8 * trip. The loop has no trip count, so the products wrap. */
+#define STRIDE_TEST(name, verdict, body) \
+SEC("raw_tp") \
+verdict \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r7 = 8;" \
+ "r8 = 0;" \
+ "*(u64 *)(r10 - 8) = r8;" \
+"2:" \
+ ".8byte %[may_goto];" \
+ "r7 += 4;" \
+ "r8 += 8;" \
+ "goto 2b;" \
+ body \
+ "r1 = r10;" \
+ "r1 += -8;" \
+ "r1 += r6;" \
+ "r0 = *(u8 *)(r1 + 0);" \
+"1:" \
+ "r0 = r8;" \
+ "exit;" \
+ : \
+ : __imm(bpf_get_prandom_u32), \
+ __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 3, 0)) \
+ : __clobber_all); \
+}
+
+#define PASSES __success __log_level(2) __not_msg("pass 1")
+#define AT_FP __failure __msg("invalid read from stack R1 off=0")
+#define NEXT_BLOCK "call %[bpf_get_prandom_u32]; if r0 > 5 goto 3f; r9 = 1; 3:"
+
+STRIDE_TEST(stride_low32, PASSES, "w6 = w7; if r7 > 7 goto 1f;")
+STRIDE_TEST(stride_low32_both, PASSES, "w6 = w7; if w7 > 7 goto 1f;")
+STRIDE_TEST(stride_low32_next_block, PASSES, "w6 = w7;" NEXT_BLOCK "if r7 > 7 goto 1f;")
+STRIDE_TEST(stride_times_2, PASSES, "r6 = r7; r6 <<= 1; if r7 > 3 goto 1f;")
+STRIDE_TEST(stride_neg, PASSES, "r6 = 7; r6 -= r7; if r7 > 7 goto 1f;")
+STRIDE_TEST(stride_shift, PASSES, "r6 = r7; r6 >>= 3; if r7 > 63 goto 1f;")
+STRIDE_TEST(stride_exact_shift, PASSES, "r6 = r7; r6 >>= 2; if r7 > 31 goto 1f;")
+STRIDE_TEST(stride_mask_low16, PASSES, "r6 = r7; r6 &= 0xfffc; if r7 > 7 goto 1f;")
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+STRIDE_TEST(stride_fill32, PASSES,
+ "*(u64 *)(r10 - 16) = r7; r6 = *(u32 *)(r10 - 16); if r7 > 7 goto 1f;")
+#endif
+STRIDE_TEST(stride_low32_far, AT_FP, "w6 = w7; if r7 > 8 goto 1f;")
+STRIDE_TEST(stride_shift_far, AT_FP, "r6 = r7; r6 >>= 3; if r7 > 64 goto 1f;")
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c
index 44730109db5..0197c2c21b8 100644
--- a/tools/testing/selftests/bpf/progs/verifier_movsx.c
+++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c
@@ -526,6 +526,26 @@ label_%=: \
: __clobber_all);
}
+/* The source is even, so the result is even */
+#define KEEPS_LOW_BIT(name, insns) \
+SEC("socket") \
+__success __retval(0) \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ insns \
+ "r0 &= 1;" \
+ "if r0 == 0 goto 1f;" \
+ "r0 /= 0;" \
+ "1: exit;" \
+ :: __imm(bpf_get_prandom_u32) : __clobber_all); \
+}
+
+KEEPS_LOW_BIT(mov64sx_s8_one_sign_keeps_low_bit, "r0 &= 0x7e; r0 |= 0x100; r0 = (s8)r0;")
+KEEPS_LOW_BIT(mov32sx_s8_one_sign_keeps_low_bit, "w0 &= 0x7e; w0 |= 0x100; w0 = (s8)w0;")
+KEEPS_LOW_BIT(mov32sx_s8_keeps_low_bit, "w0 &= -2; w0 = (s8)w0;")
+
#else
SEC("socket")
diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
index a7b11630eb6..56a316d787a 100644
--- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
+++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
@@ -1486,6 +1486,32 @@ FIRST_HALF(spill_of_64_bits_vs_spill_of_32_bits,
"r6 = r0; r6 <<= 32; if r0 > 5 goto 1f; *(u32 *)(r10 - 8) = r6; goto 2f;"
"1: r6 = 0x7ffffff000000000 ll; *(u64 *)(r10 - 8) = r6; 2:")
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+/* The low 16 bits of the fill are from the spill */
+#define FILL_32_SPILL_16(mask) \
+ asm volatile ( \
+ "call %[bpf_ktime_get_ns];" \
+ "*(u64 *)(r10 - 8) = r0; r1 = 5; *(u16 *)(r10 - 8) = r1;" \
+ "r1 = *(u32 *)(r10 - 8); r1 &= " #mask ";" \
+ "if r1 == 5 goto 1f; r0 /= 0;" \
+ "1: r0 = 0; exit;" \
+ :: __imm(bpf_ktime_get_ns) : __clobber_all)
+
+SEC("raw_tp")
+__success
+__naked void fill_32_bits_spill_16_bits(void)
+{
+ FILL_32_SPILL_16(0xffff);
+}
+
+SEC("raw_tp")
+__failure __msg("div by zero")
+__naked void fill_32_bits_spill_16_bits_bad(void)
+{
+ FILL_32_SPILL_16(0x1ffff);
+}
+#endif
+
/* check valid spill/fill, ptr to tp buffer */
SEC("raw_tracepoint.w")
__success
diff --git a/tools/testing/selftests/bpf/progs/verifier_trip_count.c b/tools/testing/selftests/bpf/progs/verifier_trip_count.c
index 69288309ef0..7079df5ee20 100644
--- a/tools/testing/selftests/bpf/progs/verifier_trip_count.c
+++ b/tools/testing/selftests/bpf/progs/verifier_trip_count.c
@@ -268,6 +268,52 @@ __naked void meet_disagrees(void)
: __clobber_all);
}
+/* A loop with no 32-bit insns can count down and store the carry */
+#define CARRY_TEST(name, initial, minus_step, msg) \
+ COND_TEST(name, initial, 0, minus_step, msg, \
+ "1: r6 <<= 32; r6 >>= 32; r6 += r7;" \
+ "r2 = r6; r2 <<= 32; r2 >>= 32;" \
+ "r3 = 1; if r2 != r6 goto 3f; r3 = 0;" \
+ "3: if r3 != 0 goto 1b;")
+
+CARRY_TEST(carry_step_1, 33, 0xffffffffULL, COUNT(34))
+CARRY_TEST(carry_step_3, 100, 0xfffffffdULL, COUNT(34))
+CARRY_TEST(carry_step_0, 33, 0x100000000ULL, NO_COUNT)
+
+/* Or it counts up until there is a carry */
+COND_TEST(carry_ends_loop, 0xffffffdeULL, 0, 1, COUNT(34),
+ "1: r6 <<= 32; r6 >>= 32; r6 += r7;"
+ "r2 = r6; r2 <<= 32; r2 >>= 32;"
+ "if r2 == r6 goto 1b;")
+
+/* Two entries have two bounds. The @test of r6 and its low 32 bits is not true. */
+#define OTHER_JUMP_TEST(name, initial, test) \
+SEC("xdp") \
+__success \
+__log_level(2) \
+ANY_COUNT \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "r9 = 20; r5 = r10;" \
+ "if r0 > 5 goto 1f;" \
+ "r9 = 40; r5 = 0;" \
+"1: r8 = 0; r4 = 0; r6 = " #initial " ll;" \
+"2: r6 += -1; r7 = r6; r7 <<= 32; r7 >>= 32;" \
+ "if " test " goto 3f;" \
+ "r4 += r8; r8 += 1;" \
+ "if r8 != r9 goto 2b;" \
+"3: if r5 == 0 goto 4f;" \
+ "*(u64 *)(r5 - 8) = r4;" \
+"4: r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_get_prandom_u32) : __clobber_all); \
+}
+
+OTHER_JUMP_TEST(low_half_wraps_with_carry, 0x20000000a, "r7 == r6")
+OTHER_JUMP_TEST(low_half_is_not_above, 0x10000000a, "r7 > r6")
+
/* 'if (c) sum += a; else sum += b;': two loads, one way to compute the sum */
SEC("socket")
__success
@@ -5176,4 +5222,432 @@ __naked void widen_pkt_pointer_beyond_pkt_end(void)
:: PKT_IMMS : __clobber_all);
}
+/* r7 += r6 & 1: r7 is not above the number of trips */
+#define SUM_OF_LOW_BITS(size) \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1:" \
+ "r1 = r6;" \
+ "r1 &= 1;" \
+ "r1 += r7;" \
+ "r7 = r1;" \
+ "r6 += 1;" \
+ "if r6 != 100000 goto 1b;" \
+ "if r7 > " #size " goto 2f;" \
+ "r0 = 0;" \
+ "exit;" \
+"2:" \
+ "r0 = *(u64 *)(r10 + 0);" \
+ "exit;"
+
+SEC("socket")
+__success __log_level(2)
+__msg("has trip count 100000")
+__naked void sum_of_low_bits(void)
+{
+ asm volatile (SUM_OF_LOW_BITS(100000) ::: __clobber_all);
+}
+
+SEC("socket")
+__failure __msg("invalid read from stack R10 off=0 size=8")
+__naked void sum_of_low_bits_bad(void)
+{
+ asm volatile (SUM_OF_LOW_BITS(49999) ::: __clobber_all);
+}
+
+/* for (i = 0..1; i <= n; i++) */
+SEC("socket")
+__success __log_level(2)
+__not_msg("pass 1")
+__naked void start_and_bound_are_ranges(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r7 = r0;"
+ "if r7 > 510 goto 9f;"
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r6 &= 1;"
+ "if w6 > w7 goto 9f;"
+ "r7 &= 0x7fffffff;"
+"1:"
+ "r0 = r6;"
+ "r2 = r10;"
+ "r2 += -512;"
+ "r2 += r0;"
+ "*(u8 *)(r2 + 0) = 1;"
+ "r6 = r0;"
+ "r6 += 1;"
+ "if r0 < r7 goto 1b;"
+"9:"
+ "r0 = 0;"
+ "exit;"
+ :: __imm(bpf_get_prandom_u32) : __clobber_all);
+}
+
+/* h wraps around in 0..3. It moves with the trip index only in the first trips. */
+SEC("socket")
+__success
+__naked void index_wraps_around(void)
+{
+ asm volatile (
+ "r6 = 0;"
+ "r7 = 0;"
+"1:"
+ "r2 = r7;"
+ "r2 <<= 3;"
+ "r1 = r10;"
+ "r1 += -32;"
+ "r1 += r2;"
+ "*(u64 *)(r1 + 0) = r6;"
+ "r3 = 0;"
+ "if r7 s> 2 goto 2f;"
+ "r7 += 1;"
+ "r3 = r7;"
+"2:"
+ "r7 = r3;"
+ "r6 += 1;"
+ "if r6 != 100000 goto 1b;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/*
+ * r8 - k >= 300, r8 - 2 * k <= 40, r8 < 560: no run gets through the three tests.
+ * The equations show it when the trip index is gone, at the exit of the loop.
+ * That state has r8 = 530. It must not subsume the state that skips the loop.
+ */
+SEC("socket")
+__failure __msg("invalid read from stack R10 off=0 size=8")
+__naked void infeasible_state_subsumes_nothing(void)
+{
+ asm volatile (
+ "r6 = 0;"
+ "r8 = 530;"
+ "call %[bpf_get_prandom_u32];"
+ "if r0 != 1 goto 3f;"
+"1:"
+ "call %[bpf_get_prandom_u32];"
+ "r8 = r0;"
+ "r8 &= 0xfff;"
+ "r2 = r8;"
+ "r2 -= r6;"
+ "r3 = r6;"
+ "r3 <<= 1;"
+ "r4 = r8;"
+ "r4 -= r3;"
+ "if r2 s< 300 goto 2f;"
+ "if r4 s> 40 goto 2f;"
+ "if r8 s< 560 goto 3f;"
+"2:"
+ "r6 += 1;"
+ "if r6 < 700 goto 1b;"
+ "r0 = 0;"
+ "exit;"
+"3:"
+ "call %[bpf_get_prandom_u32];"
+ "if r0 == 2 goto 4f;"
+ "r0 = *(u64 *)(r10 + 0);"
+"4:"
+ "r0 = r8;"
+ "exit;"
+ :: __imm(bpf_get_prandom_u32) : __clobber_all);
+}
+
+/*
+ * r6 = S64_MAX - k. No run gets to r6 == 0 or r6 == 1 by counting.
+ *
+ * k r7
+ * r6 == 0 S64_MAX 1
+ * r6 == 1 S64_MAX - 1 0 join: r7 = k - (S64_MAX - 1)
+ * bit 6 small 1 the run
+ *
+ * The run must not get the trip index of the first state: the next trip cuts it off.
+ */
+SEC("socket")
+__failure __msg("invalid read from stack R10 off=0 size=8")
+__naked void trip_index_does_not_jump_ahead(void)
+{
+ asm volatile (
+ "r6 = 0x7fffffffffffffff ll;"
+ "r7 = 0;"
+"1:"
+ "may_goto 9f;"
+ "if r7 == 0 goto 2f;"
+ "r0 = *(u64 *)(r10 + 0);"
+"2:"
+ "call %[bpf_get_prandom_u32];"
+ "if r6 == 0 goto 4f;"
+ "if r6 == 1 goto 6f;"
+ "if r6 & 0x40 goto 3f;"
+ "r0 = 1;"
+ "r6 = 0;"
+ "r7 = 1;"
+ "goto 5f;"
+"3:"
+ "r6 += -1;"
+ "goto 1b;"
+"4:"
+ "r6 = 0;"
+ "r7 = 1;"
+"5:"
+ "*(u64 *)(r10 - 8) = r0;"
+ "goto 7f;"
+"6:"
+ "r6 = 0;"
+ "r7 = 0;"
+"7:"
+ "goto 1b;"
+"9:"
+ "r0 = r7;"
+ "exit;"
+ :: __imm(bpf_get_prandom_u32) : __clobber_all);
+}
+
+#define RING(name, mask) \
+__naked void name(void) \
+{ \
+ asm volatile ( \
+ "r6 = 0;" \
+ "r7 = 0;" \
+"1:" \
+ "r1 = r10;" \
+ "r1 += -32;" \
+ "r1 += r7;" \
+ "*(u8 *)(r1 + 0) = 0;" \
+ "r7 += 1;" \
+ "r7 &= " #mask ";" \
+ "r6 += 1;" \
+ "if r6 < 50000 goto 1b;" \
+ "r0 = 0;" \
+ "exit;" \
+ ::: __clobber_all); \
+}
+
+/* pos = (pos + 1) & 31. The threshold of the trip index is not the threshold of pos. */
+SEC("socket")
+__success __log_level(2)
+__msg(RANGE("2", "R7", 0, 31))
+__not_msg("pass 2")
+RING(ring_index_is_bounded_by_mask, 31)
+
+SEC("socket")
+__failure __msg("write to stack R1")
+RING(ring_index_is_too_large, 63)
+
+/* Each add makes an equation that depends on the equation of the add before it */
+SEC("socket")
+__success
+__naked void long_chain_of_sums(void)
+{
+ asm volatile (
+ "call %[bpf_get_prandom_u32];"
+ "r6 = r0;"
+ "r6 &= 7;"
+ "call %[bpf_get_prandom_u32];"
+ "r7 = r0;"
+ "r7 &= 7;"
+ "call %[bpf_get_prandom_u32];"
+ "r0 &= 7;"
+ "r1 = 7;"
+ ".rept 2000;"
+ "r0 += r6;"
+ "r1 += 6;"
+ "if r0 > r1 goto 9f;"
+ "r0 += r7;"
+ "r1 += 6;"
+ "if r0 > r1 goto 9f;"
+ ".endr;"
+"9:"
+ "r0 += r6;"
+ "r0 += r7;"
+ "exit;"
+ :: __imm(bpf_get_prandom_u32) : __clobber_all);
+}
+
+/* The start of a block has all loops, an insn has the inner one */
+SEC("socket")
+__success __log_level(2)
+__msg("to 2: {{.*}} loop_stack=1,2")
+__msg("2: (07) r7 += 1 {{.*}} loop_stack=2{{$}}")
+__naked void log_of_nested_loops(void)
+{
+ asm volatile (
+ "r6 = 0;"
+"1:"
+ "r7 = 0;"
+"2:"
+ "r7 += 1;"
+ "if r7 < 3 goto 2b;"
+ "r6 += 1;"
+ "if r6 < 3 goto 1b;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/*
+ * r6 is 3 * trip and it can wrap. After the loop r6 == r7, so the sum is (17..25) + (20..28).
+ */
+#define SUM_WITH_PRODUCT_THAT_WRAPS(off) \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "r7 = r0;" \
+ "if r7 < 20 goto 9f;" \
+ "if r7 > 28 goto 9f;" \
+ "r6 = 0;" \
+"1:" \
+ "r8 = r6;" \
+ "r8 += r7;" \
+ "r6 += 3;" \
+ "if r6 != r7 goto 1b;" \
+ "r1 = r10;" \
+ "r1 += " #off ";" \
+ "r1 += r8;" \
+ "*(u8 *)(r1 + 0) = 1;" \
+"9:" \
+ "r0 = 0;" \
+ "exit;" \
+ :: __imm(bpf_get_prandom_u32) : __clobber_all)
+
+SEC("socket")
+__success
+__naked void sum_with_product_that_wraps(void)
+{
+ SUM_WITH_PRODUCT_THAT_WRAPS(-54);
+}
+
+/* r7 = 27 gives 51 */
+SEC("socket")
+__failure __msg("write to stack R1")
+__naked void sum_with_product_that_wraps_above(void)
+{
+ SUM_WITH_PRODUCT_THAT_WRAPS(-51);
+}
+
+/*
+ * r7 stays in 0..31. No range shows it, so all trips are peeled.
+ * A path-sensitive run must not skip any.
+ */
+SEC("socket")
+__success
+__naked void path_sensitive_does_not_skip_a_ring(void)
+{
+ asm volatile (
+ "r6 = 0;"
+ "r7 = 0;"
+"1:"
+ "r1 = r10;"
+ "r1 += -32;"
+ "r1 += r7;"
+ "*(u8 *)(r1 + 0) = 0;"
+ "r7 += 1;"
+ "r2 = r7;"
+ "r2 >>= 5;"
+ "r2 <<= 5;"
+ "r7 ^= r2;"
+ "r6 += 1;"
+ "if r6 < 6000 goto 1b;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+/* The sum of 4000, 3999, ..., -4000 is 0 */
+SEC("socket")
+__success
+__naked void path_sensitive_does_not_skip_a_sum(void)
+{
+ asm volatile (
+ "r6 = 4000;"
+ "r7 = 0;"
+"1:"
+ "r7 += r6;"
+ "r6 += -1;"
+ "if r6 s>= -4000 goto 1b;"
+ "r1 = r10;"
+ "r1 += -8;"
+ "r1 += r7;"
+ "*(u8 *)(r1 + 0) = 0;"
+ "r0 = 0;"
+ "exit;"
+ ::: __clobber_all);
+}
+
+char a1m[1 << 20] SEC(".data.a1m");
+
+/* (n, size) is (1 << 18, 4) or (@n, 8). for (i = 0; i < n; i++) { a1m[i * size]; @inner } */
+#define TWO_ENTRIES(n, inner) \
+ asm volatile ( \
+ "call %[bpf_get_prandom_u32];" \
+ "r6 = 0x40000;" \
+ "r7 = 4;" \
+ "if w0 & 1 goto 1f;" \
+ "r6 = " #n ";" \
+ "r7 = 8;" \
+"1: r8 = 0;" \
+"2: r2 = r8;" \
+ "r2 *= r7;" \
+ "r1 = %[a1m] ll;" \
+ "r1 += r2;" \
+ "r0 = *(u8 *)(r1 + 0);" \
+ inner \
+ "r8 += 1;" \
+ "if r8 != r6 goto 2b;" \
+ "exit;" \
+ :: __imm(bpf_get_prandom_u32), __imm_addr(a1m) : __clobber_all)
+
+SEC("socket")
+__success __log_level(2) __not_msg("pass 2") __retval(0)
+__naked void each_entry_has_its_threshold(void)
+{
+ TWO_ENTRIES(0x20000, "");
+}
+
+SEC("socket")
+__failure __msg("invalid access to map value")
+__naked void each_entry_has_its_threshold_bad(void)
+{
+ TWO_ENTRIES(0x20001, "");
+}
+
+/* for (j = 0; j != i + 1; j++); */
+SEC("socket")
+__success __log_level(2) __not_msg("pass 2")
+__naked void threshold_after_inner_loop(void)
+{
+ TWO_ENTRIES(0x20000, "r3 = r8; r3 += 1; r9 = 0; 3: r9 += 1; if r9 != r3 goto 3b;");
+}
+
+/* sum = @start; for (i = 0; i < 64; i++) sum += a1m[i]; */
+#define SUM_FROM(start) \
+ asm volatile ( \
+ "call %[bpf_ktime_get_ns];" \
+ "r6 = " start ";" \
+ "r7 = 0;" \
+"1: r1 = %[a1m] ll;" \
+ "r1 += r7;" \
+ "r1 = *(u8 *)(r1 + 0);" \
+ "r6 += r1;" \
+ "r7 += 1;" \
+ "if r7 < 64 goto 1b;" \
+ "r0 = r6;" \
+ "exit;" \
+ :: __imm(bpf_ktime_get_ns), __imm_addr(a1m) : __clobber_all)
+
+SEC("socket")
+__success __log_level(4) __msg("processed 12 insns")
+__naked void unknown_sum_body_once(void)
+{
+ SUM_FROM("r0");
+}
+
+SEC("socket")
+__success __log_level(4) __msg("processed 24 insns")
+__naked void known_sum_three_trips(void)
+{
+ SUM_FROM("0");
+}
+
char _license[] SEC("license") = "GPL";
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread* [PATCH bpf-next 49/49] bpf, docs: Rewrite verifier.rst
2026-11-20 7:41 [PATCH bpf-next 00/49] bpf: Verify in passes, with equations in the state Alexei Starovoitov
` (47 preceding siblings ...)
2026-11-20 7:42 ` [PATCH bpf-next 48/49] selftests/bpf: Test equations Alexei Starovoitov
@ 2026-11-20 7:42 ` Alexei Starovoitov
48 siblings, 0 replies; 79+ messages in thread
From: Alexei Starovoitov @ 2026-11-20 7:42 UTC (permalink / raw)
To: bpf; +Cc: daniel, andrii, eddyz87, memxor
From: Alexei Starovoitov <ast@kernel.org>
The document describes a walk over paths with pruning. The verifier does
not work that way. The types, the log text and the errors are old too.
Write it from scratch, in reading order, with examples:
overview the main idea, proof and guess
state numbers, pointers, ids, the stack, what is held
one insn the checks, packets
pass liveness, prune, join, join_key, the order
precision data and control deps, the join rule, on one example
equations insn, jump, call, join, prune
loops widening, trip index, may_goto, peel, suspects
functions instances, callbacks, recursion
errors partition, fork, budget, failing path, all passes
the rest fixups, speculation, limits, what is trusted, the log
Update the limits and one reference in bpf_design_QA.rst.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
Documentation/bpf/bpf_design_QA.rst | 6 +-
Documentation/bpf/verifier.rst | 2007 ++++++++++++++++++++-------
2 files changed, 1496 insertions(+), 517 deletions(-)
diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst
index f86b272920a..4956c2741cb 100644
--- a/Documentation/bpf/bpf_design_QA.rst
+++ b/Documentation/bpf/bpf_design_QA.rst
@@ -94,8 +94,8 @@ It's the maximum number of instructions that the unprivileged bpf
program can have. The verifier has various internal limits.
The largest program can consist of 1 million instructions.
There is a limit to the number of nested bpf-to-bpf
-calls, a limit to the number of the verifier states per instruction,
-a limit to the number of maps used by the program.
+calls, a limit to the work of the verifier and a limit to its memory.
+See 'Limits' in Documentation/bpf/verifier.rst.
All these limits can be hit with a sufficiently complex program.
There are also non-numerical limits that can cause the program
to be rejected. The verifier used to recognize only pointer + constant
@@ -121,7 +121,7 @@ C code cannot express them and has to use builtin intrinsics?
A: This is artifact of compatibility with classic BPF. Modern
networking code in BPF performs better without them.
-See 'direct packet access'.
+See 'Packets' in Documentation/bpf/verifier.rst.
Q: BPF instructions mapping not one-to-one to native CPU
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/Documentation/bpf/verifier.rst b/Documentation/bpf/verifier.rst
index 510d15bc697..2b76e978d79 100644
--- a/Documentation/bpf/verifier.rst
+++ b/Documentation/bpf/verifier.rst
@@ -1,560 +1,1539 @@
-
=============
eBPF verifier
=============
-The safety of the eBPF program is determined in two steps.
-
-First step does DAG check to disallow loops and other CFG validation.
-In particular it will detect programs that have unreachable instructions.
-(though classic BPF checker allows them)
+The verifier decides if a BPF program is safe to run in the kernel. It checks
+the program once, when the program is loaded. It does not run the program.
-Second step starts from the first insn and descends all possible paths.
-It simulates execution of every insn and observes the state change of
-registers and stack.
+Read the chapters in order. A new word is *in italics* where it is explained.
+The last chapter lists these words.
-At the start of the program the register R1 contains a pointer to context
-and has type PTR_TO_CTX.
-If verifier sees an insn that does R2=R1, then R2 has now type
-PTR_TO_CTX as well and can be used on the right hand side of expression.
-If R1=PTR_TO_CTX and insn is R2=R1+R1, then R2=SCALAR_VALUE,
-since addition of two valid pointers makes invalid pointer.
-(In 'secure' mode verifier will reject any type of pointer arithmetic to make
-sure that kernel addresses don't leak to unprivileged users)
+.. contents::
+ :local:
+ :depth: 1
-If register was never written to, it's not readable::
+Overview
+========
- bpf_mov R0 = R2
- bpf_exit
+The main idea
+-------------
-will be rejected, since R2 is unreadable at the start of the program.
+The verifier runs the program with sets of values. A *state* says what each
+register and each stack slot can hold before an instruction (*insn*)::
-After kernel function call, R1-R5 are reset to unreadable and
-R0 has a return type of the function.
+ r6 = any number the state before the jump
+ if r6 > 7 goto out
+ r6 = 0..7 the state after it
-Since R6-R9 are callee saved, their state is preserved across the call.
+A jump makes two states. The two states meet again where the paths meet. The
+verifier has two choices there::
-::
-
- bpf_mov R6 = 1
- bpf_call foo
- bpf_mov R0 = R6
- bpf_exit
-
-is a correct program. If there was R1 instead of R6, it would have
-been rejected.
-
-load/store instructions are allowed only with registers of valid types, which
-are PTR_TO_CTX, PTR_TO_MAP, PTR_TO_STACK. They are bounds and alignment checked.
-For example::
-
- bpf_mov R1 = 1
- bpf_mov R2 = 2
- bpf_xadd *(u32 *)(R1 + 3) += R2
- bpf_exit
-
-will be rejected, since R1 doesn't have a valid pointer type at the time of
-execution of instruction bpf_xadd.
-
-At the start R1 type is PTR_TO_CTX (a pointer to generic ``struct bpf_context``)
-A callback is used to customize verifier to restrict eBPF program access to only
-certain fields within ctx structure with specified size and alignment.
-
-For example, the following insn::
-
- bpf_ld R0 = *(u32 *)(R6 + 8)
-
-intends to load a word from address R6 + 8 and store it into R0
-If R6=PTR_TO_CTX, via is_valid_access() callback the verifier will know
-that offset 8 of size 4 bytes can be accessed for reading, otherwise
-the verifier will reject the program.
-If R6=PTR_TO_STACK, then access should be aligned and be within
-stack bounds, which are [-MAX_BPF_STACK, 0). In this example offset is 8,
-so it will fail verification, since it's out of bounds.
-
-The verifier will allow eBPF program to read data from stack only after
-it wrote into it.
-
-Classic BPF verifier does similar check with M[0-15] memory slots.
-For example::
-
- bpf_ld R0 = *(u32 *)(R10 - 4)
- bpf_exit
-
-is invalid program.
-Though R10 is correct read-only register and has type PTR_TO_STACK
-and R10 - 4 is within stack bounds, there were no stores into that location.
-
-Pointer register spill/fill is tracked as well, since four (R6-R9)
-callee saved registers may not be enough for some programs.
-
-Allowed function calls are customized with bpf_verifier_ops->get_func_proto()
-The eBPF verifier will check that registers match argument constraints.
-After the call register R0 will be set to return type of the function.
-
-Function calls is a main mechanism to extend functionality of eBPF programs.
-Socket filters may let programs to call one set of functions, whereas tracing
-filters may allow completely different set.
-
-If a function made accessible to eBPF program, it needs to be thought through
-from safety point of view. The verifier will guarantee that the function is
-called with valid arguments.
-
-seccomp vs socket filters have different security restrictions for classic BPF.
-Seccomp solves this by two stage verifier: classic BPF verifier is followed
-by seccomp verifier. In case of eBPF one configurable verifier is shared for
-all use cases.
-
-See details of eBPF verifier in kernel/bpf/verifier.c
-
-Register value tracking
-=======================
-
-In order to determine the safety of an eBPF program, the verifier must track
-the range of possible values in each register and also in each stack slot.
-This is done with ``struct bpf_reg_state``, defined in include/linux/
-bpf_verifier.h, which unifies tracking of scalar and pointer values. Each
-register state has a type, which is either NOT_INIT (the register has not been
-written to), SCALAR_VALUE (some value which is not usable as a pointer), or a
-pointer type. The types of pointers describe their base, as follows:
-
-
- PTR_TO_CTX
- Pointer to bpf_context.
- CONST_PTR_TO_MAP
- Pointer to struct bpf_map. "Const" because arithmetic
- on these pointers is forbidden.
- PTR_TO_MAP_VALUE
- Pointer to the value stored in a map element.
- PTR_TO_MAP_VALUE_OR_NULL
- Either a pointer to a map value, or NULL; map accesses
- (see maps.rst) return this type, which becomes a
- PTR_TO_MAP_VALUE when checked != NULL. Arithmetic on
- these pointers is forbidden.
- PTR_TO_STACK
- Frame pointer.
- PTR_TO_PACKET
- skb->data.
- PTR_TO_PACKET_END
- skb->data + headlen; arithmetic forbidden.
- PTR_TO_SOCKET
- Pointer to struct bpf_sock_ops, implicitly refcounted.
- PTR_TO_SOCKET_OR_NULL
- Either a pointer to a socket, or NULL; socket lookup
- returns this type, which becomes a PTR_TO_SOCKET when
- checked != NULL. PTR_TO_SOCKET is reference-counted,
- so programs must release the reference through the
- socket release function before the end of the program.
- Arithmetic on these pointers is forbidden.
-
-However, a pointer may be offset from this base (as a result of pointer
-arithmetic), and this is tracked in two parts: the 'fixed offset' and 'variable
-offset'. The former is used when an exactly-known value (e.g. an immediate
-operand) is added to a pointer, while the latter is used for values which are
-not exactly known. The variable offset is also used in SCALAR_VALUEs, to track
-the range of possible values in the register.
-
-The verifier's knowledge about the variable offset consists of:
-
-* minimum and maximum values as unsigned
-* minimum and maximum values as signed
-
-* knowledge of the values of individual bits, in the form of a 'tnum': a u64
- 'mask' and a u64 'value'. 1s in the mask represent bits whose value is unknown;
- 1s in the value represent bits known to be 1. Bits known to be 0 have 0 in both
- mask and value; no bit should ever be 1 in both. For example, if a byte is read
- into a register from memory, the register's top 56 bits are known zero, while
- the low 8 are unknown - which is represented as the tnum (0x0; 0xff). If we
- then OR this with 0x40, we get (0x40; 0xbf), then if we add 1 we get (0x0;
- 0x1ff), because of potential carries.
-
-Besides arithmetic, the register state can also be updated by conditional
-branches. For instance, if a SCALAR_VALUE is compared > 8, in the 'true' branch
-it will have a umin_value (unsigned minimum value) of 9, whereas in the 'false'
-branch it will have a umax_value of 8. A signed compare (with BPF_JSGT or
-BPF_JSGE) would instead update the signed minimum/maximum values. Information
-from the signed and unsigned bounds can be combined; for instance if a value is
-first tested < 8 and then tested s> 4, the verifier will conclude that the value
-is also > 4 and s< 8, since the bounds prevent crossing the sign boundary.
-
-PTR_TO_PACKETs with a variable offset part have an 'id', which is common to all
-pointers sharing that same variable offset. This is important for packet range
-checks: after adding a variable to a packet pointer register A, if you then copy
-it to another register B and then add a constant 4 to A, both registers will
-share the same 'id' but the A will have a fixed offset of +4. Then if A is
-bounds-checked and found to be less than a PTR_TO_PACKET_END, the register B is
-now known to have a safe range of at least 4 bytes. See 'Direct packet access',
-below, for more on PTR_TO_PACKET ranges.
-
-The 'id' field is also used on PTR_TO_MAP_VALUE_OR_NULL, common to all copies of
-the pointer returned from a map lookup. This means that when one copy is
-checked and found to be non-NULL, all copies can become PTR_TO_MAP_VALUEs.
-As well as range-checking, the tracked information is also used for enforcing
-alignment of pointer accesses. For instance, on most systems the packet pointer
-is 2 bytes after a 4-byte alignment. If a program adds 14 bytes to that to jump
-over the Ethernet header, then reads IHL and adds (IHL * 4), the resulting
-pointer will have a variable offset known to be 4n+2 for some n, so adding the 2
-bytes (NET_IP_ALIGN) gives a 4-byte alignment and so word-sized accesses through
-that pointer are safe.
-The 'id' field is also used on PTR_TO_SOCKET and PTR_TO_SOCKET_OR_NULL, common
-to all copies of the pointer returned from a socket lookup. This has similar
-behaviour to the handling for PTR_TO_MAP_VALUE_OR_NULL->PTR_TO_MAP_VALUE, but
-it also handles reference tracking for the pointer. PTR_TO_SOCKET implicitly
-represents a reference to the corresponding ``struct sock``. To ensure that the
-reference is not leaked, it is imperative to NULL-check the reference and in
-the non-NULL case, and pass the valid reference to the socket release function.
-
-Direct packet access
-====================
-
-In cls_bpf and act_bpf programs the verifier allows direct access to the packet
-data via skb->data and skb->data_end pointers.
-Ex::
-
- 1: r4 = *(u32 *)(r1 +80) /* load skb->data_end */
- 2: r3 = *(u32 *)(r1 +76) /* load skb->data */
- 3: r5 = r3
- 4: r5 += 14
- 5: if r5 > r4 goto pc+16
- R1=ctx R3=pkt(id=0,off=0,r=14) R4=pkt_end R5=pkt(id=0,off=14,r=14) R10=fp
- 6: r0 = *(u16 *)(r3 +12) /* access 12 and 13 bytes of the packet */
-
-this 2byte load from the packet is safe to do, since the program author
-did check ``if (skb->data + 14 > skb->data_end) goto err`` at insn #5 which
-means that in the fall-through case the register R3 (which points to skb->data)
-has at least 14 directly accessible bytes. The verifier marks it
-as R3=pkt(id=0,off=0,r=14).
-id=0 means that no additional variables were added to the register.
-off=0 means that no additional constants were added.
-r=14 is the range of safe access which means that bytes [R3, R3 + 14) are ok.
-Note that R5 is marked as R5=pkt(id=0,off=14,r=14). It also points
-to the packet data, but constant 14 was added to the register, so
-it now points to ``skb->data + 14`` and accessible range is [R5, R5 + 14 - 14)
-which is zero bytes.
-
-More complex packet access may look like::
-
-
- R0=inv1 R1=ctx R3=pkt(id=0,off=0,r=14) R4=pkt_end R5=pkt(id=0,off=14,r=14) R10=fp
- 6: r0 = *(u8 *)(r3 +7) /* load 7th byte from the packet */
- 7: r4 = *(u8 *)(r3 +12)
- 8: r4 *= 14
- 9: r3 = *(u32 *)(r1 +76) /* load skb->data */
- 10: r3 += r4
- 11: r2 = r1
- 12: r2 <<= 48
- 13: r2 >>= 48
- 14: r3 += r2
- 15: r2 = r3
- 16: r2 += 8
- 17: r1 = *(u32 *)(r1 +80) /* load skb->data_end */
- 18: if r2 > r1 goto pc+2
- R0=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R1=pkt_end R2=pkt(id=2,off=8,r=8) R3=pkt(id=2,off=0,r=8) R4=inv(id=0,umax_value=3570,var_off=(0x0; 0xfffe)) R5=pkt(id=0,off=14,r=14) R10=fp
- 19: r1 = *(u8 *)(r3 +4)
-
-The state of the register R3 is R3=pkt(id=2,off=0,r=8)
-id=2 means that two ``r3 += rX`` instructions were seen, so r3 points to some
-offset within a packet and since the program author did
-``if (r3 + 8 > r1) goto err`` at insn #18, the safe range is [R3, R3 + 8).
-The verifier only allows 'add'/'sub' operations on packet registers. Any other
-operation will set the register state to 'SCALAR_VALUE' and it won't be
-available for direct packet access.
-
-Operation ``r3 += rX`` may overflow and become less than original skb->data,
-therefore the verifier has to prevent that. So when it sees ``r3 += rX``
-instruction and rX is more than 16-bit value, any subsequent bounds-check of r3
-against skb->data_end will not give us 'range' information, so attempts to read
-through the pointer will give "invalid access to packet" error.
-
-Ex. after insn ``r4 = *(u8 *)(r3 +12)`` (insn #7 above) the state of r4 is
-R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) which means that upper 56 bits
-of the register are guaranteed to be zero, and nothing is known about the lower
-8 bits. After insn ``r4 *= 14`` the state becomes
-R4=inv(id=0,umax_value=3570,var_off=(0x0; 0xfffe)), since multiplying an 8-bit
-value by constant 14 will keep upper 52 bits as zero, also the least significant
-bit will be zero as 14 is even. Similarly ``r2 >>= 48`` will make
-R2=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)), since the shift is not sign
-extending. This logic is implemented in adjust_reg_min_max_vals() function,
-which calls adjust_ptr_min_max_vals() for adding pointer to scalar (or vice
-versa) and adjust_scalar_min_max_vals() for operations on two scalars.
-
-The end result is that bpf program author can access packet directly
-using normal C code as::
-
- void *data = (void *)(long)skb->data;
- void *data_end = (void *)(long)skb->data_end;
- struct eth_hdr *eth = data;
- struct iphdr *iph = data + sizeof(*eth);
- struct udphdr *udp = data + sizeof(*eth) + sizeof(*iph);
-
- if (data + sizeof(*eth) + sizeof(*iph) + sizeof(*udp) > data_end)
- return 0;
- if (eth->h_proto != htons(ETH_P_IP))
- return 0;
- if (iph->protocol != IPPROTO_UDP || iph->ihl != 5)
- return 0;
- if (udp->dest == 53 || udp->source == 9)
- ...;
-
-which makes such programs easier to write comparing to LD_ABS insn
-and significantly faster.
-
-Pruning
-=======
+ state A: i = 0..7 err = 0 --+
+ +--> ?
+ state B: i = 8..max err = -22 --+
-The verifier does not actually walk all possible paths through the program. For
-each new branch to analyse, the verifier looks at all the states it's previously
-been in when at this instruction. If any of them contain the current state as a
-subset, the branch is 'pruned' - that is, the fact that the previous state was
-accepted implies the current state would be as well. For instance, if in the
-previous state, r1 held a packet-pointer, and in the current state, r1 holds a
-packet-pointer with a range as long or longer and at least as strict an
-alignment, then r1 is safe. Similarly, if r2 was NOT_INIT before then it can't
-have been used by any path from that point, so any value in r2 (including
-another NOT_INIT) is safe. The implementation is in the function regsafe().
-Pruning considers not only the registers but also the stack (and any spilled
-registers it may hold). They must all be safe for the branch to be pruned.
-This is implemented in states_equal().
-
-Some technical details about state pruning implementation could be found below.
-
-Register liveness tracking
---------------------------
+ keep apart check the code after it two times exact, n jumps cost 2^n
+ join i = any, err = -22 or 0 cheap, the relation is lost
-In order to make state pruning effective, liveness state is tracked for each
-register and stack slot. The basic idea is to track which registers and stack
-slots are actually used during subseqeuent execution of the program, until
-program exit is reached. Registers and stack slots that were never used could be
-removed from the cached state thus making more states equivalent to a cached
-state. This could be illustrated by the following program::
+Most joins lose no needed fact, and nothing tells in advance which ones do. So
+the verifier joins first. If the check of an insn fails, it finds the values
+that the insn depends on, and starts again with these values kept apart::
- 0: call bpf_get_prandom_u32()
+ start: join all numbers
+ |
+ v
+ pass -------> error ? --no--> ACCEPT
+ ^ | yes
+ | v
+ +-- yes -- can more be kept apart ? --no--> REJECT
+
+One such check of the program is a *pass*. What a pass keeps apart is its
+*precision*.
+
+Proof and guess
+---------------
+
+One rule holds for the full design:
+
+- A pass without errors, a *good pass*, is a complete proof of the program,
+ whatever it kept apart.
+- Precision is a guess. A wrong guess costs time, or a valid program is
+ rejected. It cannot make the verifier accept an unsafe program.
+
+The same holds for other guesses, such as the bound of a loop: the pass checks
+each of them. See `What is trusted`_.
+
+The steps of a load
+-------------------
+
+``bpf_check()`` has three parts::
+
+ prepare main analysis rewrite
+ ------------------------- --------------------- --------------------------
+ functions, BTF, maps do_check_all(): add run-time checks
+ control flow graph pass 0 remove dead code
+ constant folding pass 1 convert context accesses
+ loops, dominators ... inline helpers
+ liveness JIT
+
+The first part looks only at the code. It rejects the program if an insn is not
+valid, if no path gets to an insn, or if a jump leaves its function.
+
+The state
+=========
+
+``struct bpf_verifier_state`` has a frame for each function on the call chain.
+A frame has the registers r0 - r10 and the stack::
+
+ state
+ +- frame 0: r0 .. r10, fp-8, fp-16, ..., the loops that it is in
+ +- frame 1: ...
+ +- held: references, locks
+ +- equations: r2 = 4 * r1, ...
+
+An *xreg* is a register or an 8-byte stack slot. Most of the verifier does not
+care which one it is. `Loops`_ adds one more kind.
+
+A value has three parts, see ``struct bpf_reg_state``::
+
+ type object number
+ ----------- ---------------- ------------------
+ scalar - the value
+ map_value which map the offset in it
+ fp which frame the offset in it
+ unreadable - -
+
+An *unreadable* xreg has no value. The check of an insn that reads it fails.
+
+Numbers
+-------
+
+``struct scalar`` describes a set of 64-bit numbers in four ways. The number is
+in all four sets. What one of them knows refines the others, see
+``scalar_bounds_sync()``.
+
+======= =========================================== ==========================
+part what it knows log text
+======= =========================================== ==========================
+r64 a range of the 64 bits smin, smax, umin, umax
+r32 a range of the low 32 bits smin32, ... umax32
+var_off bits that are known: (value; unknown bits) ``var_off=(0x0; 0x7c)``
+step ``base + step * k`` ``step=4+12``
+======= =========================================== ==========================
+
+A range is an arc on a circle: a start and a size. So one range gives the
+signed and the unsigned bounds, and -4..3 is one range. Example::
+
+ range var_off step
+ r1 &= 7 0..7 (0x0; 0x7)
+ r1 *= 12 0..84 (0x0; 0x7c) 0+12
+ r1 += 4 4..88 (0x0; 0x7c) 4+12
+ if r1 > 50 goto out 4..40 (0x0; 0x3c) 4+12 4, 16, 28 or 40
+
+Pointers
+--------
+
+========================== ================== ================================
+type log text points to
+========================== ================== ================================
+``PTR_TO_CTX`` ctx the context, r1 at the start
+``PTR_TO_STACK`` fp the stack of a frame
+``CONST_PTR_TO_MAP`` map_ptr a map
+``PTR_TO_MAP_VALUE`` map_value a value of a map, global data
+``PTR_TO_PACKET`` pkt packet data
+``PTR_TO_PACKET_END`` pkt_end the end of the packet
+``PTR_TO_MEM`` mem memory of a known size
+``PTR_TO_BTF_ID`` ptr\_<struct> a kernel object with a BTF type
+``PTR_TO_SOCKET``, ... sock, ... a socket
+``PTR_TO_ARENA`` arena arena memory
+``PTR_TO_FUNC`` func a BPF function
+========================== ================== ================================
+
+Flags add to a type. ``PTR_MAYBE_NULL`` is for a pointer that can be NULL
+(``map_value_or_null``). ``MEM_RDONLY`` is for read-only memory.
+``PTR_TRUSTED`` and ``PTR_UNTRUSTED`` say if the object is known to be alive.
+
+Copies of one pointer share an *id*. What the verifier learns about one copy
+holds for all of them::
+
+ r0 = bpf_map_lookup_elem() r0 = map_value_or_null(id=1)
+ r6 = r0 r6 = map_value_or_null(id=1)
+ if r0 == 0 goto out r0 and r6 are map_value
+ r0 = *(u64 *)(r6 + 0) safe
+
+An id stands for the set of xregs that have it. ``bpf_renumber_ids()`` names
+each id after the first xreg of its set. So two states with the same sets have
+the same ids. The log prints these numbers. The id of an object can change from
+one basic block to the next, and another object can get its old number.
+
+The stack
+---------
+
+Each byte of a slot has a kind:
+
+======================= ======== =============================================
+kind log text meaning
+======================= ======== =============================================
+invalid ``?`` not written
+spill value a part of a register; the slot has its value
+misc ``m`` some data
+zero ``0`` zero
+dynptr, iter name an object that the kernel put there
+irq flag ``f`` the same
+poison ``p`` nothing reads it later, see `Liveness`_
+======================= ======== =============================================
+
+A store of a register to an aligned slot is a *spill*. The load gives the type
+and the number back.
+
+An access with many offsets does not know its slot. A write makes misc of the
+bytes in its range, and a load gives a number, see *fork* in `Partition`_. A
+call with a constant size skips the bytes between two offsets: 8 bytes at
+``fp-24`` or at ``fp-8`` do not touch ``fp-16``.
+
+What is held
+------------
+
+Some calls give the program something that it has to give back: a reference, a
+spin lock, the RCU lock, disabled preemption, a dynptr, an iterator, an irq
+flag. All of that is *held*. At ``exit`` of the program all references and locks
+have to be released. A release makes all copies of the pointer unusable.
+
+Some dynptrs have no reference and need no release. They count as held too.
+
+The check of one insn
+=====================
+
+``do_check_insn()`` checks an insn in one state and changes the state. If the
+check fails, the insn is *unproven* in that state. Else it is *proven*. This is
+not the verdict: a later pass can prove the insn. Only a program is accepted or
+rejected.
+
+Registers
+ An insn can read only a register that has a value. At the start these are r1
+ and r10. A call makes r1 - r5 unreadable and keeps r6 - r9.
+
+Arithmetic on numbers
+ It is safe for all values. The result is a new set of numbers.
+
+Arithmetic on pointers
+ ``pointer += number`` and ``pointer -= number`` change the offset. The number
+ needs a lower bound. With ``CAP_PERFMON`` the pointer can go out of the
+ object: only an access has to be in bounds. A pointer that can be NULL has to
+ be tested first. A stack pointer allows only ``+=``, and some types allow no
+ arithmetic. Other operations make a number of the pointer, which needs
+ ``CAP_PERFMON``.
+
+Loads and stores
+ All offsets that the pointer can have must be good:
+
+ =========== =================================================================
+ map value in the value; special fields such as a lock need a helper
+ stack in the stack, aligned; written before, without ``CAP_PERFMON``
+ context the program type decides, see ``is_valid_access()``
+ packet in the bytes that the program checked, see below
+ BTF object in a member of the type; the member gives the type of the result
+ arena no check, the JIT keeps the address in the arena
+ =========== =================================================================
+
+ An atomic access has to be aligned to its size. With
+ ``BPF_F_STRICT_ALIGNMENT``, or if the cpu has no fast unaligned access, all
+ accesses have to be aligned. ``BPF_F_ANY_ALIGNMENT`` turns that off. The check
+ assumes that a packet starts 2 bytes after an aligned address.
+
+Jumps
+ A conditional jump makes two states and *refines* the operands in each. If
+ the values decide the jump, only one edge is taken.
+
+Calls of helpers and kfuncs
+ The prototype says what each argument has to be: a type, a size, a constant.
+ It also gives the type of r0, and says what the call acquires or releases.
+ The program type selects the functions that can be called.
+
+Exit
+ r0 must be in the range that the program type allows. All references and
+ locks must be released.
+
+Packets
+-------
+
+The length of a packet is known only at run time. A compare with the end of the
+packet tells how many bytes are there. The log shows that as ``r``. If the
+pointer has one offset, the bytes from the start of the packet up to ``r`` are
+*checked*::
+
+ r2 = skb->data r2 = pkt(r=0)
+ r3 = skb->data_end r3 = pkt_end
+ r4 = r2 + 14 r4 = pkt(r=0, offset 14)
+ if r4 > r3 goto out r2 = pkt(r=14)
+ r0 = *(u16 *)(r2 + 12) safe: 12 + 2 <= 14
+
+If a number with many values is added, the pointer gets a new id. Pointers with
+one id differ by a constant, so one compare is good for all of them. Now ``r``
+goes with the largest offset: 82 - 60 = 22 bytes after r2 are checked::
+
+ r5 = 0..60 the length of a header
+ r2 += r5 r2 = pkt(id=1, r=0, offset 0..60)
+ r4 = r2 + 22 r4 = pkt(id=1, r=0, offset 22..82)
+ if r4 > r3 goto out r2 = pkt(id=1, r=82, offset 0..60)
+ r0 = *(u16 *)(r2 + 14) safe: 60 + 14 + 2 <= 82
+
+This holds only for offsets up to 64K (``MAX_PACKET_OFF``). A helper that can
+move the packet data makes all packet pointers unusable.
+
+A pointer also keeps how many bytes are checked from the start of the packet.
+The log shows that as ``min_len``, if it is not ``r``. If the largest offset
+gets below ``min_len``, ``r`` is at least ``min_len``::
+
+ r2 += 20 r2 = pkt(r=54, offset 20)
+ r2 += 40..60 r2 = pkt(id=1, r=0, min_len=54, offset 60..80)
+ r2 += -50..-40 r2 = pkt(id=2, r=54, offset 10..40)
+
+A pass
+======
+
+A pass checks the main function, then each global function that is called.
+The states move through the program like this::
+
+ +---------+ next +-------------------+
+ +---->| pending |---------->| check basic block |
+ | +---------+ +---------+---------+
+ | new v
+ | +-------------------+ nothing new
+ +---------------------------| join point |---------------> drop
+ joined, or kept apart +-------------------+
+
+A *basic block* is a run of insns without a jump into it or out of it. A *join
+point* is an insn where states can meet. The check of a function ends when no
+state is pending.
+
+The verifier counts its work. That is the *cost*, see `Limits`_. A *budget* is a
+limit on the cost of one step.
+
+Liveness
+--------
+
+An xreg is *live* at an insn if a path from the insn reads it before it writes
+it. What is not live cannot matter. So a state that arrives at a basic block is
+cleaned first, see ``bpf_clean_verifier_state()``. Then more states are equal::
+
+ 0: r0 = random
1: r1 = 0
- 2: if r0 == 0 goto +1
+ 2: if r0 == 0 goto 4
3: r0 = 1
- --- checkpoint ---
- 4: r0 = r1
+ 4: r0 = r1 r0 is not live: both states are "r1 = 0"
5: exit
-Suppose that a state cache entry is created at instruction #4 (such entries are
-also called "checkpoints" in the text below). The verifier could reach the
-instruction with one of two possible register states:
+Prune
+-----
+
+State A *subsumes* state B if A has all values of B. If the program is safe
+from A, it is safe from B. B needs no check, it is *pruned*::
+
+ A: r1 = 0..9 r2 = buf + 0..8
+ B: r1 = 3 r2 = buf + 4 A subsumes B
+
+An unreadable xreg subsumes each value: the program is safe from A without a
+read of it. See ``bpf_state_subsumes()``.
+
+Join
+----
+
+The *join* of two states has all values of both, see ``bpf_states_join()``::
+
+ 0..7 | 9 -> 0..9
+ 0 | 56 -> 0..56, step 56
+ buf + 0 | buf + 8 -> buf + 0..8
+ map_value | scalar -> unreadable
+ buf | other_buf -> unreadable
+ ctx + 0 | ctx + 8 -> unreadable, ctx has no range of offsets
+
+If an insn reads the unreadable xreg, the insn is unproven, and the next pass
+keeps the two types apart. This is rare, see `Two types in a live xreg`_.
+
+A join can add values that no path has, for example 8 in the first line. After
+the first join the pass is *imprecise*: an error does not prove that the
+program is wrong. Later chapters have more steps that add or keep such values.
+They make the pass imprecise too.
+
+What has no join
+----------------
+
+"The lock is held or not held" has no meaning for the verifier. All such facts
+are the *join_key* of a state. States with two join_keys never meet:
+
+=============== ===============================================================
+what is held references, locks, dynptrs, iterators, irq flags
+where it is the call chain, the instance (see `Functions`_)
+which trip the peeled trip of each loop (see `Loops`_)
+=============== ===============================================================
+
+Prune or join
+-------------
+
+A state that arrives is compared with the states that are at the insn, see
+``flow_prune_or_join()``:
+
+=================================== ===========================================
+if action
+=================================== ===========================================
+they have two join_keys they do not meet
+they must stay apart (next chapter) they do not meet
+the old one subsumes the new one prune the new one
+the new one subsumes the old one prune the old one
+none of these holds join them
+=================================== ===========================================
+
+Two types in a live xreg
+------------------------
+
+This code is common::
+
+ p = NULL;
+ if (...)
+ p = buf;
+ if (p) if joined, p is unreadable and this read is unproven
+ use(p);
+
+If the two states are joined, an error is almost sure, and it costs a pass. So
+states with two types or two objects in a live xreg stay apart already in pass
+0, see ``bpf_hash_live_types()``. The same holds for a packet pointer with and
+without checked bytes.
+
+N such xregs can make 2^N states. So this rule has a budget. If a pass is over
+it, the pass joins from there on, and so do the next passes. The rule does not
+apply to the arguments of two calls, see `Instances`_.
+
+The order
+---------
+
+A join point has to get all its states before it runs. Else the code after it
+is checked again for each late state. ``flow_set_pending()`` sets the order:
+both branches before the code after them, an iteration before the next one, a
+loop before the code after the loop. See `Instances`_ for calls. The insns are
+in a weak topological order: reverse post order, and each loop is in one piece.
+
+With this order most states can meet no later state, and are freed at once.
+``flow_keep()`` keeps a state only where another one can come: in a loop, at the
+start of a function, after a call that can return with one more state, or in an
+async callback.
+
+Precision: data and control dependences
+=======================================
+
+This example is used from here on::
+
+ char buf[8]; 0: r0 = random
+ 1: r6 = r0
+ u64 i = random(); 2: r7 = 0
+ int err = 0; 3: if r6 <= 7 goto 5
+ if (i > 7) 4: r7 = -22
+ err = -22; 5: r0 = 0 join point
+ ... 6: if r7 != 0 goto 11
+ if (err) 7: r1 = buf
+ return 0; 9: r1 += r6
+ return buf[i]; 10: r0 = *(u8 *)(r1 + 0)
+ 11: exit
+
+The program is safe. Pass 0 does not prove it::
+
+ r6 r7
+ from 3 to 5 0..7 0
+ from 4 to 5 8..max -22
+ join any -22 or 0
+ after insn 6 any 0 no path has r6 = 8 with r7 = 0
+ insn 9 unproven: r6 has no bound
+
+The next pass has to keep the two states at insn 5 apart. To keep all states
+apart is too costly. Which xregs does insn 9 depend on?
+
+============== ================================================================
+*data_deps* xregs whose value flows into an operand of the unproven insn
+*control_deps* xregs whose value flows into a jump that can skip that insn
+============== ================================================================
+
+Both sets together are the *deps*.
+
+How they are found
+------------------
+
+``bpf_compute_deps()`` makes a backward slice. It goes back from *seeds*, in the
+same way as liveness. If an insn writes an xreg of the set, that xreg leaves the
+set, and what the insn reads comes in::
+
+ liveness starts from the reads of all insns
+ data_deps start from the reads of the unproven insn
+ control_deps start from the reads of the jumps that can skip it
+
+ before the insn:
+ data control
+ 1: r6 = r0 r0 -
+ 2: r7 = 0 r6 -
+ 3: if r6 <= 7 goto 5 r6 r7
+ 4: r7 = -22 r6 -
+ 5: r0 = 0 r6 r7
+ 6: if r7 != 0 goto 11 r6 r7 <- seed of control_deps
+ 7: r1 = buf r6 -
+ 9: r1 += r6 r1 r6 - <- seed of data_deps
+
+Insn 6 can skip insn 9: one edge leads to it and the other edge does not. Both
+edges of insn 3 lead to insn 9, so insn 3 is no seed. See
+``find_control_seeds()``.
+
+The sets go through spills, calls, returns and the frames of callers. What is
+held is tracked like an xreg: the insns that acquire or release write it.
+
+The join rule
+-------------
+
+One rule decides at all join points, ``bpf_must_keep_apart()``. It looks only at
+the deps that differ in the two states:
+
+================================= ========== =================================
+what differs action why
+================================= ========== =================================
+nothing in data_deps join the insn gets the same operands
+one xreg of data_deps, no gap join 0..7 and 8..9 are 0..9: nothing
+ is added
+one xreg of data_deps, with a gap keep apart 0..1 and 100: the join has 50
+a type or an object in data_deps keep apart the join is unreadable
+two xregs, one is in data_deps keep apart the join has pairs that no path
+ has
+================================= ========== =================================
+
+A *gap* is a value of the join that is in neither state. The numbers 0 and 3
+have no gap: the join is 0..3 with step 3. ``buf + 0`` and ``buf + 3`` have one,
+because a load ignores the step. Two packet pointers with one id have a gap
+if their distance differs: the join gives them two ids.
+
+The last line does not apply if one state has all values of the other one in
+the deps: then the join adds no pair.
+
+In pass 0 the deps are empty, so all numbers are joined. In pass 1 of the
+example r6 (data) and r7 (control) differ at insn 5::
+
+ r6 r7
+ from 3 to 5 0..7 0 -> insn 9 is proven
+ from 4 to 5 8..max -22 -> goes to insn 11
+
+r7 stores the result of the test at insn 3: it is a *stored condition*. Insn 6
+tests it, so the two branches are *correlated*. Other values are still joined.
+A stored condition that only selects a message keeps no states apart. This is
+why the number of states stays small.
+
+One pass finds many errors
+--------------------------
+
+A pass does not stop at its first error. It notes the insn and continues as if
+the insn was proven. This is error recovery, as in a compiler. See
+``skip_unproven_insn()``::
+
+ r0 = *(u64 *)(r2 + 0) unproven, r0 is unknown now
+ r4 = *(u8 *)(r0 + 0) unproven because of r0: not a new error
+ r5 = *(u8 *)(r7 + 0) unproven: a second seed
+
+So the next pass has the deps of all unproven insns. A pass with an error
+accepts nothing.
+
+If an unproven call changes what is held, the path ends there. The later uses
+of the lock or of the dynptr are not new errors, and must not add deps.
+
+The pass stops when it has found all errors of the last pass again, or when it
+is over a budget that starts at its first error. It does not stop at the errors
+if it found a new xreg that moves with a trip index. Later trips can show that
+the xreg does so only at first.
+
+Equations
+=========
+
+A range does not tell how two numbers are related::
+
+ r2 = r1
+ r2 <<= 3
+ if r1 > 7 goto out r2 is 0..56, but only r1 was tested
+
+So each state has a table of *equations*. A *sym* is a number that equations
+use. There are three kinds. All hold mod 2^64, as the registers do::
+
+ xreg = coef * sym + off r2 = 8 * sym0
+ sym = coef * sym + coef * sym + off (sum) sym2 = sym0 + sym1
+ sym = what an insn makes of two values (op) sym3 = sym2 & 0xf8
+
+For a pointer the number is its offset. A state says: there are numbers for the
+syms, in their ranges, for which all equations hold.
+
+An insn makes equations
+-----------------------
+
+An insn deletes the equation of the xreg that it writes. The result gets a new
+one::
+
+ r2 = r1 r1 = sym0, r2 = sym0
+ r2 += 8 r2 = sym0 + 8
+ r2 <<= 2 r2 = 4 * sym0 + 32
+ r2 += r3 r2 = sym2, sum: sym2 = 4 * sym0 + sym1 + 32
+ w4 = w1 r4 = low, sum: low = sym0 - 2^32 * high
+ r1 ^= 5 r1 = sym3, op: sym3 = sym0 ^ 5
+ *(fp - 8) = r1 fp-8 has the equation of r1
+
+A jump uses them
+----------------
+
+A jump refines its operand, then all numbers that are related::
+
+ xreg -> its sym -> the other syms -> their xregs
+
+ if r1 > 7 goto out r1 0..7 -> sym0 0..7 -> r2 0..56
+
+A sum refines all of its syms. An op refines only its result. A jump that
+compares two numbers also compares their difference with 0, if the difference
+cannot wrap::
+
+ off = 0..63 off = sym0
+ left = 64 - off left = 64 - sym0
+ if (len > left) goto out len = sym1, sym2 = len - left, sym2 <= 0
+ so off + len = 64 + sym2 <= 64
+
+An equation has a cost. So this jump makes sym2 only if left is a constant minus
+a number, and len is in data_deps. If len is not there, the pass is imprecise.
+Pass 0 has no deps, so this example needs pass 1.
+
+A jump can compare a number with its low 32 bits. Code with no 32-bit insns does
+that to test the carry of a sum::
+
+ r5 = r2; r5 <<= 32; r5 >>= 32
+ if r5 != r2 goto carry if the jump is taken, r2 is above U32_MAX
-* r0 = 1, r1 = 0
-* r0 = 0, r1 = 0
+If a range becomes empty, the equations cannot hold. No run of the program has
+such a state: it is *infeasible*. It ends after the next conditional jump.
-However, only the value of register ``r1`` is important to successfully finish
-verification. The goal of the liveness tracking algorithm is to spot this fact
-and figure out that both states are actually equivalent.
+A call uses them
+----------------
-Understanding eBPF verifier messages
-====================================
+A helper or a kfunc gets a pointer and a size. The largest offset and the
+largest size do not have to go together::
-The following are few examples of invalid eBPF programs and verifier error
-messages as seen in the log:
+ n = bpf_get_stack(ctx, buf, 800, 0);
+ if (n < 0) return; n is 0..800
+ bpf_get_stack(ctx, buf + n, 800 - n, 0);
-Program with unreachable instructions::
+ offset size offset + size
+ ranges 0..800 0..800 0..1600
+ equations sym0 800 - sym0 800
- static struct bpf_insn prog[] = {
- BPF_EXIT_INSN(),
- BPF_EXIT_INSN(),
- };
+The check asks the equations for the largest sum. It does that for map_value, fp
+and mem. It does not do that for packets: their checked bytes are relative to
+the pointer.
-Error::
+A join guesses them
+-------------------
- unreachable insn 1
+A join looks for equations that hold in both states. It makes a sym that has
+one value in each state::
-Program that reads uninitialized register::
+ r6 r7
+ state A 0 56
+ state B 56 0
+ join 56 * sym0 56 - 56 * sym0 sym0 is 0..1
+ r6 += r7 56 not 0..112
- BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
- BPF_EXIT_INSN(),
+So many programs need no second pass. These equations are a guess.
+``check_equations()`` checks them in both states and deletes what fails.
-Error::
+A prune checks them
+-------------------
- 0: (bf) r0 = r2
- R2 !read_ok
+A state with an equation has fewer values than its ranges show::
-Program that doesn't initialize R0 before exiting::
+ old: r1 0..7, r2 0..7, r1 == r2
+ new: r1 0..7, r2 0..7 old does not subsume new
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_1),
- BPF_EXIT_INSN(),
+``check_equations()`` looks for a value in the new state for each sym of the
+old state. If there is none, there is no prune.
-Error::
+A state without an equation has more values. So it is always safe to delete an
+equation. ``normalize_equations()`` deletes those that say nothing, and keeps
+the table small.
- 0: (bf) r2 = r1
- 1: (95) exit
- R0 !read_ok
+The join rule uses the equations too. If only one of two states has
+``r1 == r6``, the two xregs count as different.
-Program that accesses stack out of bounds::
+Loops
+=====
- BPF_ST_MEM(BPF_DW, BPF_REG_10, 8, 0),
- BPF_EXIT_INSN(),
+A *trip* is one iteration. The *header* is the insn where each trip starts. A
+back edge leads to it. The verifier has to prove two things: each insn is safe
+in all trips, and the loop ends.
-Error::
+One state for all trips
+-----------------------
- 0: (7a) *(u64 *)(r10 +8) = 0
- invalid stack off=8 size=8
+The state that comes back to the header is joined with the state that is there.
+The body is checked again with the join. This stops when the state at the
+header subsumes the state that comes back. That is the *fixpoint*.
-Program that doesn't initialize stack before passing its address into function::
+A counter grows in each trip, so plain joins do not stop. The first growth is a
+join. After that the bound is *widened*: it jumps to a *threshold*::
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_LD_MAP_FD(BPF_REG_1, 0),
- BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
- BPF_EXIT_INSN(),
+ i = 0;
+ while (random()) {
+ if (!big[i]) char big[64]
+ continue;
+ i += 3;
+ if (i >= 30)
+ break;
+ }
-Error::
+ i at the header:
+ pass 0 0 -> 0..3 -> 0..S32_MAX big[i] is unproven
+ pass 1 0 -> 0..3 -> 0..29 i is in the deps now
- 0: (bf) r2 = r10
- 1: (07) r2 += -8
- 2: (b7) r1 = 0x0
- 3: (85) call 1
- invalid indirect read from stack off -8+0 size 8
+The thresholds of an xreg in the deps are the constants of the loop, one less,
+and one step more. Other xregs stop at 0, ``S32_MAX``, ``U32_MAX`` or
+``S32_MIN``. A threshold is a guess. If it is too small, the state that comes
+back is not subsumed, and the bound goes to the next threshold.
+
+An *entry* is a state that enters the loop. The join rule keeps two entries
+apart, as at any join point. A state that comes back meets the state of its own
+entry. There the join rule does not apply to numbers, else the check of the
+loop has no end.
+
+The trip index
+--------------
+
+The *trip index* k counts how many times the program came back to the header.
+It is an xreg: 0 at the entry, + 1 at each back edge. So it is the canonical
+induction variable of the loop. The join at the header finds the xregs that move
+with it, the other induction variables (*iv*). A *walk* is one check of the
+body::
+
+ i = 0; p = big;
+ do {
+ use(*(u64 *)p);
+ i++; p += 8;
+ } while (i < 8);
+
+ at the header:
+ walk 1 k = 0 i = 0 p = big
+ walk 2 k = 0..1 i = k p = big + 8 * k the join finds these
+ walk 3 k = 0..7 i = k p = big + 8 * k 'i < 8': threshold 7
+
+A jump that tests a number that moves with k gives k a threshold. This needs no
+precision, so it works in pass 0.
+
+In walk 3 the jump refines i, so k, so p. No jump tests p. The back edge brings
+k = 1..7, and the header subsumes that.
+
+Walks 1 and 2 only collect the steps and the threshold. Often the code shows
+them. Then ``bpf_guess_first_trips()`` starts with walk 3, and the body is
+checked once. This is a guess too.
+
+Two entries can have two bounds. So each entry has its own threshold for k.
+
+The loop must end
+-----------------
+
+A state that is pruned by its own ancestor closes a cycle::
+
+ t --> ... --> header --> ... --> s
+ ^ |
+ `----------- subsumes -----------'
+
+Each state also counts how many times the verifier came back to the header with
+it. A join has the larger count. All pruned states go through
+``flow_check_trip_counts()``. If s has a larger count than t, s can come from
+t, and something has to stop the loop:
+
+===================================== =========================================
+if the loop ends because
+===================================== =========================================
+k of s is at most 2^32 k grows in each trip and has a bound;
+ the largest k + 1 is the *trip count*
+each trip runs ``may_goto`` it leaves the loop after some time or
+ some number of trips
+each trip takes from an iterator an iterator has an end
+none of these holds the verifier adds ``may_goto``
+===================================== =========================================
+
+*may_goto* is a jump that is taken when the program has looped too long. The
+verifier puts it before a jump that leaves the loop, or at the header. It
+checks the new edge like any other edge::
+
+ do { do {
+ p = lookup(); p = lookup();
+ } while (!p); may_goto leave;
+ use(p); } while (!p);
+ use(p);
+ ...
+ leave: return 0;
+
+Where does the edge go? The verifier takes the first line that is possible:
+
+- In the main function: ``return 0``, if nothing has to be released and 0 is a
+ valid result.
+- In another function: *unwind*, if nothing has to be released and each call has
+ an ``unwind_goto`` after it. The function leaves with ``unwind_exit``, and
+ ``unwind_goto`` jumps to the code of the caller that cleans up.
+- The exit of the loop. The code there has to be safe. From a header the edge
+ leaves the outer loops too. If the code there reads a register that is not
+ set, the function returns 0.
+
+A loop without an exit is an error.
+
+A pass assumes what stops a loop
+--------------------------------
+
+At run time ``may_goto`` is there for all states. So a pass has to know at the
+start which loops get it:
+
+- Pass 0 does not know the trip counts. It checks the new edge in all loops.
+ After a good pass the verifier drops each ``may_goto`` that no state needed.
+- A later pass checks the edge only in the loops that needed it before.
+- An iterator is assumed to stop its loop. It does not stop a loop that creates
+ it in each trip, or a loop that goes on when no element is left.
+
+If an assumption is wrong, the pass starts again, see ``pass_needs_restart()``.
+
+Peeled trips and the leftover
+-----------------------------
+
+One state for all trips loses what differs between trips::
+
+ x = 1;
+ for (i = 0; i < 5; i++)
+ x *= 2;
+ big[x] x is 32, but the widened x has no bound
+
+To *peel* a trip is to check it apart from the other trips. All trips that are
+not peeled are the *leftover*::
+
+ entry -> [0] -> [1] -> ... -> [d-1] -> +----------+ <--+
+ peeled trips | LEFTOVER | |
+ | widened |----+
+ +----------+
+
+Pass 0 peels nothing. A later pass peels if an error asks for it. Peeling stops
+when the loop exits, when a trip has nothing new in the deps, or when the budget
+is used up. If all trips are peeled, no state closes a cycle, and the loop needs
+nothing to stop it. If an earlier trip subsumes a peeled trip, the trip is
+pruned, and the loop gets ``may_goto``.
+
+Which loop has to be peeled? If the join at a header adds to an xreg of
+data_deps, the state notes that loop. If an insn is unproven in the state later,
+the noted loops are the *suspects*::
+
+ for (i = 0; i < 8; i++)
+ sum += i; makes sum imprecise: suspect
+ while (random())
+ n++; not a suspect, it is not peeled
+ big[sum] unproven
+
+Pass 0 has no deps, so it finds no suspect here. Pass 1 finds it, and pass 2
+peels.
+
+A state also notes a loop if it leaves by a ``may_goto`` that the verifier
+added, or if it holds more after each trip.
+
+What is held in a loop
+----------------------
+
+What is held has no join. A loop that acquires in each trip makes new states
+without end. ``check_held_in_leftover()`` stops that: a reference that is in no
+xreg is a leak, and the number of peeled trips limits the locks.
+
+Functions
+=========
+
+================== =========================== ===============================
+function checked the caller gets
+================== =========================== ===============================
+global for all arguments of its a number
+ BTF prototype
+static, on its own once for each instance the exit states of the instance
+static, in frames at each call, with the the state at the exit
+ frames of the callers
+================== =========================== ===============================
+
+A function that is checked *on its own* has a state without the frames of its
+callers. A static function is checked *in frames* if it can change what its
+callers see: it gets a pointer to the stack, tests a pointer for the caller,
+moves packet data, or releases what the caller holds. A callback is always in
+frames. See ``func_is_on_its_own()``. If only the exit state shows such a
+change, the call is checked again in frames. An error can ask for frames too,
+see IN_FRAMES in `Partition`_.
+
+Instances
+---------
+
+18 levels with 2 calls each are 2^18 call chains. To check a function on its
+own avoids that. The states of two calls meet only at the first insn of the
+function. An *instance* is an entry state, its exit states and the calls that
+wait for them. It is a function summary for one entry state.
+
+Callers run first, so that calls collect at the first insn. The function runs
+when a call has no exit state to continue with. Here two calls are on two
+branches, and both arrive before f starts::
+
+ call1 f(7) --+
+ call2 f(any) --+--> first insn
+
+ in pass 0 one instance, both calls get "any"
+ if r1 is in data_deps two instances, call1 gets 7
+
+At the first insn one xreg of data_deps that differs is sufficient to keep
+the calls apart.
+
+An instance that has started does not grow. If a call comes later, the join
+starts a new instance::
+
+ call1 f(0) instance 1: f(0) exits 1 --> call1
+ call2 f(1) instance 2: f(0..1) exits 1 --> call2 at once
+ exits 2 --> call2 later, if they have more
+
+Instance 2 subsumes instance 1, so exits 1 are a part of exits 2. They can add
+values to call2. They cannot hide a value.
+
+Instances can multiply too, so they have a budget. When a function has used its
+part, the next calls are joined.
+
+Calls that go around
+--------------------
+
+The calls of a callback and recursive calls go around like the body of a loop.
+They use the same code: join, widen, peel.
+
+================== ============================ ==============================
+kind header what stops it
+================== ============================ ==============================
+loop the target of the back edge trip count or ``may_goto``
+callback the call of the helper the helper
+recursion the start of the function a check at run time
+================== ============================ ==============================
+
+A recursive function starts with ``may_goto`` and a test of the free stack. If
+there is no room, it returns 0 or unwinds. The verifier checks that edge too.
+A recursion in frames is not joined. It has to end in ``MAX_CALL_FRAMES``.
+
+An async callback, for example of a timer, runs later. It starts as a new state
+with an empty stack. ``callx`` is checked as a call of each function that the
+register can hold. ``bpf_throw()`` ends the program from any frame, so nothing
+can be held there.
+
+After an error
+==============
+
+If an insn is unproven, the state can have values that no path has. The
+verifier has two tools:
+
+============ ================= ===============================================
+tool when what
+============ ================= ===============================================
+partition after the pass keep more states apart in the next pass
+failing path after a partition if the error is real: limit the passes, log it
+============ ================= ===============================================
+
+Partition
+---------
+
+A *partition* keeps more states apart in the next pass. Abstract interpretation
+calls this trace partitioning. More precision costs more. So
+``bpf_partition_more()`` takes the cheap steps first. It takes the first line of
+this list that adds something.
+
+ADD_DEPS
+ Add the deps of the unproven insns. If the insn reads an unreadable xreg,
+ keep only the types apart first.
+
+PEEL
+ Peel the suspects. If the last trip is known: all trips. Else one trip, then
+ as many as the budget pays for. The first trip is often different::
+
+ off = 255; ok = 0;
+ while (...) { if (ok) big[off]; off = x & 63; ok = x | 1; }
+
+PEELED_APART
+ States that start the same peeled trip are joined. Now the join rule applies
+ there too::
+
+ for (i = 0; i < 7; i++) { if (x) a++; else b++; big[a * a + b * b]; }
+
+TRANSITIVE_DEPS
+ A jump can decide which value an xreg of data_deps gets. Add control_deps to
+ data_deps, and add the jumps that can skip a write to data_deps. Each such
+ partition goes one level further::
+
+ ok = i <= 7; ... off = 0; if (ok) off = i; buf[off];
+
+DISJOINT_APART
+ Values in data_deps without a common number stay apart, also without a gap::
+
+ d = x ? 2 : 3; buf[6 / d * d + 1];
+
+LOOP_DISJOINT_APART
+ The same in a suspect, also at the header of the leftover. It is for an xreg
+ that has few values in a long loop. It replaces peeling::
+
+ while (...) { if (n * n < 49) n++; if (x) n = 0; buf[n]; }
+
+PEEL_MORE
+ If PEEL was too costly, or if it stopped at a pruned trip, peel a fixed number
+ of trips, more in each pass. Do not prune them.
+
+IN_FRAMES
+ On its own a function does not get the equations of its arguments. If such an
+ argument is in the deps and the function is not recursive, check the call in
+ frames. That can multiply the work, so this line is the last one::
+
+ if (!ok(x)) return 0; buf[x];
+
+After an error two insns can *fork* a state: the insn is checked once for each
+part of the values of an operand, and the states meet at the next insn.
+
+- ``x u>= 3`` leaves -4..-1 and 3 of the range -4..3, and one range cannot show
+ the gap. If x is in the deps, ``cut_range()`` does the jump once for -4..-1
+ and once for 0..3.
+- A load from the stack with many offsets makes a number of a spilled pointer.
+ If an insn needed a pointer and got that number, ``fork_stack_load()`` does
+ the load once for each slot::
+
+ char *p[2] = { a, b };
+ bpf_for(i, 0, 2) sum += p[i][63];
+
+A partition can be the wrong one. To peel a loop of 100000 trips does not help
+if the error needs control_deps. So each partition but ADD_DEPS has a budget,
+and is undone if the pass is over it::
+
+ +-----------------+ in the budget -> keep for now
+ +--->| partition, pass |-- it helped -> keep for good
+ | +--------+--------+
+ | | over the budget
+ | v
+ +--- undo, take the next line
+ | | no line left, one was too costly
+ | v
+ +--- a larger budget, undo all since a partition helped
+
+A partition *helped* if the first unproven insn of the last pass has no error
+now, and no old error came back, see ``partition_helped()``::
+
+ for (i = 0; i < 8; i++) sum += i;
+ big[sum] A
+ ok = x <= 7; off = 0; if (ok) off = x;
+ big[off] B
+
+ pass partition A B
+ 0 - unproven unproven
+ 1 ADD_DEPS unproven unproven
+ 2 PEEL proven unproven PEEL helped
+ 3 TRANSITIVE_DEPS proven proven
+
+If a partition was too costly for a loop, a later one is a second choice. It is
+undone when the budget grows, also if it helped.
+
+A pass can cost what a pass cost before, plus the budget. The suspects divide
+a part of it. The first budget grows with the cost of a pass, and it is at least
+``BPF_MIN_BUDGET``.
+
+The failing path
+----------------
+
+After a join a pass cannot tell a real error from a false one. If the error is
+real, more passes are wasted work, and the log of a joined state does not show
+the bug. So ``find_failing_path()`` tries to reproduce the error with no joins.
+It checks one path after another until the check of an unproven insn fails. This
+is a *path-sensitive run*. The path on which the check fails is the *failing
+path*. It has two uses:
+
+- The program is probably wrong, so the next passes get a small budget. A path
+ with a guess is the exception, see below.
+- The log shows this path, from the first insn to the error.
+
+If there is no failing path, the passes continue with the same limit.
+
+The example with ``i > 8``::
+
+ insns r6 r7
+ 0 1 2 3 any 0 both edges are possible
+ 3 -> 4 9..max 0 waits
+ 3 -> 5 0..8 0 goes first, it is nearer to insn 9
+ 5 6 7 9 0..8 0
+ 10 0..8 0 r1 = buf + 0..8: buf[8] fails
+
+The failing path is 0 1 2 3 5 6 7 9 10. The check of insn 9 does not fail now.
+Insn 10 reads the result of insn 9, so it was unproven in pass 0 too.
+
+The run has the same checks as a pass, and other rules:
+
+- There are no joins. Each call is in frames. Each trip is peeled.
+- It goes depth first. The edge with the shortest way to an unproven insn goes
+ first.
+- A state is dropped if a state with the same values in the deps was at the
+ insn.
+- If only the jump on the iv depends on the xregs that moved in a trip, and all
+ other live xregs are the same, the path skips trips.
+- If a loop does not end, the path continues where ``may_goto`` leaves it.
+
+The path can end only at an unproven insn of the last pass. This includes an
+insn that was unproven because of another one. An error at any other insn drops
+the path.
+
+The run needs the deps, so it comes after the partition. It starts when there
+is something new: after ADD_DEPS, after a partition helped, after the budget
+grew, and before the final reject. It does not start again when there is a
+failing path.
+
+Three guesses of the run can make a path that the program cannot take. If such
+a path cuts the budget, a valid program can be rejected. So a path of such a
+run, a *path with a guess*, goes to the log, but the budget is not cut. If a
+pass has a real error, the log has that error.
+
+The first guess is the skip. Two trips show a step also for an xreg that has no
+constant step::
+
+ for (i = 0; i < 1000; i++) sum += i;
+ buf[sum - 499500] valid, this is buf[0]
+
+ trip i sum real sum
+ 2 2 1 1 saved
+ 3 3 3 3
+ 4 4 6 6 since trip 2: i +2, sum +5; 996 trips left
+ skip 998 2491 497503 497 times: (996 - 1) / 2
+ 999 999 3489 498501
+ exit 1000 4488 499500 buf[-495012] fails
+
+``bpf_steps_are_constant()`` looks at the xregs that the skip moves, and at the
+operands of the jump on the iv. One "yes" makes the skip a guess:
+
+================================== ============================= =====
+if the loop writes the xreg with example guess
+================================== ============================= =====
+an add of an immediate ``i += 1`` no
+a copy of an xreg with equal step ``x = (u32)i`` no
+what a pass found linear a spilled iv no
+anything else ``sum += i``, ``i < n / i`` yes
+================================== ============================= =====
+
+The second guess is "the loop is infinite". Two trips give more than 2^32 trips
+also for a ring index. The same function finds it::
+
+ u64 i = 600; do { i = (i + 1) & 1023; } while (i != 88);
+
+ the jump sees 601, 602 step 1, 2^64 - 514 trips left
+ real 601 .. 1023, 0 .. 88 512 trips
+
+The next runs keep this answer. So it is also a guess if the run made a guess
+before it.
+
+The third guess is the target of ``may_goto``. All states have the same one::
+
+ large = x > 7;
+ while (...) {
+ if (large && x <= 7) lock(); only a join gets here
+ do { p = lookup(); } while (!p); gets may_goto
+ *p
+ }
+
+With no lock the target is ``return 0``. The joined state gets to ``lock()``, so
+the pass selects the code after the loop. A path with no lock goes there too,
+and ``*p`` fails.
+
+The run accepts nothing: it drops states and skips trips. It is not the verdict
+either: a path has ranges too, and it can fail at an insn that a pass proves.
+
+All passes
+----------
+
+``do_check_all()`` puts the parts together::
+
+ pass 0: join all numbers
+ |
+ +---------> run a pass
+ | |
+ | error ? --- no ---> pessimized ? --- no ---> ACCEPT
+ | | yes | yes
+ | v v
+ | good pass before ? save the good pass,
+ | | no | yes partition, cut the budget
+ | | v |
+ | | ACCEPT, use that pass +---> run a pass
+ | v
+ | partition
+ | |
+ | failing path ? --- yes ---> log it; if no guess, cut the budget
+ | | no |
+ | +<-------------------------+
+ | v
+ +- yes - is more kept apart ? --- no ---> REJECT
+
+The last answer is no in three cases:
+
+- Nothing made the pass imprecise before its first error. The error is real.
+- No partition adds something.
+- The budget is used up.
+
+A pass that is over the limit of the load ends it too. The next chapter explains
+"pessimized".
+
+The example, and the same program with ``i > 8``::
+
+ i > 7 i > 8
+ pass 0 insn 9 is unproven insn 9 is unproven
+ partition ADD_DEPS ADD_DEPS
+ failing path none 0 1 2 3 5 6 7 9 10: buf[8]
+ pass 1 no error insn 10 is unproven
+ result ACCEPT REJECT, the log has the failing path
+
+What a pass leaves behind
+=========================
+
+Fixups
+------
+
+After the last pass the verifier rewrites the program. A load from the context
+gets the real offset. A load through an untrusted pointer gets a fault handler.
+``bpf_obj_drop()`` gets the fields of the type. A pass notes what the rewrite
+needs: these notes are the *fixups*.
+
+An insn has one fixup for all states. If two states need two fixups, the
+verifier takes one that fits both: no inlining, a fault handler. If there is
+none, the insn is unproven. If both states are real, no pass can prove it, and
+the program is rejected::
+
+ if (...) p = a; else p = b; a has a kptr, b has none
+ bpf_obj_drop(p); unproven: one call, two types
+
+A pass can write a fixup on a path that only a join makes. The next pass must
+not see it. A *snapshot* is a copy of the fixups and of the bounds that a pass
+writes to the program. Each pass starts from the snapshot that was taken before
+pass 0.
+
+What a pass learns about loops is not in a snapshot. The next pass gets it as a
+guess.
+
+``check_after_pass()`` checks what holds for the program as a whole, such as the
+depth of the stack over all call chains. Its error is handled like an unproven
+insn.
+
+Pessimized insns
+----------------
+
+A good pass can still ask for too much::
+
+ (off, len) = (0, 64) or (56, 8) each path needs 64 bytes of the context
+ join: off = 0..56, len = 8..64 the pass asks for 120 bytes
+
+Such a bound is checked when the program is attached or run. The insn that
+raised it is *pessimized* if the pass is imprecise and an operand has a range.
+An insn that gets a barrier (next chapter) in an imprecise pass is pessimized
+too. The verifier saves the good pass in a second snapshot, and partitions more
+in a small budget. If a later pass has an error, the saved pass is the result.
+
+Dead code
+---------
+
+An insn that no state got to is dead. A jump with a dead edge becomes a goto,
+and the dead insns are removed. Without ``CAP_BPF`` they are replaced by
+``goto -1``. If both edges of a jump are dead, no run gets to the jump. It
+becomes ``goto -1``.
+
+Only a conditional jump can lose an edge. So an infeasible state ends only
+after such a jump.
+
+Offload
+-------
+
+The driver of an offloaded program sees each insn with its state, and cannot
+forget. So it sees only a pass without errors: that pass runs once more for the
+driver, after ``prepare()``. An insn that the driver refuses is unproven. If a
+later pass has no error, the driver gets ``destroy()``, ``prepare()`` and that
+pass. What it noted in a map stays. An offloaded program gets no ``may_goto``
+and no rewrite of fastcall patterns.
+
+Speculation and privileges
+==========================
+
+The cpu can run code after a jump before it knows the result of the jump. What
+such code loads can leak through the cache::
+
+ if (i > 7) goto out; the cpu predicts "not taken" for i = 1000
+ x = buf[i]; reads out of bounds
+
+Without ``CAP_PERFMON`` the verifier prevents that, unless the mitigations are
+off or the cpu does not need them:
+
+========================== ====================================================
+``pointer += number`` The number gets a mask that cuts it to the object.
+ If no mask fits all states, the insn gets a
+ *barrier*: ``nospec`` stops speculation.
+a jump that values decide The dead edge is checked too, as a *speculative*
+ state.
+an error in such a state It is no error. The insn gets a barrier.
+a store to the stack A barrier follows if a later load could see an old
+ value that matters.
+========================== ====================================================
+
+A barrier has to stop each edge that leads to the insn::
+
+ fall-through, jump, call, return barrier before the insn
+ exit of a callback --> helper barrier at the exit
+ may_goto --> r0 = 0; exit barrier before r0 = 0
+ call --> recursion in frames barrier at the call
+ return --> dead code goto -1 replaces the code
+
+A speculative state is joined only with speculative states, and it subsumes
+only speculative states.
+
+Some mechanisms depend on a capability. The last line is there because the cpu
+can go past the jump that a proof needs:
+
+================================= =============================================
+``CAP_BPF`` loops, calls of functions, equations, numbers
+ in stack slots, 1M insns (else 4096), the
+ full `Limits`_
+``CAP_PERFMON`` trip count, reads of stack that is not
+ written, pointers as numbers, access to BTF
+ objects, arena
+``CAP_PERFMON`` or no mitigations equations of pointers, equations that a join
+ makes from constants, the end of infeasible
+ states, skipped first trips, stack access
+ with many offsets, pointers out of the object
+================================= =============================================
+
+Without a trip count a loop gets ``may_goto``, or it is peeled.
+
+Limits
+======
+
+There is no limit on the number of states or of checked insns. There are two
+limits for the full load:
+
+====== ========================================================================
+cost ``BPF_MAX_COST``. Each kind of work has a weight, see ``BPF_COST_SCAN``.
+memory ``BPF_MAX_MEM``. It counts the states and the large tables. The passes
+ of a small program get less, see ``BPF_PASS_MEM_BASE``.
+====== ========================================================================
+
+Both limits are smaller without ``CAP_BPF``. If a load is over a limit, the log
+shows both limits.
+
+Cost counts work, not time. So the verdict does not depend on the speed or the
+load of the machine.
+
+A program that no precision can prove must not use up the limit on cost. So the
+verifier cuts what is left after the first error. It cuts more after a failing
+path with no guess, and after a good pass with a pessimized insn.
+
+Other limits:
+
+======================= =======================================================
+``MAX_CALL_FRAMES`` 32 frames in a state
+``BPF_ID_MAP_SIZE`` less than 9152 ids in a state
+``LOOP_STACK_SIZE`` 16 nested loops in a function
+``MAX_BPF_STACK`` 512 bytes of stack; 2048 if the JIT supports it
+``BPF_MAX_VAR_OFF`` a pointer that can be 2^29 away gives no access
+``BPF_MAX_TRIPS`` a loop gets ``may_goto`` if k can be above 2^32
+======================= =======================================================
+
+What is trusted
+===============
+
+A bug in trusted code can accept an unsafe program. A bug in a guess cannot.
+
+The table has the guesses and what checks them. All other code is trusted.
+
+======================== ======================================================
+guess check
+======================== ======================================================
+what is kept apart none is needed: a join and two states are both proofs
+sym of a difference none is needed: a state without it has more values
+equations of a join ``check_equations()`` in both states
+threshold the state that comes back has to be subsumed
+first trips they are joined with the entry; then the same check
+what stops a loop ``pass_needs_restart()``
+trip count the trip index of each state that closes a cycle
+path-sensitive run none is needed: it accepts nothing
+======================== ======================================================
+
+The log
+=======
-Program that uses invalid map_fd=0 while calling to map_lookup_elem() function::
+===== =========================================================================
+level content
+===== =========================================================================
+1 the failing path; if there is none, or if it has a guess and a pass has a
+ real error: the basic block of an error
+2 the analyses of the code, all passes and path-sensitive runs
+4 time, stack depth and insns of each function
+===== =========================================================================
- BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_LD_MAP_FD(BPF_REG_1, 0),
- BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
- BPF_EXIT_INSN(),
-
-Error::
+In level 1 the later passes and the path-sensitive runs have no log. The one
+that gives the log runs once more. That work is not in the limit on cost. It
+costs at most as much as the work before it.
- 0: (7a) *(u64 *)(r10 -8) = 0
- 1: (bf) r2 = r10
- 2: (07) r2 += -8
- 3: (b7) r1 = 0x0
- 4: (85) call 1
- fd 0 is not pointing to valid bpf_map
-
-Program that doesn't check return value of map_lookup_elem() before accessing
-map element::
-
- BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_LD_MAP_FD(BPF_REG_1, 0),
- BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
- BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
- BPF_EXIT_INSN(),
-
-Error::
-
- 0: (7a) *(u64 *)(r10 -8) = 0
- 1: (bf) r2 = r10
- 2: (07) r2 += -8
- 3: (b7) r1 = 0x0
- 4: (85) call 1
- 5: (7a) *(u64 *)(r0 +0) = 0
- R0 invalid mem access 'map_value_or_null'
-
-Program that correctly checks map_lookup_elem() returned value for NULL, but
-accesses the memory with incorrect alignment::
-
- BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_LD_MAP_FD(BPF_REG_1, 0),
- BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
- BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1),
- BPF_ST_MEM(BPF_DW, BPF_REG_0, 4, 0),
- BPF_EXIT_INSN(),
-
-Error::
-
- 0: (7a) *(u64 *)(r10 -8) = 0
- 1: (bf) r2 = r10
- 2: (07) r2 += -8
- 3: (b7) r1 = 1
- 4: (85) call 1
- 5: (15) if r0 == 0x0 goto pc+1
- R0=map_ptr R10=fp
- 6: (7a) *(u64 *)(r0 +4) = 0
- misaligned access off 4 size 8
-
-Program that correctly checks map_lookup_elem() returned value for NULL and
-accesses memory with correct alignment in one side of 'if' branch, but fails
-to do so in the other side of 'if' branch::
-
- BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_LD_MAP_FD(BPF_REG_1, 0),
- BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
- BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
- BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
- BPF_EXIT_INSN(),
- BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 1),
- BPF_EXIT_INSN(),
-
-Error::
-
- 0: (7a) *(u64 *)(r10 -8) = 0
- 1: (bf) r2 = r10
- 2: (07) r2 += -8
- 3: (b7) r1 = 1
- 4: (85) call 1
- 5: (15) if r0 == 0x0 goto pc+2
- R0=map_ptr R10=fp
- 6: (7a) *(u64 *)(r0 +0) = 0
- 7: (95) exit
-
- from 5 to 8: R0=imm0 R10=fp
- 8: (7a) *(u64 *)(r0 +0) = 1
- R0 invalid mem access 'imm'
-
-Program that performs a socket lookup then sets the pointer to NULL without
-checking it::
-
- BPF_MOV64_IMM(BPF_REG_2, 0),
- BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -8),
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_MOV64_IMM(BPF_REG_3, 4),
- BPF_MOV64_IMM(BPF_REG_4, 0),
- BPF_MOV64_IMM(BPF_REG_5, 0),
- BPF_EMIT_CALL(BPF_FUNC_sk_lookup_tcp),
- BPF_MOV64_IMM(BPF_REG_0, 0),
- BPF_EXIT_INSN(),
-
-Error::
-
- 0: (b7) r2 = 0
- 1: (63) *(u32 *)(r10 -8) = r2
- 2: (bf) r2 = r10
- 3: (07) r2 += -8
- 4: (b7) r3 = 4
- 5: (b7) r4 = 0
- 6: (b7) r5 = 0
- 7: (85) call bpf_sk_lookup_tcp#65
- 8: (b7) r0 = 0
- 9: (95) exit
- Unreleased reference id=1, alloc_insn=7
-
-Program that performs a socket lookup but does not NULL-check the returned
-value::
-
- BPF_MOV64_IMM(BPF_REG_2, 0),
- BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -8),
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
- BPF_MOV64_IMM(BPF_REG_3, 4),
- BPF_MOV64_IMM(BPF_REG_4, 0),
- BPF_MOV64_IMM(BPF_REG_5, 0),
- BPF_EMIT_CALL(BPF_FUNC_sk_lookup_tcp),
- BPF_EXIT_INSN(),
-
-Error::
-
- 0: (b7) r2 = 0
- 1: (63) *(u32 *)(r10 -8) = r2
- 2: (bf) r2 = r10
- 3: (07) r2 += -8
- 4: (b7) r3 = 4
- 5: (b7) r4 = 0
- 6: (b7) r5 = 0
- 7: (85) call bpf_sk_lookup_tcp#65
- 8: (95) exit
- Unreleased reference id=1, alloc_insn=7
+A line has the insn and the registers that it reads or changes::
+
+ 9: (0f) r1 += r6 ; R1=map_value(map=.data.buf,ks=4,vs=8,smin=0,smax=8,...)
+ 10: (71) r0 = *(u8 *)(r1 +0)
+ invalid access to map value, value_size=8 off=8 size=1
+ R1 max value is outside of the allowed memory range
+
+ Verification failed: Memory Safety: Access outside bounds
+ ...
+
+Most errors have a report: the reason, the source line, the insns that made the
+value, and a suggestion. An error that does not end the path gets no report:
+
+- an error in a speculative state,
+- an argument that does not fit the BTF of a static function.
+
+In level 2 each pass and each path-sensitive run has its report.
+
+Other lines
+-----------
+
+Most of these lines are only in level 2.
+
+===================================== =========================================
+text meaning
+===================================== =========================================
+``pass 1`` a new pass starts
+``path-sensitive run after pass 0`` see `The failing path`_
+``from 4 to 5: R6=...`` the state at the start of a basic block
+``from 5 to 6: pruned`` another state subsumes it
+``from 3 to 7: joined`` it is joined with another state
+``from 8 to 2: widened`` the same, at the header of a leftover
+``(speculative execution)`` a speculative state
+``loop_stack=3`` in the loop with header 3
+``iter(3)=1`` in peeled trip 1 of that loop
+``0:r7=8*sym0+0 sym0=[0,+7]`` frame 0: r7 = 8 * sym0; sym0 is 0..0+7
+``0:trip0=1*sym0+0`` the trip index of the first loop
+``sym2=1*sym0-1*sym1+4`` a sum: sym2 = sym0 - sym1 + 4
+``sym3=insn5(1*sym2+0,248)`` an op: what insn 5 makes of sym2 and 248
+``loop at 3 has trip count 8`` no ``may_goto`` is needed
+``loop at 0 has no trip count`` the loop gets ``may_goto``
+``call at 1 waits for func#1`` the function is checked on its own
+``fork R1`` the load is checked once for each slot
+===================================== =========================================
+
+Common errors
+-------------
+
+``unreachable insn 2``
+ No path gets to the insn.
+
+``R2 !read_ok``
+ The register was not written, a call clobbered it, or it has two types on two
+ paths.
+
+``R1 invalid mem access 'scalar'``
+ A number is used as a pointer.
+
+``R0 invalid mem access 'map_value_or_null'``
+ Test the pointer for NULL first.
+
+``invalid access to map value, value_size=8 off=8 size=1``
+ The largest offset is out of the value. Test the index.
+
+``math between map_value pointer and register with unbounded min value is not allowed``
+ The number that is added has no lower bound.
+
+``invalid access to packet, off=12 size=2, R2(id=0,off=12,r=0)``
+ Compare the pointer with ``data_end`` first.
+
+``invalid write to stack R10 off=8 size=8``
+ The stack is below r10.
+
+``misaligned stack access off 0+-12 size 8``
+ An access to the stack has to be aligned to its size.
+
+``invalid read from stack R10 off=-8 size=8``, ``invalid read from stack off -8+0 size 8``
+ The bytes were not written.
+
+``invalid bpf_context access off=1000 size=8``
+ The program type does not allow this field.
+
+``Unreleased reference id=5 alloc_insn=4``
+ A path does not release what insn 4 acquired.
+
+``loop at insn 2 never exits``
+ The loop has no edge that leaves it.
+
+``BPF program is too complex``, ``the limit on cost or memory ended pass 6``
+ The load is over a limit. If a pass had an error before, the log has that
+ error, or a path with a guess. That path can be false.
+
+Words
+=====
+
+================== ============================================================
+barrier the ``nospec`` insn, it stops speculative execution
+basic block a run of insns without a jump into it or out of it
+budget a limit on the cost of one step
+checked the bytes of a packet that are known to be before its end
+control_deps xregs whose value flows to a jump that can skip an unproven
+ insn
+correlated two branches: one tests a result that the other one stored
+cost the work that is done
+data_deps xregs whose value flows into an operand of an unproven insn
+deps data_deps and control_deps
+entry a state that enters a loop
+equation how two or three numbers are related
+failing path a path without joins on which the check of an unproven insn
+ fails; it is the result of a path-sensitive run
+fixpoint the state at a header subsumes all states that come back
+fixups what a pass notes for the rewrite of the program
+fork check an insn once for each part of the values of an operand
+gap a value of a join that is in neither state
+good pass a pass without errors
+guess a choice that can be wrong and cannot accept an unsafe
+ program
+header the insn where each trip of a loop starts
+held references, locks, dynptrs, iterators, irq flags
+helped after a partition the first unproven insn of the last pass
+ has no error, and no old error came back
+id the name of a set of xregs that hold copies of one pointer
+imprecise a state of the pass possibly has values that no path has
+in frames a function is checked with the frames of its callers
+infeasible no run of the program has the state
+insn an instruction
+instance an entry state of a function, its exit states and its calls
+iv induction variable: each trip adds a constant to it
+join two states become one that has the values of both
+join point an insn where states can meet
+join_key what has no join: states with two join_keys never meet
+leftover all trips that are not peeled
+live read later, before it is written
+may_goto a jump that is taken when the program has looped too long
+on its own a function is checked without the frames of its callers
+partition keep more states apart in the next pass
+pass one run of the main analysis over all functions
+path with a guess the failing path of a run that made a guess; it can be false
+path-sensitive run a search for a failing path, one path after another
+peel check a trip apart from the other trips
+pessimized proven, but a join possibly raised a bound
+precision what a pass keeps apart
+proven an insn whose check did not fail in a state of the pass
+prune drop a state that another state subsumes
+refine remove values that a jump excludes
+seed an insn whose reads start the backward slice
+snapshot a copy of the fixups and bounds that a pass writes
+speculative the cpu can have the state only after a wrong prediction
+spill a store of a register to a stack slot
+state what each xreg can hold before an insn
+step all values of a number are ``base + step * k``; in a loop:
+ what one trip adds
+stored condition an xreg that stores the result of a test
+subsumes has all values of another state
+suspect a loop that made an xreg of data_deps imprecise
+sym a number that equations use
+threshold a value at which a range that grows can stop
+trip one iteration of a loop
+trip count the largest trip index + 1
+trip index how many times the program came back to the header
+unproven an insn or a part whose check failed in a state of the pass
+unreadable an xreg without a value: an insn that reads it is unproven
+unwind leave a function with ``unwind_exit``
+walk one check of the body of a loop
+widen move a bound that grows to a threshold
+xreg a register, a stack slot or the trip index of a loop
+================== ============================================================
--
2.55.0
^ permalink raw reply related [flat|nested] 79+ messages in thread