* [PATCH bpf v3 1/2] bpf: Verify global subprogs in each sleepability context
2026-09-14 13:19 [PATCH bpf v3 0/2] Fix global subprog verification context Kumar Kartikeya Dwivedi
@ 2026-09-14 13:19 ` Kumar Kartikeya Dwivedi
2026-09-15 7:50 ` Eduard Zingerman
2026-09-14 13:19 ` [PATCH bpf v3 2/2] selftests/bpf: Test global subprog callback contexts Kumar Kartikeya Dwivedi
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Kumar Kartikeya Dwivedi @ 2026-09-14 13:19 UTC (permalink / raw)
To: bpf
Cc: Nicholas Carlini, Alexei Starovoitov, Andrii Nakryiko,
Daniel Borkmann, Eduard Zingerman, Emil Tsalapatis, kkd,
kernel-team
Global subprograms are verified independently with a fresh verifier root.
do_check_common() currently seeds that root's in_sleepable state from the
program, even though a global subprogram can also run from callbacks whose
execution context differs from the program's main entry point.
In particular, workqueue and task-work callbacks are sleepable even when
the containing program is not. A global subprogram of that program is
therefore verified as non-sleepable, making in_rcu_cs() true and allowing
loads of RCU-protected kptrs to produce trusted MEM_RCU pointers. The same
subprogram can then be called from a sleepable callback without a classic
RCU reader. It can retain such a pointer while the object is freed and use
it after free.
The verifier's execution-context predicates are complementary. A state is
sleepable only when in_sleepable is set and no RCU, preemption, IRQ, or lock
region is active. Each condition which prevents sleeping also provides RCU
protection, while in_rcu_cs() treats a non-sleepable state as implicitly
protected.
Use this relationship to represent a global subprogram caller with only the
result of in_sleepable_context(). A protected sleepable caller is normalized
to in_sleepable=false at the independent verification root. This both
prevents sleepable operations and makes in_rcu_cs() true without copying
caller-owned lock state.
Track only the contexts in which each global subprogram is actually
reached. Verify it once if all reachable calls use the same context, and
twice only if both sleepable and non-sleepable calls reach it. Calls found
while verifying globals or asynchronous callbacks mark further contexts
for checking. Repeat the existing subprogram walk until all called
contexts have been verified; unreachable global calls remain unchecked.
Accumulate instruction counts over those verification passes. Preserve
the total recorded before each pass, since path accounting has already
added this pass's synchronous instructions and its root total must also
include asynchronous subprograms.
This makes an unprotected callback verify the global subprogram as
sleepable, turning its RCU-protected kptr load into an untrusted pointer.
Protected callers and global subprograms which do not depend on implicit RCU
protection remain valid.
Fixes: 81f1d7a583fa ("bpf: wq: add bpf_wq_set_callback_impl")
Fixes: 38aa7003e369 ("bpf: task work scheduling kfuncs")
Reported-by: Nicholas Carlini <npc@anthropic.com>
Suggested-by: Nicholas Carlini <npc@anthropic.com>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
---
include/linux/bpf.h | 5 +--
kernel/bpf/verifier.c | 71 ++++++++++++++++++++++---------------------
2 files changed, 40 insertions(+), 36 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 2a5fa346aada..7217e8ddf64b 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1682,8 +1682,9 @@ static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags
struct bpf_func_info_aux {
u16 linkage;
bool unreliable;
- bool called : 1;
- bool verified : 1;
+ /* Indexed by in_sleepable. */
+ bool called[2];
+ bool verified[2];
};
enum bpf_jit_poke_reason {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6c6b8d8520cd..bb9a02ad4f11 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -10559,6 +10559,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
if (err == -EFAULT)
return err;
if (bpf_subprog_is_global(env, subprog)) {
+ struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog);
const char *sub_name = bpf_subprog_name(env, subprog);
const char *operation;
bool returns_void;
@@ -10590,11 +10591,10 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
if (env->log.level & BPF_LOG_LEVEL)
verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
subprog, sub_name);
+ sub_aux->called[in_sleepable_context(env)] = true;
returns_void = subprog_returns_void(env, subprog);
if (env->subprog_info[subprog].changes_pkt_data)
clear_all_pkt_pointers(env);
- /* mark global subprog for verifying after main prog */
- subprog_aux(env, subprog)->called = true;
if (returns_void)
bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
else
@@ -11434,11 +11434,7 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,
/* Check if we're in a sleepable context. */
static inline bool in_sleepable_context(struct bpf_verifier_env *env)
{
- return !env->cur_state->active_rcu_locks &&
- !env->cur_state->active_preempt_locks &&
- !env->cur_state->active_locks &&
- !env->cur_state->active_irq_id &&
- in_sleepable(env);
+ return !in_rcu_cs(env);
}
static const char *non_sleepable_context_description(struct bpf_verifier_env *env)
@@ -19780,13 +19776,14 @@ static void free_states(struct bpf_verifier_env *env)
}
}
-static int do_check_common(struct bpf_verifier_env *env, int subprog)
+static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable)
{
bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
struct bpf_subprog_info *sub = subprog_info(env, subprog);
struct bpf_prog_aux *aux = env->prog->aux;
struct bpf_verifier_state *state;
struct bpf_reg_state *regs;
+ u32 old_insns_total = sub->insns_total;
u32 insn_processed = env->insn_processed;
int ret, i;
@@ -19799,7 +19796,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
state->curframe = 0;
state->speculative = false;
state->branches = 1;
- state->in_sleepable = env->prog->sleepable;
+ state->in_sleepable = is_sleepable;
state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT);
if (!state->frame[0]) {
kfree(state);
@@ -19943,8 +19940,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
* not accounted as callees by account_current_path().
* Accumulate their total counts as total counts of the main or
* global subprog hosting the async call.
+ * Start from the saved total of earlier contexts: adding to the current
+ * total would count this pass's synchronous paths twice.
*/
- env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed;
+ sub->insns_total = old_insns_total + (env->insn_processed - insn_processed);
return ret;
}
@@ -19972,14 +19971,19 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
{
struct bpf_prog_aux *aux = env->prog->aux;
struct bpf_func_info_aux *sub_aux;
- int i, ret, new_cnt;
+ int context, i, ret, new_cnt;
if (!aux->func_info)
return 0;
- /* exception callback is presumed to be always called */
- if (env->exception_callback_subprog)
- subprog_aux(env, env->exception_callback_subprog)->called = true;
+ /*
+ * Callbacks cannot throw, so the exception callback always runs in the
+ * main program's context. It is presumed to be always called.
+ */
+ if (env->exception_callback_subprog) {
+ sub_aux = subprog_aux(env, env->exception_callback_subprog);
+ sub_aux->called[env->prog->sleepable] = true;
+ }
again:
new_cnt = 0;
@@ -19988,29 +19992,28 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
continue;
sub_aux = subprog_aux(env, i);
- if (!sub_aux->called || sub_aux->verified)
- continue;
+ for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) {
+ if (!sub_aux->called[context] || sub_aux->verified[context])
+ continue;
- env->insn_idx = env->subprog_info[i].start;
- WARN_ON_ONCE(env->insn_idx == 0);
- ret = do_check_common(env, i);
- if (ret) {
- return ret;
- } else if (env->log.level & BPF_LOG_LEVEL) {
- verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",
- i, bpf_subprog_name(env, i));
- }
+ env->insn_idx = env->subprog_info[i].start;
+ WARN_ON_ONCE(env->insn_idx == 0);
+ ret = do_check_common(env, i, context);
+ if (ret)
+ return ret;
+ if (env->log.level & BPF_LOG_LEVEL)
+ verbose(env, "Func#%d ('%s') is safe for any args "
+ "that match its prototype\n",
+ i, bpf_subprog_name(env, i));
- /* We verified new global subprog, it might have called some
- * more global subprogs that we haven't verified yet, so we
- * need to do another pass over subprogs to verify those.
- */
- sub_aux->verified = true;
- new_cnt++;
+ sub_aux->verified[context] = true;
+ new_cnt++;
+ }
}
- /* We can't loop forever as we verify at least one global subprog on
- * each pass.
+ /*
+ * We can't loop forever as each pass verifies at least one new context,
+ * and there are only two contexts per global subprog.
*/
if (new_cnt)
goto again;
@@ -20023,7 +20026,7 @@ static int do_check_main(struct bpf_verifier_env *env)
int ret;
env->insn_idx = 0;
- ret = do_check_common(env, 0);
+ ret = do_check_common(env, 0, env->prog->sleepable);
if (!ret)
env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
return ret;
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH bpf v3 2/2] selftests/bpf: Test global subprog callback contexts
2026-09-14 13:19 [PATCH bpf v3 0/2] Fix global subprog verification context Kumar Kartikeya Dwivedi
2026-09-14 13:19 ` [PATCH bpf v3 1/2] bpf: Verify global subprogs in each sleepability context Kumar Kartikeya Dwivedi
@ 2026-09-14 13:19 ` Kumar Kartikeya Dwivedi
2026-09-14 13:27 ` [PATCH bpf v3 0/2] Fix global subprog verification context Kumar Kartikeya Dwivedi
2026-09-16 17:00 ` patchwork-bot+netdevbpf
3 siblings, 0 replies; 7+ messages in thread
From: Kumar Kartikeya Dwivedi @ 2026-09-14 13:19 UTC (permalink / raw)
To: bpf
Cc: Eduard Zingerman, Alexei Starovoitov, Andrii Nakryiko,
Daniel Borkmann, Emil Tsalapatis, Nicholas Carlini, kkd,
kernel-team
Exercise global subprogram verification from workqueue callbacks, which
can run in a sleepable context even when the containing program is not
sleepable. An unprotected callback must not let the global subprogram use
implicit RCU protection inherited from the program.
Add a negative case which loads an RCU-protected task kptr in a global
subprogram reached from a workqueue callback. It fails on an unfixed
kernel because the program is incorrectly accepted. Also cover a
workqueue callback protected by an explicit RCU read-side critical
section, where the same global subprogram remains valid.
Call the same harmless global subprogram directly from the main program
and from an unprotected callback. Mark it __weak __noinline so both calls
survive optimization, and check that its instruction statistics account
for both verification contexts. This also verifies that global calls
from callbacks are not rejected wholesale. Workqueue callbacks return
zero explicitly after the global call, as required by their contract.
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
---
.../bpf/progs/verifier_async_cb_context.c | 106 ++++++++++++++++++
1 file changed, 106 insertions(+)
diff --git a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
index e0926767bbd3..1b653bfb63eb 100644
--- a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
+++ b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
@@ -9,6 +9,11 @@
char _license[] SEC("license") = "GPL";
+struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
+void bpf_task_release(struct task_struct *p) __ksym;
+void bpf_rcu_read_lock(void) __ksym;
+void bpf_rcu_read_unlock(void) __ksym;
+
/* Timer tests */
struct timer_elem {
@@ -164,6 +169,7 @@ int syscall_btf_find_prog(void *ctx)
struct wq_elem {
struct bpf_wq w;
+ struct task_struct __kptr *task;
};
struct {
@@ -217,6 +223,106 @@ int wq_sleepable_prog(void *ctx)
return 0;
}
+__noinline int wq_global_acquire(void)
+{
+ struct task_struct *task, *acquired;
+ struct wq_elem *val;
+ int key = 0;
+
+ val = bpf_map_lookup_elem(&wq_map, &key);
+ if (!val)
+ return 0;
+
+ task = val->task;
+ if (!task)
+ return 0;
+
+ acquired = bpf_task_acquire(task);
+ if (acquired)
+ bpf_task_release(acquired);
+ return 0;
+}
+
+static int wq_global_rcu_cb(void *map, int *key, void *value)
+{
+ wq_global_acquire();
+ return 0;
+}
+
+SEC("fentry/bpf_fentry_test1")
+__failure __msg("R1 must be a rcu pointer")
+int wq_global_rcu_prog(void *ctx)
+{
+ struct wq_elem *val;
+ int key = 0;
+
+ val = bpf_map_lookup_elem(&wq_map, &key);
+ if (!val)
+ return 0;
+
+ bpf_wq_init(&val->w, &wq_map, 0);
+ bpf_wq_set_callback(&val->w, wq_global_rcu_cb, 0);
+ return 0;
+}
+
+static int wq_global_rcu_lock_cb(void *map, int *key, void *value)
+{
+ bpf_rcu_read_lock();
+ wq_global_acquire();
+ bpf_rcu_read_unlock();
+ return 0;
+}
+
+SEC("fentry/bpf_fentry_test1")
+__success
+int wq_global_rcu_lock_prog(void *ctx)
+{
+ struct wq_elem *val;
+ int key = 0;
+
+ /* Verify the same global subprog in non-sleepable and protected contexts. */
+ wq_global_acquire();
+
+ val = bpf_map_lookup_elem(&wq_map, &key);
+ if (!val)
+ return 0;
+
+ bpf_wq_init(&val->w, &wq_map, 0);
+ bpf_wq_set_callback(&val->w, wq_global_rcu_lock_cb, 0);
+ return 0;
+}
+
+__weak __noinline int wq_global_no_rcu(void)
+{
+ return 0;
+}
+
+static int wq_global_no_rcu_cb(void *map, int *key, void *value)
+{
+ wq_global_no_rcu();
+ return 0;
+}
+
+SEC("fentry/bpf_fentry_test1")
+__success __log_level(4)
+__msg("subprog {{[0-9]+}} (wq_global_no_rcu) global insns_self 4 insns_total 4 stack 0")
+int wq_global_no_rcu_prog(void *ctx)
+{
+ struct wq_elem *val;
+ int key = 0;
+
+ /* Verify the same global in non-sleepable and unprotected contexts. */
+ wq_global_no_rcu();
+
+ val = bpf_map_lookup_elem(&wq_map, &key);
+ if (!val)
+ return 0;
+
+ bpf_wq_init(&val->w, &wq_map, 0);
+ bpf_wq_set_callback(&val->w, wq_global_no_rcu_cb, 0);
+ return 0;
+}
+
/* Task work tests */
struct task_work_elem {
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread