* [PATCH bpf-next v3 1/4] bpf: Avoid per-CPU private stack in preemptible tracing progs
2026-11-17 11:29 [PATCH bpf-next v3 0/4] bpf: Check recursion per task for preemptible programs Mike Bazov via B4 Relay
@ 2026-11-17 11:29 ` Mike Bazov via B4 Relay
2026-11-17 11:44 ` sashiko-bot
2026-11-17 11:29 ` [PATCH bpf-next v3 2/4] bpf: Mark programs whose runs must not overlap on a CPU Mike Bazov via B4 Relay
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Mike Bazov via B4 Relay @ 2026-11-17 11:29 UTC (permalink / raw)
To: bpf, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: Martin KaFai Lau, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Song Liu, Yonghong Song, Jiri Olsa, John Fastabend,
Emil Tsalapatis, Ihor Solodrai, Steven Rostedt, Masami Hiramatsu,
Mathieu Desnoyers, JP Kobryn, linux-trace-kernel, linux-kernel,
Mike Bazov
From: Mike Bazov <mikeba@sentinelone.com>
A per-CPU private stack is only safe if a program never runs in two
tasks at once on one CPU. Commit a76ab5731e32 ("bpf: Find eligible
subprogs for private stack support") gave one to tracing, raw_tp and
tracepoint programs, but they run preemptibly, and two tasks can
already be inside one of them on a CPU:
- Two raw_tp dispatchers that tail-call the same handler each pass
their own per-CPU recursion check, so both tasks run the handler.
- A tracepoint program attached to a syscall tracepoint and to a
regular tracepoint can be entered by a second task through the
regular one, which only checks bpf_prog_active.
The second task then overwrites the preempted task's stack. Neither
case can be seen when the program is loaded, so never give these
program types a private stack. The next patches make such overlap the
normal case.
check_recursion() told tracing programs apart by whether they can get a
private stack. Make it check the program type instead, so that
recursive subprograms stay rejected in these program types.
Move the selftests that relied on these program types having a private
stack to kprobe.multi and perf_event programs.
Trade-offs:
- These programs use the kernel stack again.
- A program with two 2000-byte frames no longer loads: "combined stack
size of 2 calls is 4000. Too large".
- Unwinding on a private stack is no longer tested at run time: raw_tp
was the only such program type that BPF_PROG_TEST_RUN runs.
Signed-off-by: Mike Bazov <mikeba@sentinelone.com>
---
kernel/bpf/verifier.c | 42 +++++++----
.../selftests/bpf/prog_tests/may_goto_priv_stack.c | 6 +-
.../selftests/bpf/prog_tests/timed_may_goto.c | 6 +-
.../selftests/bpf/progs/may_goto_priv_stack.c | 14 +++-
.../selftests/bpf/progs/test_global_func3.c | 9 +++
.../selftests/bpf/progs/timed_may_goto_user.c | 16 ++++-
.../selftests/bpf/progs/verifier_private_stack.c | 81 ++++++++++++----------
.../testing/selftests/bpf/progs/verifier_unwind.c | 33 +++++++--
8 files changed, 146 insertions(+), 61 deletions(-)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1a8e5a108a68..2ca1469d8dee 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5331,9 +5331,15 @@ static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
if (!bpf_jit_supports_private_stack())
return NO_PRIV_STACK;
- /* bpf_prog_check_recur() checks all prog types that use bpf trampoline
- * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked
- * explicitly.
+ /*
+ * A private stack is per CPU, so a program using it must never run in two
+ * tasks at once on one CPU. Tracing, raw_tp and tracepoint programs run
+ * preemptibly, and the same program can be entered by another task on the
+ * same CPU, e.g. through a second tail-call dispatcher, or as a tracepoint
+ * program attached to both a syscall and a regular tracepoint. They always
+ * use the task's kernel stack. Kprobe multi, kprobe session and perf_event
+ * programs are serialized per CPU by bpf_prog_active, and struct_ops
+ * programs that request a private stack keep per-CPU recursion protection.
*/
switch (prog->type) {
case BPF_PROG_TYPE_KPROBE:
@@ -5346,14 +5352,10 @@ static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
prog->expected_attach_type != BPF_TRACE_KPROBE_SESSION)
return NO_PRIV_STACK;
fallthrough;
- case BPF_PROG_TYPE_TRACEPOINT:
case BPF_PROG_TYPE_PERF_EVENT:
- case BPF_PROG_TYPE_RAW_TRACEPOINT:
return PRIV_STACK_ADAPTIVE;
- case BPF_PROG_TYPE_TRACING:
- case BPF_PROG_TYPE_LSM:
case BPF_PROG_TYPE_STRUCT_OPS:
- if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog))
+ if (prog->aux->priv_stack_requested)
return PRIV_STACK_ADAPTIVE;
fallthrough;
default:
@@ -5363,6 +5365,22 @@ static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
return NO_PRIV_STACK;
}
+/* Progs that run on top of other progs, whose frames are still on the stack */
+static bool bpf_prog_runs_nested(struct bpf_prog *prog)
+{
+ switch (prog->type) {
+ case BPF_PROG_TYPE_KPROBE:
+ case BPF_PROG_TYPE_PERF_EVENT:
+ case BPF_PROG_TYPE_TRACEPOINT:
+ case BPF_PROG_TYPE_RAW_TRACEPOINT:
+ return true;
+ case BPF_PROG_TYPE_TRACING:
+ return bpf_prog_check_recur(prog);
+ default:
+ return false;
+ }
+}
+
static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
{
if (env->prog->jit_requested)
@@ -5494,12 +5512,10 @@ static int check_recursion(struct bpf_verifier_env *env, struct bpf_call_graph *
}
if (env->subprog_info[s].recursive) {
/*
- * These progs run on top of other progs. Their frames belong in the
- * private stack, which has room for one frame of a subprog. A kprobe prog
- * that can be attached to a uprobe has no private stack, and is no better.
+ * These progs run on top of other progs. A private stack has room for one
+ * frame of a subprog, and the kernel stack under them is already in use.
*/
- if ((bpf_enable_priv_stack(env->prog) != NO_PRIV_STACK ||
- env->prog->type == BPF_PROG_TYPE_KPROBE) &&
+ if (bpf_prog_runs_nested(env->prog) &&
!env->prog->aux->priv_stack_requested) {
verbose(env, "recursive %s() in a tracing program\n",
bpf_subprog_name(env, s));
diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c
index 4f8476d36250..b4aab50b8a77 100644
--- a/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c
+++ b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c
@@ -24,13 +24,17 @@ void test_may_goto_priv_stack(void)
return;
err = may_goto_priv_stack__attach(skel);
+ if (err == -EOPNOTSUPP) {
+ test__skip();
+ goto out;
+ }
if (!ASSERT_OK(err, "attach"))
goto out;
memset(skel->bss->regs_64, 0xff, sizeof(skel->bss->regs_64));
memset(skel->bss->regs_128, 0xff, sizeof(skel->bss->regs_128));
- err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.priv_stack_64), &topts);
+ err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), &topts);
if (!ASSERT_OK(err, "test_run"))
goto out;
diff --git a/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c b/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c
index 3da3f33a2898..ee7b79ef77e0 100644
--- a/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c
+++ b/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c
@@ -131,10 +131,14 @@ static void test_user_ringbuf(struct timed_may_goto_user *skel)
static void test_priv_stack(struct timed_may_goto_user *skel)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
- int fd = bpf_program__fd(skel->progs.priv_stack);
+ int fd = bpf_program__fd(skel->progs.trigger);
struct bpf_link *link;
link = bpf_program__attach(skel->progs.priv_stack);
+ if (!link && errno == EOPNOTSUPP) {
+ test__skip();
+ return;
+ }
if (!ASSERT_OK_PTR(link, "attach"))
return;
ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "test_run");
diff --git a/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c
index 5c2aa0aa5055..e11a7f7493a6 100644
--- a/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c
+++ b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c
@@ -10,11 +10,11 @@ __u64 regs_64[7];
__u64 regs_128[7];
/*
- * fentry prog with 64 or more bytes of stack uses private stack.
+ * kprobe.multi prog with 64 or more bytes of stack uses private stack.
* Loop until may_goto expires. r0-r5 should stay zero.
*/
#define TIMED_MAY_GOTO_PRIV_STACK(size, func) \
-SEC("fentry/" #func) \
+SEC("kprobe.multi/" #func) \
__naked void priv_stack_##size(void) \
{ \
asm volatile ( \
@@ -49,4 +49,14 @@ __naked void priv_stack_##size(void) \
TIMED_MAY_GOTO_PRIV_STACK(64, bpf_fentry_test1)
TIMED_MAY_GOTO_PRIV_STACK(128, bpf_fentry_test2)
+/*
+ * No tests in here, just to trigger 'bpf_fentry_test*'
+ * through tracing test_run
+ */
+SEC("fentry/bpf_modify_return_test")
+int trigger(void *ctx)
+{
+ return 0;
+}
+
char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_global_func3.c b/tools/testing/selftests/bpf/progs/test_global_func3.c
index b916355c5e59..a473bed346d2 100644
--- a/tools/testing/selftests/bpf/progs/test_global_func3.c
+++ b/tools/testing/selftests/bpf/progs/test_global_func3.c
@@ -267,6 +267,15 @@ int global_func3_recursion_in_tracing_prog(void *ctx)
return depth(again + 7);
}
+SEC("raw_tp")
+__arch_x86_64
+__arch_arm64
+__failure __msg("recursive depth() in a tracing program")
+int global_func3_recursion_in_raw_tp(void *ctx)
+{
+ return depth(again + 7);
+}
+
__attribute__ ((noinline))
int redirects(struct __sk_buff *skb, int n)
{
diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto_user.c b/tools/testing/selftests/bpf/progs/timed_may_goto_user.c
index 8b85815d45d2..ed56b6afb607 100644
--- a/tools/testing/selftests/bpf/progs/timed_may_goto_user.c
+++ b/tools/testing/selftests/bpf/progs/timed_may_goto_user.c
@@ -277,7 +277,7 @@ int drain(void *ctx)
}
/*
- * A tracing prog with large frames: they are in a private stack.
+ * A kprobe.multi prog with large frames: they are in a private stack.
*/
int priv_stack_spun, priv_stack_cb_spun;
@@ -296,8 +296,8 @@ static int priv_stack_cb(__u64 i, int *spun)
return 0;
}
-SEC("fentry/bpf_fentry_test1")
-int BPF_PROG(priv_stack)
+SEC("kprobe.multi/bpf_fentry_test1")
+int BPF_KPROBE(priv_stack)
{
volatile char buf[128] = {};
int i, n = buf[zero & 127], spun = 0;
@@ -309,3 +309,13 @@ int BPF_PROG(priv_stack)
priv_stack_cb_spun = spun;
return 0;
}
+
+/*
+ * No tests in here, just to trigger 'bpf_fentry_test*'
+ * through tracing test_run
+ */
+SEC("fentry/bpf_modify_return_test")
+int BPF_PROG(trigger)
+{
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c
index e7f410a74a21..299ce68cb212 100644
--- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c
+++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c
@@ -52,7 +52,7 @@ __naked void private_stack_single_prog(void)
" ::: __clobber_all);
}
-SEC("raw_tp")
+SEC("kprobe.multi")
__description("No private stack")
__success
__arch_x86_64
@@ -82,7 +82,7 @@ __naked static long subprog_6args(long a, long b, long c, long d, long e, long f
}
/* x86-64 has one register for the private stack and for the sixth argument */
-SEC("raw_tp")
+SEC("kprobe.multi")
__description("No private stack, stack arguments")
__success
__load_if_JITed()
@@ -100,7 +100,7 @@ __naked void no_private_stack_stack_args(void)
" ::: __clobber_all);
}
#else
-SEC("raw_tp")
+SEC("kprobe.multi")
__description("No private stack, stack arguments")
__skip("stack_arg is not supported by compiler")
__success
@@ -127,6 +127,23 @@ __naked void no_private_stack_kprobe(void)
" ::: __clobber_all);
}
+SEC("fentry/bpf_fentry_test9")
+__description("No private stack, fentry")
+__success
+__arch_x86_64
+__jited(" subq $0x100, %rsp")
+__arch_arm64
+__jited(" sub sp, sp, #0x100")
+__naked void no_private_stack_fentry(void)
+{
+ asm volatile (" \
+ r1 = 42; \
+ *(u64 *)(r10 - 256) = r1; \
+ r0 = 0; \
+ exit; \
+" ::: __clobber_all);
+}
+
SEC("uprobe")
__description("No private stack, uprobe")
__success
@@ -215,7 +232,7 @@ static unsigned long loop_callback(void)
: __clobber_common);
}
-SEC("raw_tp")
+SEC("kprobe.multi")
__description("Private stack, callback")
__success
__arch_x86_64
@@ -273,9 +290,9 @@ __naked void private_stack_callback(void)
: __clobber_common);
}
-SEC("fentry/bpf_fentry_test9")
+SEC("perf_event")
__description("Private stack, exception in main prog")
-__success __retval(0)
+__success
__arch_x86_64
__jited(" pushq %r9")
__jited("...")
@@ -321,9 +338,9 @@ __used static int subprog_exception(void)
return 0;
}
-SEC("fentry/bpf_fentry_test9")
+SEC("perf_event")
__description("Private stack, exception in subprog")
-__success __retval(0)
+__success
__arch_x86_64
__jited(" movq %rdi, -0x200(%r9)")
__jited(" pushq %r9")
@@ -355,20 +372,16 @@ int private_stack_exception_sub_prog(void)
}
/*
- * Nothing attaches the two fentry progs above, so they are loaded, but never
- * run. BPF_PROG_TEST_RUN does run a raw_tp prog. The default exception callback
- * returns the cookie of bpf_throw().
+ * Nothing attaches the two perf_event progs above, so they are loaded, but
+ * never run. BPF_PROG_TEST_RUN does run a raw_tp prog. The default exception
+ * callback returns the cookie of bpf_throw().
*/
SEC("raw_tp")
-__description("Private stack, run, exception in main prog")
+__description("No private stack, run, exception in main prog")
__success __retval(1)
__arch_x86_64
-__jited(" movq %rdi, -0x200(%r9)")
-__jited(" movl $0x1, %edi")
-__jited(" pushq %r9")
-__jited("...")
-__jited(" callq")
-__naked void private_stack_run_exception_main_prog(void)
+__jited(" subq $0x200, %rsp")
+__naked void big_stack_run_exception_main_prog(void)
{
asm volatile (" \
r1 = 42; \
@@ -396,15 +409,11 @@ __naked static void subprog_throw(void)
}
SEC("raw_tp")
-__description("Private stack, run, exception in subprog")
+__description("No private stack, run, exception in subprog")
__success __retval(2)
__arch_x86_64
-__jited(" movq %rdi, -0x200(%r9)")
-__jited(" pushq %r9")
-__jited("...")
-__jited(" callq")
-__jited(" popq %r9")
-__naked void private_stack_run_exception_sub_prog(void)
+__jited(" subq $0x200, %rsp")
+__naked void big_stack_run_exception_sub_prog(void)
{
asm volatile (" \
r1 = 42; \
@@ -416,7 +425,7 @@ __naked void private_stack_run_exception_sub_prog(void)
}
__used
-__naked static void private_stack_subprog_throw(void)
+__naked static void big_stack_subprog_throw(void)
{
asm volatile (" \
r1 = 42; \
@@ -430,24 +439,22 @@ __naked static void private_stack_subprog_throw(void)
: __clobber_all);
}
-/* Only the frame of the main prog counts, and it has nothing pushed at the call */
+/* The subprog that throws has a large frame, the main prog a small one */
SEC("raw_tp")
-__description("Private stack, run, exception in subprog, no private stack in main prog")
+__description("No private stack, run, exception in subprog with a large stack")
__success __retval(3)
__arch_x86_64
__jited(" subq $0x8, %rsp")
__jited("...")
__jited("func #1")
__jited("...")
-__jited(" movq %rdi, -0x200(%r9)")
-__jited(" movl $0x3, %edi")
-__jited(" pushq %r9")
+__jited(" subq $0x200, %rsp")
__naked void no_private_stack_run_exception_sub_prog(void)
{
asm volatile (" \
r1 = 42; \
*(u64 *)(r10 - 8) = r1; \
- call private_stack_subprog_throw; \
+ call big_stack_subprog_throw; \
r0 = 0; \
exit; \
" ::: __clobber_all);
@@ -478,9 +485,9 @@ __noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer)
return 0;
}
-SEC("fentry/bpf_fentry_test9")
+SEC("kprobe.multi")
__description("Private stack, async callback, not nested")
-__success __retval(0)
+__success
__arch_x86_64
__jited(" movabsq $0x{{.*}}, %r9")
__arch_arm64
@@ -503,9 +510,9 @@ int private_stack_async_callback_1(void)
return 0;
}
-SEC("fentry/bpf_fentry_test9")
+SEC("kprobe.multi")
__description("Private stack, async callback, potential nesting")
-__success __retval(0)
+__success
__load_if_JITed()
__log_level(4) __msg("stack depth max 272")
__msg("subprog 0 (private_stack_async_callback_2) main {{.*}} stack 8")
@@ -532,7 +539,7 @@ int private_stack_async_callback_2(void)
return 0;
}
-SEC("fentry/bpf_fentry_test9")
+SEC("kprobe.multi")
__description("private stack, max stack depth is private stack")
__success
__log_level(4) __msg("stack depth max 256")
diff --git a/tools/testing/selftests/bpf/progs/verifier_unwind.c b/tools/testing/selftests/bpf/progs/verifier_unwind.c
index 596e5e0301e5..c779b455e1db 100644
--- a/tools/testing/selftests/bpf/progs/verifier_unwind.c
+++ b/tools/testing/selftests/bpf/progs/verifier_unwind.c
@@ -1197,9 +1197,9 @@ static unsigned long may_unwind_big_stack(void)
* x86-64 keeps the pointer to the private stack in r9 and saves it around a call.
* The value that is read back in the pad is there only when r9 is restored.
*/
-SEC("raw_tp")
+SEC("kprobe.multi")
__description("unwind: private stack, the callee unwinds")
-__success __retval(47)
+__success
__load_if_JITed()
__arch_x86_64
__jited(" movl $0x1234, %esi") /* past the load, see above */
@@ -1232,12 +1232,37 @@ __naked int private_stack_unwinds(void)
}
SEC("raw_tp")
-__description("unwind: private stack, the callee returns")
+__description("unwind: big stack, the callee unwinds")
+__success __retval(47)
+__load_if_JITed()
+__arch_x86_64
+__arch_arm64
+__naked int big_stack_unwinds(void)
+{
+ asm volatile (
+ "*(u64 *)(r10 - 200) = 42;"
+ "r1 = %[one] ll;"
+ "r1 = *(u64 *)(r1 + 0);"
+ "call may_unwind_big_stack;"
+ unwind_goto(1)
+ "exit;"
+ "r1 = *(u64 *)(r10 - 200);"
+ "r0 += r1;"
+ "exit;"
+ :
+ : __imm_addr(one),
+ __imm_unwind_goto(1)
+ : __clobber_all
+ );
+}
+
+SEC("raw_tp")
+__description("unwind: big stack, the callee returns")
__success __retval(49)
__load_if_JITed()
__arch_x86_64
__arch_arm64
-__naked int private_stack_returns(void)
+__naked int big_stack_returns(void)
{
asm volatile (
"*(u64 *)(r10 - 200) = 42;"
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH bpf-next v3 1/4] bpf: Avoid per-CPU private stack in preemptible tracing progs
2026-11-17 11:29 ` [PATCH bpf-next v3 1/4] bpf: Avoid per-CPU private stack in preemptible tracing progs Mike Bazov via B4 Relay
@ 2026-11-17 11:44 ` sashiko-bot
0 siblings, 0 replies; 7+ messages in thread
From: sashiko-bot @ 2026-11-17 11:44 UTC (permalink / raw)
To: Mike Bazov, bpf; +Cc: linux-trace-kernel
Thank you for your contribution! Sashiko AI review found 1 potential
issue to consider:
- [Low] bpf: missing Fixes trailer for commit a76ab5731e32 in commit
message
--
commit 3dec129be9ecaaef86d1be659b28f0d6ce4d6395
Author: Mike Bazov <mikeba@sentinelone.com>
> bpf: Avoid per-CPU private stack in preemptible tracing progs
>
> A per-CPU private stack is only safe if a program never runs in two
> tasks at once on one CPU. Commit a76ab5731e32 ("bpf: Find eligible
> subprogs for private stack support") gave one to tracing, raw_tp and
> tracepoint programs, but they run preemptibly, and two tasks can
> already be inside one of them on a CPU:
>
> - Two raw_tp dispatchers that tail-call the same handler each pass
> their own per-CPU recursion check, so both tasks run the handler.
> - A tracepoint program attached to a syscall tracepoint and to a
> regular tracepoint can be entered by a second task through the
> regular one, which only checks bpf_prog_active.
>
> The second task then overwrites the preempted task's stack. Neither
> case can be seen when the program is loaded, so never give these
> program types a private stack. The next patches make such overlap the
> normal case.
>
> check_recursion() told tracing programs apart by whether they can get a
> private stack. Make it check the program type instead, so that
> recursive subprograms stay rejected in these program types.
>
> Move the selftests that relied on these program types having a private
> stack to kprobe.multi and perf_event programs.
>
> Trade-offs:
>
> - These programs use the kernel stack again.
> - A program with two 2000-byte frames no longer loads: "combined stack
> size of 2 calls is 4000. Too large".
> - Unwinding on a private stack is no longer tested at run time: raw_tp
> was the only such program type that BPF_PROG_TEST_RUN runs.
>
> Signed-off-by: Mike Bazov <mikeba@sentinelone.com>
[Severity: Low]
Since this fixes per-CPU private stack corruption in existing tracing,
raw_tp and tracepoint programs, should it include a Fixes: tag?
Fixes: a76ab5731e32 ("bpf: Find eligible subprogs for private stack support")
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261117-bpf-preempt-v2-r2-v3-0-586376ec3672@sentinelone.com?part=1
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH bpf-next v3 2/4] bpf: Mark programs whose runs must not overlap on a CPU
2026-11-17 11:29 [PATCH bpf-next v3 0/4] bpf: Check recursion per task for preemptible programs Mike Bazov via B4 Relay
2026-11-17 11:29 ` [PATCH bpf-next v3 1/4] bpf: Avoid per-CPU private stack in preemptible tracing progs Mike Bazov via B4 Relay
@ 2026-11-17 11:29 ` Mike Bazov via B4 Relay
2026-11-17 11:29 ` [PATCH bpf-next v3 3/4] bpf: Check program recursion per task via bpf_ctx Mike Bazov via B4 Relay
2026-11-17 11:29 ` [PATCH bpf-next v3 4/4] selftests/bpf: Add preemption recursion test for tracing programs Mike Bazov via B4 Relay
3 siblings, 0 replies; 7+ messages in thread
From: Mike Bazov via B4 Relay @ 2026-11-17 11:29 UTC (permalink / raw)
To: bpf, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: Martin KaFai Lau, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Song Liu, Yonghong Song, Jiri Olsa, John Fastabend,
Emil Tsalapatis, Ihor Solodrai, Steven Rostedt, Masami Hiramatsu,
Mathieu Desnoyers, JP Kobryn, linux-trace-kernel, linux-kernel,
Mike Bazov
From: Mike Bazov <mikeba@sentinelone.com>
The per-CPU recursion counter of tracing and raw tracepoint programs
also keeps a second task out of a program while another task is
preempted inside it on the same CPU. Programs rely on that:
- A per-CPU map value that a program looked up stays its own until the
run ends, which is how programs use these maps as scratch space.
- The budget of timed may_goto is per-CPU memory that the first insns
of a run reset, see commit ec274bb4665c ("bpf: One may_goto time
slice per prog invocation"). A second run on the CPU gives the first
one more time.
- A private stack is per CPU. After the previous patch, struct_ops
programs that request one are the only programs with this check that
still have it.
The next patch checks recursion per task, which lets a second task in.
For these programs that would turn a counted, dropped run into silently
corrupted data, or into a may_goto loop that another task keeps
renewing. Mark them at load time, so that they keep the per-CPU counter.
Set prog->recur_per_cpu when the verifier sees:
- a helper called with a per-CPU map, also through an inner map,
- the address of a per-CPU array value loaded directly,
- bpf_this_cpu_ptr() or bpf_per_cpu_ptr(),
- a tail call: the target runs without its own recursion check,
- timed may_goto,
- a private stack, in a struct_ops program that requests one.
Only verified code counts: a per-CPU map referenced in a branch that a
constant in .rodata disables doesn't mark the program.
No functional change: nothing reads the flag yet.
Trade-offs, once the next patch reads it:
- A program that keeps per-CPU slots of its own, e.g. in a regular
array indexed by bpf_get_smp_processor_id(), is not marked.
- Marked programs keep losing runs to preemption.
Signed-off-by: Mike Bazov <mikeba@sentinelone.com>
---
include/linux/bpf.h | 3 ++-
kernel/bpf/fixups.c | 1 +
kernel/bpf/verifier.c | 11 +++++++++++
3 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a5ad08b7937c..7873f0ef2817 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2063,7 +2063,8 @@ struct bpf_prog {
call_get_func_ip:1, /* Do we call get_func_ip() */
call_session_cookie:1, /* Do we call bpf_session_cookie() */
tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
- sleepable:1; /* BPF program is sleepable */
+ sleepable:1, /* BPF program is sleepable */
+ recur_per_cpu:1; /* Runs must not overlap on a CPU */
enum bpf_prog_type type; /* Type of BPF program */
enum bpf_attach_type expected_attach_type; /* For some prog types */
u32 len; /* Number of filter blocks */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 4d9b67981520..82ad0d3bf465 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -2591,6 +2591,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (!tmg)
return -ENOMEM;
prog->aux->timed_may_goto = tmg;
+ prog->recur_per_cpu = true;
}
prog->jit_required = 1;
cnt = may_goto_expand_timed(insn_buf, &jmp, tmg, reg, save_off);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 2ca1469d8dee..3d54520df3e3 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6050,6 +6050,7 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
if (round_up_stack_depth(env, si[s].stack_depth) >= BPF_PRIV_STACK_MIN_SIZE) {
si[s].priv_stack_mode = PRIV_STACK_ADAPTIVE;
env->prog->aux->jits_use_priv_stack = true;
+ env->prog->recur_per_cpu = true;
}
}
@@ -12916,6 +12917,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
if (err)
return err;
+ /* A per-CPU value stays the run's own only while no other run starts on the CPU */
+ if (meta.map.ptr && bpf_map_is_percpu_map(meta.map.ptr->map_type))
+ env->prog->recur_per_cpu = true;
+
err = record_func_map(env, &meta, func_id, insn_idx);
if (err)
return err;
@@ -12942,6 +12947,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
switch (func_id) {
case BPF_FUNC_tail_call:
+ /* The target runs without a recursion check of its own */
+ env->prog->recur_per_cpu = true;
err = check_resource_leak(env, false, true, "tail_call");
if (err)
return err;
@@ -13048,6 +13055,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
struct bpf_reg_state *reg = ®s[BPF_REG_1];
const struct btf_type *type;
+ env->prog->recur_per_cpu = true;
+
if (reg->type & MEM_RCU) {
type = btf_type_by_id(reg->btf, reg->btf_id);
if (!type || !btf_type_is_struct(type)) {
@@ -19690,6 +19699,8 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
__mark_reg_known(dst_reg, aux->map_off);
dst_reg->type = PTR_TO_MAP_VALUE;
dst_reg->map_ptr = map;
+ if (bpf_map_is_percpu_map(map->map_type))
+ env->prog->recur_per_cpu = true;
WARN_ON_ONCE(map->map_type != BPF_MAP_TYPE_INSN_ARRAY &&
map->max_entries != 1);
/* We want reg->id to be same (0) as map_value is not distinct */
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH bpf-next v3 3/4] bpf: Check program recursion per task via bpf_ctx
2026-11-17 11:29 [PATCH bpf-next v3 0/4] bpf: Check recursion per task for preemptible programs Mike Bazov via B4 Relay
2026-11-17 11:29 ` [PATCH bpf-next v3 1/4] bpf: Avoid per-CPU private stack in preemptible tracing progs Mike Bazov via B4 Relay
2026-11-17 11:29 ` [PATCH bpf-next v3 2/4] bpf: Mark programs whose runs must not overlap on a CPU Mike Bazov via B4 Relay
@ 2026-11-17 11:29 ` Mike Bazov via B4 Relay
2026-11-17 12:20 ` bot+bpf-ci
2026-11-17 11:29 ` [PATCH bpf-next v3 4/4] selftests/bpf: Add preemption recursion test for tracing programs Mike Bazov via B4 Relay
3 siblings, 1 reply; 7+ messages in thread
From: Mike Bazov via B4 Relay @ 2026-11-17 11:29 UTC (permalink / raw)
To: bpf, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: Martin KaFai Lau, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Song Liu, Yonghong Song, Jiri Olsa, John Fastabend,
Emil Tsalapatis, Ihor Solodrai, Steven Rostedt, Masami Hiramatsu,
Mathieu Desnoyers, JP Kobryn, linux-trace-kernel, linux-kernel,
Mike Bazov
From: Mike Bazov <mikeba@sentinelone.com>
Trampoline programs (fentry, fexit, fsession, fmod_ret and their
sleepable variants), raw tracepoint programs (raw_tp, raw_tp.s, tp_btf)
and tracepoint programs on syscall tracepoints run preemptibly, but
their recursion check is a per-CPU, per-program counter. When a task is
preempted inside a program, the counter stays raised, and every other
task that enters the program on that CPU is dropped and counted as a
recursion miss, although nothing recursed. A program can't avoid that:
the counter is checked before its first insn.
Since commit 7dadeaa6e851 ("sched: Further restrict the preemption
modes") x86 and arm64 have only lazy and full preemption, so this
happens in normal operation. A security agent that pairs fentry and
fexit programs on inet_sendmsg() and inet_recvmsg() loses one half of a
pair with every drop. With 200,000 HTTP requests and four CPU-bound
tasks on a 4-CPU VM, 9,254 of 7,966,760 runs of these programs were
dropped on bpf-next, and none of 7,975,426 with this series.
Check recursion per task instead. The runners already link a
stack-allocated bpf_run_ctx into current->bpf_ctx. Move saved_run_ctx
into struct bpf_run_ctx, so every frame links to the outer one, and
record in each frame the program it runs. A program is recursing if a
frame in the current task's chain already runs it. Interrupts and NMIs
run on top of the interrupted task's chain, so recursion through them
is still caught.
bpf_prog_enter_run_ctx() and bpf_prog_exit_run_ctx() do the check for
all runners. The syscall tracepoint runner linked one frame around its
program array and now links one per program. bpf_set_run_ctx() accepts
a NULL ctx, which io_uring passes to hide the BPF context; the frames
below are not walked while it is hidden.
Programs marked by the previous patch keep the per-CPU counter and
behave exactly as before, so no counted drop is traded for corrupted
per-CPU data or a renewed may_goto budget. An extension (freplace) runs
under the check of its target, so a marked one can't be attached to a
target that is checked per task.
Trade-offs:
- An unmarked program that keeps per-CPU slots of its own, see the
previous patch, can have two runs use one slot, and nothing counts
that.
- Marked programs lose runs to preemption as before.
- bench trig-rawtp is about 2% slower: 42.6 and 42.1 M/s on bpf-next,
41.3 and 41.5 M/s with the series. trig-fentry and trig-fexit are
not slower.
- Attaching a marked extension to an unmarked raw_tp, tp_btf or
tracepoint program now fails with -EINVAL.
Signed-off-by: Mike Bazov <mikeba@sentinelone.com>
---
include/linux/bpf.h | 95 ++++++++++++++++++++++++++++++++++++++------
include/linux/bpf_verifier.h | 19 +++++++++
kernel/bpf/syscall.c | 10 +++++
kernel/bpf/trampoline.c | 26 +++++-------
kernel/trace/bpf_trace.c | 11 ++---
net/bpf/test_run.c | 9 +----
net/ipv4/bpf_tcp_ops.c | 6 +--
7 files changed, 129 insertions(+), 47 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 7873f0ef2817..97cbcd4b0d67 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2783,7 +2783,16 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array,
struct bpf_prog_array **new_array);
struct bpf_prog *bpf_prog_dummy(void);
-struct bpf_run_ctx {};
+/*
+ * A frame in the chain of the programs that a task is running, from
+ * current->bpf_ctx to the outermost one.
+ */
+struct bpf_run_ctx {
+ /* The next outer frame, NULL in the outermost one */
+ struct bpf_run_ctx *saved_run_ctx;
+ /* Set while a program whose recursion is checked per task runs in the frame */
+ const struct bpf_prog *prog;
+};
struct bpf_cg_run_ctx {
struct bpf_run_ctx run_ctx;
@@ -2801,7 +2810,6 @@ struct bpf_tramp_run_ctx {
struct bpf_run_ctx run_ctx;
u64 bpf_cookie;
int retval;
- struct bpf_run_ctx *saved_run_ctx;
};
static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
@@ -2809,8 +2817,17 @@ static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
struct bpf_run_ctx *old_ctx = NULL;
#ifdef CONFIG_BPF_SYSCALL
- old_ctx = current->bpf_ctx;
- current->bpf_ctx = new_ctx;
+ old_ctx = READ_ONCE(current->bpf_ctx);
+ if (new_ctx) {
+ new_ctx->saved_run_ctx = old_ctx;
+ new_ctx->prog = NULL;
+ }
+ /* Initialize the frame before interrupts and NMIs on this CPU see it. */
+ barrier();
+ WRITE_ONCE(current->bpf_ctx, new_ctx);
+#else
+ if (new_ctx)
+ new_ctx->saved_run_ctx = NULL;
#endif
return old_ctx;
}
@@ -2818,8 +2835,64 @@ static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
{
#ifdef CONFIG_BPF_SYSCALL
- current->bpf_ctx = old_ctx;
+ WRITE_ONCE(current->bpf_ctx, old_ctx);
+ /* Unlink the frame before its stack storage can be reused. */
+ barrier();
+#endif
+}
+
+/*
+ * Enter @prog, unless that would be recursion:
+ *
+ * - A program that keeps per-CPU state (prog->recur_per_cpu) takes the per-CPU
+ * counter. No other run of it starts on this CPU until it exits.
+ * - Any other program recurses only if the current task is already inside it:
+ * a frame in the task's chain runs it. Interrupts and NMIs run on top of the
+ * chain of the task they interrupt, so they are covered too.
+ *
+ * Returns false if @prog must not run. Call bpf_prog_exit_run_ctx() either way.
+ *
+ * The chain is walked before the frame is linked, so a run that must not start
+ * links nothing. Always inline: the callers are notrace.
+ */
+static __always_inline bool bpf_prog_enter_run_ctx(struct bpf_prog *prog,
+ struct bpf_run_ctx *run_ctx)
+{
+ struct bpf_run_ctx *old_ctx = NULL;
+
+#ifdef CONFIG_BPF_SYSCALL
+ old_ctx = READ_ONCE(current->bpf_ctx);
+#endif
+ run_ctx->saved_run_ctx = old_ctx;
+ if (prog->recur_per_cpu) {
+ run_ctx->prog = NULL;
+ if (!bpf_prog_get_recursion_context(prog))
+ return false;
+ } else {
+ for (struct bpf_run_ctx *ctx = old_ctx; ctx; ctx = ctx->saved_run_ctx) {
+ if (READ_ONCE(ctx->prog) == prog)
+ return false;
+ }
+ run_ctx->prog = prog;
+ }
+#ifdef CONFIG_BPF_SYSCALL
+ /* Initialize the frame before interrupts and NMIs on this CPU see it. */
+ barrier();
+ WRITE_ONCE(current->bpf_ctx, run_ctx);
#endif
+ return true;
+}
+
+/*
+ * Also correct after an enter that returned false: the frame was not linked,
+ * so resetting to its saved_run_ctx changes nothing.
+ */
+static __always_inline void bpf_prog_exit_run_ctx(struct bpf_prog *prog,
+ struct bpf_run_ctx *run_ctx)
+{
+ bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+ if (prog->recur_per_cpu)
+ bpf_prog_put_recursion_context(prog);
}
/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
@@ -3492,7 +3565,6 @@ bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
{
const struct bpf_prog_array_item *item;
struct bpf_prog *prog;
- struct bpf_run_ctx *old_run_ctx;
struct bpf_trace_run_ctx run_ctx;
u32 ret = 1;
@@ -3503,7 +3575,6 @@ bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
run_ctx.is_uprobe = false;
- old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
item = &array->items[0];
while ((prog = READ_ONCE(item->prog))) {
/* Skip dummy_bpf_prog placeholder (len == 0) */
@@ -3512,15 +3583,14 @@ bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
continue;
}
- if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+ run_ctx.bpf_cookie = item->bpf_cookie;
+ if (unlikely(!bpf_prog_enter_run_ctx(prog, &run_ctx.run_ctx))) {
bpf_prog_inc_misses_counter(prog);
- bpf_prog_put_recursion_context(prog);
+ bpf_prog_exit_run_ctx(prog, &run_ctx.run_ctx);
item++;
continue;
}
- run_ctx.bpf_cookie = item->bpf_cookie;
-
if (!prog->sleepable) {
guard(rcu)();
ret &= run_prog(prog, ctx);
@@ -3528,10 +3598,9 @@ bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
ret &= run_prog(prog, ctx);
}
- bpf_prog_put_recursion_context(prog);
+ bpf_prog_exit_run_ctx(prog, &run_ctx.run_ctx);
item++;
}
- bpf_reset_run_ctx(old_run_ctx);
migrate_enable();
return ret;
}
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index f45c18b3cfa2..086d59588d81 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1827,6 +1827,25 @@ static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)
}
}
+/* Is the recursion of @prog checked per task, so that two tasks can run it at once on a CPU? */
+static inline bool bpf_prog_recur_per_task(const struct bpf_prog *prog)
+{
+ if (prog->recur_per_cpu)
+ return false;
+
+ switch (resolve_prog_type(prog)) {
+ case BPF_PROG_TYPE_RAW_TRACEPOINT:
+ case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
+ /* It may be attached to a syscall tracepoint */
+ case BPF_PROG_TYPE_TRACEPOINT:
+ return true;
+ case BPF_PROG_TYPE_TRACING:
+ return bpf_prog_check_recur(prog);
+ default:
+ return false;
+ }
+}
+
#define BPF_REG_TRUSTED_MODIFIERS (MEM_ALLOC | PTR_TRUSTED | NON_OWN_REF)
static inline bool bpf_type_has_unsafe_modifiers(u32 type)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index ec8e78792911..a05eda7c1eeb 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3849,6 +3849,16 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
err = -EINVAL;
goto out_unlock;
}
+ /*
+ * An extension runs under the recursion check of its target. One that
+ * keeps per-CPU state relies on its runs not overlapping on a CPU, which
+ * a target whose recursion is checked per task doesn't provide.
+ */
+ if (prog->type == BPF_PROG_TYPE_EXT && prog->recur_per_cpu &&
+ bpf_prog_recur_per_task(tgt_prog)) {
+ err = -EINVAL;
+ goto out_unlock;
+ }
/*
* The size of these contexts depends on what the program is attached
* to, or run with. That is compared with max_ctx_offset of the program,
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 21d460866dfe..195379cd81ab 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -1342,9 +1342,7 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
{
rcu_read_lock_dont_migrate();
- run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
-
- if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+ if (unlikely(!bpf_prog_enter_run_ctx(prog, &run_ctx->run_ctx))) {
bpf_prog_inc_misses_counter(prog);
if (prog->aux->recursion_detected)
prog->aux->recursion_detected(prog);
@@ -1385,10 +1383,9 @@ static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
__releases(RCU)
{
- bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+ bpf_prog_exit_run_ctx(prog, &run_ctx->run_ctx);
update_prog_stats(prog, start);
- bpf_prog_put_recursion_context(prog);
rcu_read_unlock_migrate();
}
@@ -1401,7 +1398,7 @@ static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
*/
rcu_read_lock_dont_migrate();
- run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+ bpf_set_run_ctx(&run_ctx->run_ctx);
return NO_START_TIME;
}
@@ -1410,7 +1407,7 @@ static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
__releases(RCU)
{
- bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+ bpf_reset_run_ctx(run_ctx->run_ctx.saved_run_ctx);
rcu_read_unlock_migrate();
}
@@ -1422,9 +1419,7 @@ u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
migrate_disable();
might_fault();
- run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
-
- if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+ if (unlikely(!bpf_prog_enter_run_ctx(prog, &run_ctx->run_ctx))) {
bpf_prog_inc_misses_counter(prog);
if (prog->aux->recursion_detected)
prog->aux->recursion_detected(prog);
@@ -1436,10 +1431,9 @@ u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
{
- bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+ bpf_prog_exit_run_ctx(prog, &run_ctx->run_ctx);
update_prog_stats(prog, start);
- bpf_prog_put_recursion_context(prog);
migrate_enable();
rcu_read_unlock_trace();
}
@@ -1451,7 +1445,7 @@ static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,
migrate_disable();
might_fault();
- run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+ bpf_set_run_ctx(&run_ctx->run_ctx);
return bpf_prog_start_time();
}
@@ -1459,7 +1453,7 @@ static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,
static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
{
- bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+ bpf_reset_run_ctx(run_ctx->run_ctx.saved_run_ctx);
update_prog_stats(prog, start);
migrate_enable();
@@ -1472,7 +1466,7 @@ static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
{
rcu_read_lock_dont_migrate();
- run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+ bpf_set_run_ctx(&run_ctx->run_ctx);
return bpf_prog_start_time();
}
@@ -1481,7 +1475,7 @@ static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
__releases(RCU)
{
- bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+ bpf_reset_run_ctx(run_ctx->run_ctx.saved_run_ctx);
update_prog_stats(prog, start);
rcu_read_unlock_migrate();
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index b4ff9d4e86b3..4cb8a51ba999 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -2167,7 +2167,6 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
struct srcu_ctr __percpu *scp = NULL;
struct bpf_prog *prog = link->link.prog;
bool sleepable = prog->sleepable;
- struct bpf_run_ctx *old_run_ctx;
struct bpf_trace_run_ctx run_ctx;
if (sleepable) {
@@ -2177,19 +2176,15 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
rcu_read_lock_dont_migrate();
}
- if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+ run_ctx.bpf_cookie = link->cookie;
+ if (unlikely(!bpf_prog_enter_run_ctx(prog, &run_ctx.run_ctx))) {
bpf_prog_inc_misses_counter(prog);
goto out;
}
- run_ctx.bpf_cookie = link->cookie;
- old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
-
(void)bpf_prog_run(prog, args);
-
- bpf_reset_run_ctx(old_run_ctx);
out:
- bpf_prog_put_recursion_context(prog);
+ bpf_prog_exit_run_ctx(prog, &run_ctx.run_ctx);
if (sleepable) {
migrate_enable();
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index cc3236e3c0c4..8b8c4e9ded9d 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -748,9 +748,6 @@ __bpf_prog_test_run_raw_tp(void *data)
struct bpf_raw_tp_test_run_info *info = data;
struct srcu_ctr __percpu *scp = NULL;
struct bpf_trace_run_ctx run_ctx = {};
- struct bpf_run_ctx *old_run_ctx;
-
- old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
if (info->prog->sleepable) {
scp = rcu_read_lock_tasks_trace();
@@ -759,7 +756,7 @@ __bpf_prog_test_run_raw_tp(void *data)
rcu_read_lock();
}
- if (unlikely(!bpf_prog_get_recursion_context(info->prog))) {
+ if (unlikely(!bpf_prog_enter_run_ctx(info->prog, &run_ctx.run_ctx))) {
bpf_prog_inc_misses_counter(info->prog);
goto out;
}
@@ -767,7 +764,7 @@ __bpf_prog_test_run_raw_tp(void *data)
info->retval = bpf_prog_run(info->prog, info->ctx);
out:
- bpf_prog_put_recursion_context(info->prog);
+ bpf_prog_exit_run_ctx(info->prog, &run_ctx.run_ctx);
if (info->prog->sleepable) {
migrate_enable();
@@ -775,8 +772,6 @@ __bpf_prog_test_run_raw_tp(void *data)
} else {
rcu_read_unlock();
}
-
- bpf_reset_run_ctx(old_run_ctx);
}
int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c
index 2ba73dd6c52c..e1ca34e63de0 100644
--- a/net/ipv4/bpf_tcp_ops.c
+++ b/net/ipv4/bpf_tcp_ops.c
@@ -207,11 +207,11 @@ BPF_CALL_0(bpf_tcp_ops_get_retval)
* return value chained across the bpf_tcp_ops attached to the cgroup
* and is this program's saved_run_ctx.
*/
- if (WARN_ON_ONCE(!ctx->saved_run_ctx))
+ if (WARN_ON_ONCE(!ctx->run_ctx.saved_run_ctx))
return 0;
- return container_of(ctx->saved_run_ctx, struct bpf_tramp_run_ctx,
- run_ctx)->retval;
+ return container_of(ctx->run_ctx.saved_run_ctx,
+ struct bpf_tramp_run_ctx, run_ctx)->retval;
}
const struct bpf_func_proto bpf_tcp_ops_get_retval_proto = {
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH bpf-next v3 3/4] bpf: Check program recursion per task via bpf_ctx
2026-11-17 11:29 ` [PATCH bpf-next v3 3/4] bpf: Check program recursion per task via bpf_ctx Mike Bazov via B4 Relay
@ 2026-11-17 12:20 ` bot+bpf-ci
0 siblings, 0 replies; 7+ messages in thread
From: bot+bpf-ci @ 2026-11-17 12:20 UTC (permalink / raw)
To: mikeba, bpf, ast, daniel, andrii
Cc: martin.lau, eddyz87, memxor, song, yonghong.song, jolsa,
john.fastabend, emil, ihor.solodrai, rostedt, mhiramat,
mathieu.desnoyers, jp.kobryn, linux-trace-kernel, linux-kernel,
mikeba, ast, andrii, daniel, martin.lau, eddyz87, yonghong.song,
mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 4346 bytes --]
> bpf: Check program recursion per task via bpf_ctx
TL;DR: Commit message only: the trade-off list omits that attaching a marked
freplace extension to an fmod_ret target now also fails with -EINVAL, a
userspace-visible change.
> bpf: Check program recursion per task via bpf_ctx
>
> Trampoline programs (fentry, fexit, fsession, fmod_ret and their
> sleepable variants), raw tracepoint programs (raw_tp, raw_tp.s, tp_btf)
> and tracepoint programs on syscall tracepoints run preemptibly, but
> their recursion check is a per-CPU, per-program counter. When a task is
> preempted inside a program, the counter stays raised, and every other
> task that enters the program on that CPU is dropped and counted as a
> recursion miss, although nothing recursed. A program can't avoid that:
> the counter is checked before its first insn.
>
> Since commit 7dadeaa6e851 ("sched: Further restrict the preemption
> modes") x86 and arm64 have only lazy and full preemption, so this
> happens in normal operation. A security agent that pairs fentry and
> fexit programs on inet_sendmsg() and inet_recvmsg() loses one half of a
> pair with every drop. With 200,000 HTTP requests and four CPU-bound
> tasks on a 4-CPU VM, 9,254 of 7,966,760 runs of these programs were
> dropped on bpf-next, and none of 7,975,426 with this series.
>
> Check recursion per task instead. The runners already link a
> stack-allocated bpf_run_ctx into current->bpf_ctx. Move saved_run_ctx
> into struct bpf_run_ctx, so every frame links to the outer one, and
> record in each frame the program it runs. A program is recursing if a
> frame in the current task's chain already runs it. Interrupts and NMIs
> run on top of the interrupted task's chain, so recursion through them
> is still caught.
>
> bpf_prog_enter_run_ctx() and bpf_prog_exit_run_ctx() do the check for
> all runners. The syscall tracepoint runner linked one frame around its
> program array and now links one per program. bpf_set_run_ctx() accepts
> a NULL ctx, which io_uring passes to hide the BPF context; the frames
> below are not walked while it is hidden.
>
> Programs marked by the previous patch keep the per-CPU counter and
> behave exactly as before, so no counted drop is traded for corrupted
> per-CPU data or a renewed may_goto budget. An extension (freplace) runs
> under the check of its target, so a marked one can't be attached to a
> target that is checked per task.
>
> Trade-offs:
>
> - An unmarked program that keeps per-CPU slots of its own, see the
> previous patch, can have two runs use one slot, and nothing counts
> that.
> - Marked programs lose runs to preemption as before.
> - bench trig-rawtp is about 2% slower: 42.6 and 42.1 M/s on bpf-next,
> 41.3 and 41.5 M/s with the series. trig-fentry and trig-fexit are
> not slower.
> - Attaching a marked extension to an unmarked raw_tp, tp_btf or
> tracepoint program now fails with -EINVAL.
>
> Signed-off-by: Mike Bazov <mikeba@sentinelone.com>
>
The trade-off list says a marked extension fails with -EINVAL when
attached to an unmarked raw_tp, tp_btf or tracepoint program. Is the list
complete?
bpf_prog_recur_per_task() returns bpf_prog_check_recur(prog) for
BPF_PROG_TYPE_TRACING:
case BPF_PROG_TYPE_TRACING:
return bpf_prog_check_recur(prog);
That is true for every expected_attach_type except BPF_TRACE_ITER, so it
is also true for BPF_MODIFY_RETURN. bpf_check_attach_target() only rejects
an extension when the target is fentry, fexit, fsession or one of their
_MULTI variants, so a non-sleepable fmod_ret program with a global subprog
is a valid freplace target.
An extension that uses a per-CPU map, a tail call, timed may_goto or
bpf_this_cpu_ptr()/bpf_per_cpu_ptr() could be attached to such a target
before this patch. It now hits the new check in bpf_tracing_prog_attach():
if (prog->type == BPF_PROG_TYPE_EXT && prog->recur_per_cpu &&
bpf_prog_recur_per_task(tgt_prog)) {
err = -EINVAL;
goto out_unlock;
}
Should the commit message list fmod_ret targets next to raw_tp, tp_btf and
tracepoint, since this is a userspace-visible attach failure?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/44076814540
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH bpf-next v3 4/4] selftests/bpf: Add preemption recursion test for tracing programs
2026-11-17 11:29 [PATCH bpf-next v3 0/4] bpf: Check recursion per task for preemptible programs Mike Bazov via B4 Relay
` (2 preceding siblings ...)
2026-11-17 11:29 ` [PATCH bpf-next v3 3/4] bpf: Check program recursion per task via bpf_ctx Mike Bazov via B4 Relay
@ 2026-11-17 11:29 ` Mike Bazov via B4 Relay
3 siblings, 0 replies; 7+ messages in thread
From: Mike Bazov via B4 Relay @ 2026-11-17 11:29 UTC (permalink / raw)
To: bpf, Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: Martin KaFai Lau, Eduard Zingerman, Kumar Kartikeya Dwivedi,
Song Liu, Yonghong Song, Jiri Olsa, John Fastabend,
Emil Tsalapatis, Ihor Solodrai, Steven Rostedt, Masami Hiramatsu,
Mathieu Desnoyers, JP Kobryn, linux-trace-kernel, linux-kernel,
Mike Bazov
From: Mike Bazov <mikeba@sentinelone.com>
Pin two threads to one CPU. Thread B is SCHED_FIFO and waits for a
signal that thread A sends from inside the program under test, so B
preempts A there and enters the same program on the same CPU. Each
subtest runs 20 rounds and checks in every round that A was inside the
program when B ran. It is skipped if that never happens, as on a kernel
that doesn't preempt kernel code.
- fentry, fentry_s, raw_tp, raw_tp_s, tp_syscall: a program without
per-CPU state. Both threads must run it in every round, without
recursion misses.
- *_percpu, percpu_var, this_cpu_ptr, tail_call, may_goto: a program
with a per-CPU map, a per-CPU variable, bpf_this_cpu_ptr(), a tail
call or timed may_goto. B must be dropped and counted in every
round. A constant selects the per-CPU state that the handler
touches; with none selected, the same code doesn't mark the program.
- two_dispatchers: two raw tracepoint programs tail-call one handler.
Both threads run it at once, and A checks that the stack buffer it
filled before B ran is intact.
- recursion_raw_tp, recursion_two_hops, recursion_test_run: true
recursion is still dropped and counted: directly, through a second
program, and from BPF_PROG_TEST_RUN.
- ext_attach: an extension that uses a per-CPU map can't be attached
to a raw tracepoint program that doesn't, while one without can.
Signed-off-by: Mike Bazov <mikeba@sentinelone.com>
---
.../selftests/bpf/prog_tests/trampoline_preempt.c | 352 +++++++++++++++++++++
.../selftests/bpf/progs/trampoline_preempt.c | 258 +++++++++++++++
2 files changed, 610 insertions(+)
diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_preempt.c b/tools/testing/selftests/bpf/prog_tests/trampoline_preempt.c
new file mode 100644
index 000000000000..f282fdba5d16
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/trampoline_preempt.c
@@ -0,0 +1,352 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <sys/prctl.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <test_progs.h>
+#include "trampoline_preempt.skel.h"
+
+#define NR_ROUNDS 20
+
+/* The kernel's own "not supported" error, which the libc headers don't have */
+#ifndef ENOTSUPP
+#define ENOTSUPP 524
+#endif
+
+/*
+ * Threads A and B are pinned to one CPU. B is SCHED_FIFO and waits for a signal:
+ *
+ * A: prctl() -> enters the program under test
+ * the program sends B the signal
+ * B: wakes up and preempts A, which is still inside the program
+ * prctl() -> enters the same program on the same CPU
+ *
+ * A program whose recursion is checked per task runs for B too. A program that
+ * keeps per-CPU state keeps the per-CPU check: B's run is dropped and counted
+ * as a recursion miss.
+ */
+
+/* The per-CPU state that a program touches, see progs/trampoline_preempt.c */
+enum { NO_PERCPU, PERCPU_MAP, PERCPU_VAR, THIS_CPU_PTR };
+
+enum expect {
+ B_RUNS, /* B runs the program, without recursion misses */
+ B_DROPPED, /* B's runs are dropped and counted as recursion misses */
+ B_RUNS_HANDLER, /* two_dispatchers: dropped by one dispatcher, runs through the other */
+};
+
+/* What threads A and B share */
+struct ctx {
+ struct trampoline_preempt *skel;
+ sigset_t sigset;
+ int cpu, err_a, err_b;
+ int overlaps; /* rounds in which B found A still inside the program */
+};
+
+/* Only timed may_goto keeps its budget per CPU, and only these JITs have it */
+static bool has_timed_may_goto(void)
+{
+#if defined(__x86_64__) || defined(__aarch64__) || defined(__s390x__) || \
+ defined(__powerpc64__) || (defined(__riscv) && __riscv_xlen == 64) || \
+ (defined(__loongarch__) && __loongarch_grlen == 64)
+ return is_jit_enabled();
+#else
+ return false;
+#endif
+}
+
+static int pin_to_cpu(int cpu)
+{
+ cpu_set_t set;
+
+ CPU_ZERO(&set);
+ CPU_SET(cpu, &set);
+ return sched_setaffinity(0, sizeof(set), &set) ? -errno : 0;
+}
+
+/* A enters the program under test, which wakes B from inside it */
+static void *worker_a(void *arg)
+{
+ struct ctx *c = arg;
+ int i;
+
+ c->err_a = pin_to_cpu(c->cpu);
+ c->skel->bss->tid_a = syscall(SYS_gettid);
+ for (i = 0; i < NR_ROUNDS && !c->err_a; i++) {
+ /* Passes every hook that the programs under test are attached to */
+ if (prctl(PR_SET_NAME, "worker_a"))
+ c->err_a = -errno;
+ }
+ return NULL;
+}
+
+/* B is SCHED_FIFO on A's CPU: A runs only while B waits, and B preempts A as soon as A wakes it */
+static void *worker_b(void *arg)
+{
+ struct sched_param sp = { .sched_priority = 1 };
+ struct timespec ts = { .tv_sec = 5 }; /* give up if the signal never comes */
+ struct ctx *c = arg;
+ int i;
+
+ c->err_b = pin_to_cpu(c->cpu);
+ if (!c->err_b && sched_setscheduler(0, SCHED_FIFO, &sp))
+ c->err_b = -errno;
+ c->skel->bss->tid_b = syscall(SYS_gettid);
+ for (i = 0; i < NR_ROUNDS && !c->err_b; i++) {
+ while (sigtimedwait(&c->sigset, NULL, &ts) < 0) {
+ if (errno != EINTR) {
+ c->err_b = -errno;
+ return NULL;
+ }
+ }
+ /* B preempted A only if A is still inside the program */
+ if (*(volatile int *)&c->skel->bss->a_inside)
+ c->overlaps++;
+ /* B's own run of the program, while A is preempted inside it */
+ if (prctl(PR_SET_NAME, "worker_b"))
+ c->err_b = -errno;
+ }
+ return NULL;
+}
+
+/* Load and attach only programs @a, @b and @c. NULL after a failure or a skip. */
+static struct trampoline_preempt *load_progs(const char *a, const char *b, const char *c,
+ int percpu)
+{
+ struct trampoline_preempt *skel = trampoline_preempt__open();
+ struct bpf_program *p;
+ const char *name;
+ int err;
+
+ if (!ASSERT_OK_PTR(skel, "open"))
+ return NULL;
+ bpf_object__for_each_program(p, skel->obj) {
+ name = bpf_program__name(p);
+ bpf_program__set_autoload(p, !strcmp(name, a) || (b && !strcmp(name, b)) ||
+ (c && !strcmp(name, c)));
+ }
+ skel->bss->sig = SIGUSR1;
+ skel->rodata->percpu = percpu;
+ err = trampoline_preempt__load(skel);
+ /* The kernel or its JIT doesn't support one of the programs */
+ if (err == -EOPNOTSUPP || err == -ENOTSUPP)
+ test__skip();
+ else if (ASSERT_OK(err, "load") && ASSERT_OK(trampoline_preempt__attach(skel), "attach"))
+ return skel;
+ trampoline_preempt__destroy(skel);
+ return NULL;
+}
+
+static const struct subtest {
+ const char *name;
+ const char *prog; /* A enters it; its recursion misses are checked */
+ int percpu; /* the per-CPU state that the program touches */
+ enum expect expect;
+ const char *prog_b; /* two_dispatchers: B reaches the handler through this one */
+ const char *handler; /* two_dispatchers: the target of both tail calls */
+} subtests[] = {
+ /* Each kind of program, without per-CPU state and with a per-CPU map lookup */
+ { "fentry", "on_fentry", NO_PERCPU, B_RUNS },
+ { "fentry_percpu", "on_fentry", PERCPU_MAP, B_DROPPED },
+ { "fentry_s", "on_fentry_s", NO_PERCPU, B_RUNS },
+ { "fentry_s_percpu", "on_fentry_s", PERCPU_MAP, B_DROPPED },
+ { "raw_tp", "on_raw_tp", NO_PERCPU, B_RUNS },
+ { "raw_tp_percpu", "on_raw_tp", PERCPU_MAP, B_DROPPED },
+ { "raw_tp_s", "on_raw_tp_s", NO_PERCPU, B_RUNS },
+ { "raw_tp_s_percpu", "on_raw_tp_s", PERCPU_MAP, B_DROPPED },
+ { "tp_syscall", "on_tp_syscall", NO_PERCPU, B_RUNS },
+ { "tp_syscall_percpu", "on_tp_syscall", PERCPU_MAP, B_DROPPED },
+ /* The other things that keep a program on the per-CPU check */
+ { "percpu_var", "on_raw_tp", PERCPU_VAR, B_DROPPED },
+ { "this_cpu_ptr", "on_raw_tp", THIS_CPU_PTR, B_DROPPED },
+ { "tail_call", "on_tail_call", NO_PERCPU, B_DROPPED },
+ { "may_goto", "on_may_goto", NO_PERCPU, B_DROPPED },
+ /* Both threads are inside the handler at once, and A's stack must stay intact */
+ { "two_dispatchers", "dispatcher_a", NO_PERCPU, B_RUNS_HANDLER,
+ "dispatcher_b", "handler" },
+};
+
+static void run_subtest(const struct subtest *t)
+{
+ struct bpf_prog_info info = {};
+ __u32 info_len = sizeof(info), zero = 0;
+ struct ctx c = {};
+ sigset_t oldset;
+ pthread_t ta, tb;
+ int err, fd;
+
+ if (!strcmp(t->name, "may_goto") && !has_timed_may_goto()) {
+ test__skip();
+ return;
+ }
+ c.skel = load_progs(t->prog, t->prog_b, t->handler, t->percpu);
+ if (!c.skel)
+ return;
+ if (t->handler) {
+ /* Both dispatchers tail-call the handler through the prog array */
+ fd = bpf_program__fd(bpf_object__find_program_by_name(c.skel->obj, t->handler));
+ err = bpf_map_update_elem(bpf_map__fd(c.skel->maps.prog_array), &zero, &fd,
+ BPF_ANY);
+ if (!ASSERT_OK(err, "prog_array_update"))
+ goto out;
+ }
+
+ /* Both threads go on this CPU. B takes the signal with sigtimedwait(), so block it. */
+ c.cpu = sched_getcpu();
+ sigemptyset(&c.sigset);
+ sigaddset(&c.sigset, SIGUSR1);
+ if (!ASSERT_OK(pthread_sigmask(SIG_BLOCK, &c.sigset, &oldset), "sigmask"))
+ goto out;
+ if (ASSERT_OK(pthread_create(&tb, NULL, worker_b, &c), "pthread_b")) {
+ /* A's program needs tid_b to wake B */
+ while (!c.skel->bss->tid_b)
+ usleep(1000);
+ if (ASSERT_OK(pthread_create(&ta, NULL, worker_a, &c), "pthread_a"))
+ pthread_join(ta, NULL);
+ pthread_join(tb, NULL);
+ }
+ pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+
+ /* B may not be allowed to become SCHED_FIFO */
+ if (c.err_b == -EPERM || c.err_b == -EACCES) {
+ test__skip();
+ goto out;
+ }
+ ASSERT_OK(c.err_a, "worker_a");
+ ASSERT_OK(c.err_b, "worker_b");
+ ASSERT_EQ(c.skel->bss->sig_err, 0, "sig_err");
+ if (!c.overlaps) {
+ /* B never preempted A inside the program, so nothing was tested */
+ test__skip();
+ goto out;
+ }
+ ASSERT_EQ(c.overlaps, NR_ROUNDS, "overlaps");
+ ASSERT_EQ(c.skel->bss->runs_a, NR_ROUNDS, "runs_a");
+ ASSERT_EQ(c.skel->bss->runs_b, t->expect == B_DROPPED ? 0 : NR_ROUNDS, "runs_b");
+ if (t->handler)
+ ASSERT_EQ(c.skel->bss->stack_corrupt, 0, "stack_corrupt");
+ fd = bpf_program__fd(bpf_object__find_program_by_name(c.skel->obj, t->prog));
+ err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
+ if (!ASSERT_OK(err, "prog_info"))
+ goto out;
+ /* A program on sys_enter also misses B's other syscalls: at least one miss per round */
+ if (t->expect == B_RUNS)
+ ASSERT_EQ(info.recursion_misses, 0, "recursion_misses");
+ else
+ ASSERT_GE(info.recursion_misses, NR_ROUNDS, "recursion_misses");
+out:
+ trampoline_preempt__destroy(c.skel);
+}
+
+/*
+ * The program calls bpf_printk(), which fires the bpf_trace_printk tracepoint
+ * that it is attached to: the nested run must be rejected as recursion. With a
+ * hop, a second program runs in between, so the check has to look past the
+ * frame right below.
+ */
+static const struct recursion {
+ const char *name, *prog, *hop;
+ bool test_run; /* Start the outer run with BPF_PROG_TEST_RUN of @prog */
+} recursions[] = {
+ { "recursion_raw_tp", "recur_raw_tp" },
+ { "recursion_two_hops", "recur_two_hops", "recur_hop" },
+ { "recursion_test_run", "recur_raw_tp", .test_run = true },
+};
+
+static void run_recursion(const struct recursion *t)
+{
+ struct trampoline_preempt *skel = load_progs(t->prog, t->hop, "recur_trigger", NO_PERCPU);
+ LIBBPF_OPTS(bpf_test_run_opts, opts);
+ struct bpf_prog_info info = {};
+ __u32 info_len = sizeof(info);
+ int err, fd, run_fd;
+
+ if (!skel)
+ return;
+ fd = bpf_program__fd(bpf_object__find_program_by_name(skel->obj, t->prog));
+ /* The outer run: of @prog itself, or of a program that fires its tracepoint */
+ run_fd = t->test_run ? fd : bpf_program__fd(skel->progs.recur_trigger);
+ skel->bss->recur_tid = syscall(SYS_gettid);
+ err = bpf_prog_test_run_opts(run_fd, &opts);
+ skel->bss->recur_tid = 0;
+ /* The outer run got in; the nested one was dropped and counted */
+ if (ASSERT_OK(err, "test_run")) {
+ ASSERT_EQ(skel->bss->recur_runs, 1, "recur_runs");
+ ASSERT_EQ(skel->bss->hop_runs, t->hop ? 1 : 0, "hop_runs");
+ err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
+ if (ASSERT_OK(err, "prog_info"))
+ ASSERT_EQ(info.recursion_misses, 1, "recursion_misses");
+ }
+ trampoline_preempt__destroy(skel);
+}
+
+/*
+ * An extension runs under the recursion check of its target. A target that
+ * keeps no per-CPU state is checked per task, so an extension that does keep
+ * such state can't be attached to it.
+ */
+static void run_ext_subtest(void)
+{
+ static const struct {
+ const char *prog;
+ int err;
+ } exts[] = {
+ { "ext_plain", 0 }, /* no per-CPU state: attaches */
+ { "ext_percpu", -EINVAL }, /* a per-CPU map lookup: rejected */
+ };
+ struct trampoline_preempt *tgt_skel, *skel = NULL;
+ struct bpf_program *ext, *p;
+ struct bpf_link *link;
+ int i, err, tgt_fd;
+
+ tgt_skel = trampoline_preempt__open();
+ if (!ASSERT_OK_PTR(tgt_skel, "open_tgt"))
+ return;
+ bpf_object__for_each_program(p, tgt_skel->obj)
+ bpf_program__set_autoload(p, p == tgt_skel->progs.ext_target);
+ if (!ASSERT_OK(trampoline_preempt__load(tgt_skel), "load_tgt"))
+ goto out;
+ tgt_fd = bpf_program__fd(tgt_skel->progs.ext_target);
+
+ for (i = 0; i < ARRAY_SIZE(exts); i++) {
+ skel = trampoline_preempt__open();
+ if (!ASSERT_OK_PTR(skel, "open_ext"))
+ goto out;
+ ext = bpf_object__find_program_by_name(skel->obj, exts[i].prog);
+ bpf_object__for_each_program(p, skel->obj)
+ bpf_program__set_autoload(p, p == ext);
+ err = bpf_program__set_attach_target(ext, tgt_fd, "ext_target");
+ if (!ASSERT_OK(err, "set_attach_target"))
+ goto out;
+ if (!ASSERT_OK(trampoline_preempt__load(skel), "load_ext"))
+ goto out;
+
+ link = bpf_program__attach_freplace(ext, tgt_fd, "ext_target");
+ ASSERT_EQ(libbpf_get_error(link), exts[i].err, "attach_ext");
+ bpf_link__destroy(link);
+ trampoline_preempt__destroy(skel);
+ skel = NULL;
+ }
+out:
+ trampoline_preempt__destroy(skel);
+ trampoline_preempt__destroy(tgt_skel);
+}
+
+void test_trampoline_preempt(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(subtests); i++) {
+ if (test__start_subtest(subtests[i].name))
+ run_subtest(&subtests[i]);
+ }
+ for (i = 0; i < ARRAY_SIZE(recursions); i++) {
+ if (test__start_subtest(recursions[i].name))
+ run_recursion(&recursions[i]);
+ }
+ if (test__start_subtest("ext_attach"))
+ run_ext_subtest();
+}
diff --git a/tools/testing/selftests/bpf/progs/trampoline_preempt.c b/tools/testing/selftests/bpf/progs/trampoline_preempt.c
new file mode 100644
index 000000000000..ce596528c8a5
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/trampoline_preempt.c
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "vmlinux.h"
+#include <asm/unistd.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "bpf_compiler.h"
+#include <bpf_may_goto.h>
+
+/* The programs of prog_tests/trampoline_preempt.c. The comment there describes the test. */
+
+char _license[] SEC("license") = "GPL";
+
+struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
+void bpf_task_release(struct task_struct *p) __ksym;
+int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, u64 value) __ksym;
+
+/* The target of the tail calls: empty for tail_call, the handler for two_dispatchers */
+struct {
+ __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, __u32);
+ __type(value, __u32);
+} prog_array SEC(".maps");
+
+/* recur_two_hops deletes from it, which runs recur_hop */
+struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, int);
+} recur_map SEC(".maps");
+
+/* The three kinds of per-CPU state: a map, a variable, and a kernel variable for the helper */
+struct {
+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, int);
+ __type(value, __u64);
+} percpu_map SEC(".maps");
+
+__u64 percpu_var SEC(".percpu");
+extern const struct rq runqueues __ksym;
+
+/* The per-CPU state that a program touches. The test picks one before the load. */
+enum { NO_PERCPU, PERCPU_MAP, PERCPU_VAR, THIS_CPU_PTR };
+const volatile int percpu;
+
+/* Set by the test */
+int tid_a, tid_b, sig, recur_tid;
+int loops; /* 0, read at run time so that the may_goto loop is not optimized away */
+
+/* Read by the test */
+int runs_a, runs_b, sig_err, stack_corrupt, recur_runs, hop_runs;
+int a_inside; /* A is inside the program: B reads it to see that it preempted A there */
+
+static void signal_worker_b(void)
+{
+ struct task_struct *task = bpf_task_from_pid(tid_b);
+
+ if (!task || bpf_send_signal_task(task, sig, PIDTYPE_PID, 0))
+ sig_err = 1;
+ if (task)
+ bpf_task_release(task);
+}
+
+/*
+ * The body of every program under test: count the run of A or of B. A also
+ * wakes B from here, so B preempts A inside the program and enters it too.
+ */
+static __always_inline int record_run(void)
+{
+ __u32 tid = (__u32)bpf_get_current_pid_tgid();
+ int zero = 0;
+
+ /* The verifier walks only the selected branch, so only that one marks the program */
+ switch (percpu) {
+ case PERCPU_MAP:
+ bpf_map_lookup_elem(&percpu_map, &zero);
+ break;
+ case PERCPU_VAR:
+ percpu_var++;
+ break;
+ case THIS_CPU_PTR:
+ bpf_this_cpu_ptr(&runqueues);
+ break;
+ }
+
+ if (tid == tid_a) {
+ runs_a++;
+ /* volatile: both stores must stay, B reads a_inside between them */
+ *(volatile int *)&a_inside = 1;
+ signal_worker_b();
+ *(volatile int *)&a_inside = 0;
+ } else if (tid == tid_b) {
+ runs_b++;
+ }
+ return 0;
+}
+
+/* Both threads call prctl(PR_SET_NAME), which passes each of the hooks below */
+
+SEC("fentry/__set_task_comm")
+int BPF_PROG(on_fentry)
+{
+ return record_run();
+}
+
+SEC("fentry.s/" SYS_PREFIX "sys_prctl")
+int BPF_PROG(on_fentry_s)
+{
+ return record_run();
+}
+
+SEC("raw_tp/task_rename")
+int BPF_PROG(on_raw_tp)
+{
+ return record_run();
+}
+
+SEC("raw_tp.s/sys_enter")
+int BPF_PROG(on_raw_tp_s, struct pt_regs *regs, long id)
+{
+ return id == __NR_prctl ? record_run() : 0;
+}
+
+SEC("tp/syscalls/sys_enter_prctl")
+int on_tp_syscall(void *ctx)
+{
+ return record_run();
+}
+
+/* The prog array is empty, so the tail call does nothing */
+SEC("raw_tp/task_rename")
+int on_tail_call(void *ctx)
+{
+ bpf_tail_call(ctx, &prog_array, 0);
+ return record_run();
+}
+
+/* can_loop is a may_goto. With timed may_goto its time budget is per CPU. */
+SEC("raw_tp/task_rename")
+int on_may_goto(void *ctx)
+{
+ int i;
+
+ for (i = loops; i < 1 && can_loop; i++)
+ ;
+ return record_run();
+}
+
+/* Two dispatchers tail-call one handler: A enters through dispatcher_a, B through dispatcher_b */
+SEC("raw_tp/task_rename")
+int dispatcher_a(void *ctx)
+{
+ if ((__u32)bpf_get_current_pid_tgid() == tid_a)
+ bpf_tail_call(ctx, &prog_array, 0);
+ return 0;
+}
+
+SEC("raw_tp/sys_enter")
+int BPF_PROG(dispatcher_b, struct pt_regs *regs, long id)
+{
+ if (id == __NR_prctl && (__u32)bpf_get_current_pid_tgid() == tid_b)
+ bpf_tail_call(ctx, &prog_array, 0);
+ return 0;
+}
+
+/* The handler keeps 64 bytes of stack across the point where B runs it too */
+SEC("raw_tp")
+int handler(void *ctx)
+{
+ __u32 tid = (__u32)bpf_get_current_pid_tgid();
+ __u64 buf[8];
+
+ __pragma_loop_unroll
+ for (int i = 0; i < 8; i++)
+ buf[i] = ((__u64)tid << 32) | i;
+ /* Make the compiler keep buf on the stack across record_run() */
+ asm volatile("" : "+m"(buf));
+
+ record_run();
+
+ asm volatile("" : "+m"(buf));
+ /* Don't trust a copy of tid that may have been spilled to the stack under test */
+ tid = (__u32)bpf_get_current_pid_tgid();
+ __pragma_loop_unroll
+ for (int i = 0; i < 8; i++) {
+ if (buf[i] != (((__u64)tid << 32) | i))
+ stack_corrupt = 1;
+ }
+ return 0;
+}
+
+/* bpf_printk() fires the bpf_trace_printk tracepoint that the program is attached to */
+SEC("raw_tp/bpf_trace_printk")
+int recur_raw_tp(void *ctx)
+{
+ /* Other tasks print too: only the runs of the test's thread count */
+ if ((__u32)bpf_get_current_pid_tgid() != recur_tid)
+ return 0;
+ recur_runs++;
+ bpf_printk("recur");
+ return 0;
+}
+
+/* The test runs it with BPF_PROG_TEST_RUN to fire the tracepoint once */
+SEC("syscall")
+int recur_trigger(void *ctx)
+{
+ bpf_printk("trigger");
+ return 0;
+}
+
+/* The same through a second program: recur_two_hops -> recur_hop -> recur_two_hops */
+SEC("raw_tp/bpf_trace_printk")
+int recur_two_hops(void *ctx)
+{
+ int zero = 0;
+
+ if ((__u32)bpf_get_current_pid_tgid() != recur_tid)
+ return 0;
+ recur_runs++;
+ bpf_map_delete_elem(&recur_map, &zero);
+ return 0;
+}
+
+SEC("fentry/htab_map_delete_elem")
+int BPF_PROG(recur_hop, struct bpf_map *map)
+{
+ if (map != (struct bpf_map *)&recur_map || (__u32)bpf_get_current_pid_tgid() != recur_tid)
+ return 0;
+ hop_runs++;
+ bpf_printk("hop");
+ return 0;
+}
+
+/* For ext_attach: a target without per-CPU state, and an extension without and with it */
+SEC("raw_tp/task_rename")
+int ext_target(struct bpf_raw_tracepoint_args *ctx)
+{
+ return 0;
+}
+
+SEC("freplace")
+int ext_plain(struct bpf_raw_tracepoint_args *ctx)
+{
+ return 0;
+}
+
+SEC("freplace")
+int ext_percpu(struct bpf_raw_tracepoint_args *ctx)
+{
+ int zero = 0;
+
+ bpf_map_lookup_elem(&percpu_map, &zero);
+ return 0;
+}
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread