[PATCH bpf] bpf: guard classic uprobes against private-stack corruption

[Christian Simon <simon@swine.de>]

Eligible BPF programs use one private stack per program and CPU.
bpf_prog_run_array_uprobe() uses migrate_disable() to keep an
invocation on one CPU, but another task can still preempt it and run
the same program on that CPU. The second invocation then reuses and
can overwrite the first invocation's private stack.

Protect each real program invocation with the existing per-program
recursion context. When the program is already active on this CPU,
account for the missed invocation and skip it. This matches the
private-stack protection in bpf_prog_run_array_sleepable().

Skip dummy_bpf_prog before acquiring the recursion context because its
active pointer is NULL.

Fixes: 7d1cd70d4b16 ("bpf, x86: Support private stack in jit")
Closes: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/issues/3056
Cc: stable@vger.kernel.org
Signed-off-by: Christian Simon <simon@swine.de>
---

This is my first contribution to the BPF subsystem. I reproduced the
failure described in the linked report and would particularly appreciate
review of the recursion-context handling.

I have a reproducer, that this patch fixes, I am also unsure if this is
something that should be part of the selftests.

Tests: https://github.com/simonswine/beyla/commit/20c9912a02c5d5924c992faf7ee353c7d71c9fa0 


The analogous uprobe-multi path appears affected but is not addressed by
this patch.


 include/linux/bpf.h | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 7719f6528445..11e3d7f00543 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2572,12 +2572,20 @@ static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
 
 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);
 
+#ifdef CONFIG_BPF_SYSCALL
+void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
+#else
+static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
+{
+}
+#endif
+
 static __always_inline u32
 bpf_prog_run_array(const struct bpf_prog_array *array,
 		   const void *ctx, bpf_prog_run_fn run_prog)
 {
 	const struct bpf_prog_array_item *item;
-	const struct bpf_prog *prog;
+	struct bpf_prog *prog;
 	struct bpf_run_ctx *old_run_ctx;
 	struct bpf_trace_run_ctx run_ctx;
 	u32 ret = 1;
@@ -2635,15 +2643,30 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,
 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
 	item = &array->items[0];
 	while ((prog = READ_ONCE(item->prog))) {
+		/* dummy_bpf_prog has no recursion state. */
+		if (unlikely(!prog->len)) {
+			item++;
+			continue;
+		}
+
+		if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+			bpf_prog_inc_misses_counter(prog);
+			bpf_prog_put_recursion_context(prog);
+			item++;
+			continue;
+		}
+
 		if (!prog->sleepable)
 			rcu_read_lock();
 
 		run_ctx.bpf_cookie = item->bpf_cookie;
 		ret &= run_prog(prog, ctx);
-		item++;
 
 		if (!prog->sleepable)
 			rcu_read_unlock();
+
+		bpf_prog_put_recursion_context(prog);
+		item++;
 	}
 	bpf_reset_run_ctx(old_run_ctx);
 	migrate_enable();
@@ -3208,8 +3231,6 @@ static inline bool has_current_bpf_ctx(void)
 	return !!current->bpf_ctx;
 }
 
-void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
-
 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
 		     enum bpf_dynptr_type type, u32 offset, u32 size);
 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
@@ -3538,10 +3559,6 @@ static inline bool has_current_bpf_ctx(void)
 	return false;
 }
 
-static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
-{
-}
-
 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
 {
 }
-- 
2.54.0