[PATCHv2 bpf-next 2/2] bpf: Disable private stacks for untagged kprobe programs

[Jiri Olsa <jolsa@kernel.org>]

Uprobe programs can be preempted and re-entered on the same CPU. Since
the private stack is shared per program and CPU, another invocation can
overwrite the preempted invocation's stack.

Programs without a multi or session attach type may be standard kprobes
or uprobes. Disable private stacks for them, and retain private stacks
only for kprobe multi and session programs, which have attach types that
identify them as kprobes.

Update verifier tests that rely on private stacks to use kprobe.multi,
and add coverage that untagged kprobe and uprobe programs do not use
private stacks.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 kernel/bpf/verifier.c                         |  9 +++++
 .../bpf/progs/verifier_large_stack.c          |  2 +-
 .../bpf/progs/verifier_private_stack.c        | 38 ++++++++++++++++++-
 3 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index b7329d164771..1b58a1796a2e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5749,6 +5749,15 @@ static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
 	 */
 	switch (prog->type) {
 	case BPF_PROG_TYPE_KPROBE:
+		/*
+		 * Uprobes are preemptible and lack recursion protection. Classic
+		 * kprobe and uprobe progs have no expected attach type, so enable
+		 * private stacks only for kprobe.multi and kprobe.session progs.
+		 */
+		if (prog->expected_attach_type != BPF_TRACE_KPROBE_MULTI &&
+		    prog->expected_attach_type != BPF_TRACE_KPROBE_SESSION)
+			return NO_PRIV_STACK;
+		fallthrough;
 	case BPF_PROG_TYPE_TRACEPOINT:
 	case BPF_PROG_TYPE_PERF_EVENT:
 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
diff --git a/tools/testing/selftests/bpf/progs/verifier_large_stack.c b/tools/testing/selftests/bpf/progs/verifier_large_stack.c
index 7e4f5e3463a2..cbabb41d973c 100644
--- a/tools/testing/selftests/bpf/progs/verifier_large_stack.c
+++ b/tools/testing/selftests/bpf/progs/verifier_large_stack.c
@@ -243,7 +243,7 @@ static void priv_stack_frame_2048(void)
 "	::: __clobber_all);
 }
 
-SEC("kprobe")
+SEC("kprobe.multi")
 __description("private stack: two frames of 2048 bytes")
 __load_if_large_stack()
 __arch_x86_64
diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c
index aa15361bcca5..0ce07bf6b566 100644
--- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c
+++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c
@@ -22,7 +22,7 @@ struct {
 	__type(value, struct elem);
 } array SEC(".maps");
 
-SEC("kprobe")
+SEC("kprobe.multi")
 __description("Private stack, single prog")
 __success
 __arch_x86_64
@@ -110,6 +110,40 @@ int no_private_stack_stack_args(void)
 }
 #endif
 
+SEC("kprobe")
+__description("No private stack, untagged kprobe")
+__success
+__arch_x86_64
+__jited("	subq	$0x100, %rsp")
+__arch_arm64
+__jited("	sub	sp, sp, #0x100")
+__naked void no_private_stack_kprobe(void)
+{
+	asm volatile ("			\
+	r1 = 42;			\
+	*(u64 *)(r10 - 256) = r1;	\
+	r0 = 0;				\
+	exit;				\
+"	::: __clobber_all);
+}
+
+SEC("uprobe")
+__description("No private stack, uprobe")
+__success
+__arch_x86_64
+__jited("	subq	$0x100, %rsp")
+__arch_arm64
+__jited("	sub	sp, sp, #0x100")
+__naked void no_private_stack_uprobe(void)
+{
+	asm volatile ("			\
+	r1 = 42;			\
+	*(u64 *)(r10 - 256) = r1;	\
+	r0 = 0;				\
+	exit;				\
+"	::: __clobber_all);
+}
+
 __used
 __naked static void cumulative_stack_depth_subprog(void)
 {
@@ -123,7 +157,7 @@ __naked static void cumulative_stack_depth_subprog(void)
 	: __clobber_all);
 }
 
-SEC("kprobe")
+SEC("kprobe.multi")
 __description("Private stack, subtree > MAX_BPF_STACK")
 __success
 __log_level(4) __msg("stack depth max 512")
-- 
2.43.0