* [PATCH bpf-next 1/2] srcu: Add lock guard for srcu_fast_updown flavor
2026-07-06 17:27 [PATCH bpf-next 0/2] uprobes: Switch uretprobes_srcu to SRCU-fast-updown Puranjay Mohan
@ 2026-07-06 17:27 ` Puranjay Mohan
2026-07-06 17:36 ` Paul E. McKenney
2026-07-06 17:27 ` [PATCH bpf-next 2/2] uprobes: Switch uretprobes_srcu to SRCU-fast-updown Puranjay Mohan
1 sibling, 1 reply; 4+ messages in thread
From: Puranjay Mohan @ 2026-07-06 17:27 UTC (permalink / raw)
To: Lai Jiangshan, Paul E. McKenney, Josh Triplett, Masami Hiramatsu,
Oleg Nesterov, Peter Zijlstra, Ingo Molnar,
Arnaldo Carvalho de Melo, Namhyung Kim, Alexei Starovoitov,
Andrii Nakryiko
Cc: Puranjay Mohan, Steven Rostedt, Mathieu Desnoyers, Mark Rutland,
Alexander Shishkin, Jiri Olsa, Ian Rogers, Adrian Hunter,
James Clark, rcu, linux-kernel, linux-trace-kernel,
linux-perf-users, bpf
Add a guard(srcu_fast_updown) definition for scoped
SRCU-fast-updown read-side critical sections, following the
existing pattern of guard(srcu) and guard(srcu_fast).
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
include/linux/srcu.h | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index a54ce9e808b92..72c86d3b23f2e 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -638,4 +638,11 @@ DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct,
DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
#define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T)
+DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct,
+ _T->scp = srcu_read_lock_fast_updown(_T->lock),
+ srcu_read_unlock_fast_updown(_T->lock, _T->scp),
+ struct srcu_ctr __percpu *scp)
+DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T))
+#define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T)
+
#endif
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH bpf-next 2/2] uprobes: Switch uretprobes_srcu to SRCU-fast-updown
2026-07-06 17:27 [PATCH bpf-next 0/2] uprobes: Switch uretprobes_srcu to SRCU-fast-updown Puranjay Mohan
2026-07-06 17:27 ` [PATCH bpf-next 1/2] srcu: Add lock guard for srcu_fast_updown flavor Puranjay Mohan
@ 2026-07-06 17:27 ` Puranjay Mohan
1 sibling, 0 replies; 4+ messages in thread
From: Puranjay Mohan @ 2026-07-06 17:27 UTC (permalink / raw)
To: Lai Jiangshan, Paul E. McKenney, Josh Triplett, Masami Hiramatsu,
Oleg Nesterov, Peter Zijlstra, Ingo Molnar,
Arnaldo Carvalho de Melo, Namhyung Kim, Alexei Starovoitov,
Andrii Nakryiko
Cc: Puranjay Mohan, Steven Rostedt, Mathieu Desnoyers, Mark Rutland,
Alexander Shishkin, Jiri Olsa, Ian Rogers, Adrian Hunter,
James Clark, rcu, linux-kernel, linux-trace-kernel,
linux-perf-users, bpf
uretprobes_srcu currently uses normal SRCU, which issues
two smp_mb() per read lock/unlock pair. This overhead is
paid on every uretprobe hit.
Switch to SRCU-fast-updown, which eliminates the per-reader
memory barriers by moving the ordering cost to the
grace-period side (synchronize_rcu() instead of smp_mb()).
This is acceptable because grace periods (uprobe
unregistration) are infrequent compared to reader-side
uretprobe hits.
The updown flavor is required because the SRCU read lock is
taken in prepare_uretprobe() when a return instance is
created and is held until that return instance is finalized.
The traced thread returns to user space in between, so the
lock is inherently released in a different context from
where it was acquired: on the normal return path via
uprobe_handle_trampoline() -> hprobe_finalize(), or from
ri_timer() (expiry) or dup_utask() (fork) via
hprobe_expire(). srcu_down_read_fast() / srcu_up_read_fast()
are designed for this acquire-here / release-elsewhere
pattern and, unlike the same-context srcu_read_lock_fast()
variant, do not carry the lockdep read-side tracking that
would warn on it.
The short, same-context SRCU sections in ri_timer() and
dup_utask() (which guard the uprobe against reuse across the
hprobe_expire() cmpxchg) instead use guard(srcu_fast_updown)
for proper lockdep coverage.
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
include/linux/uprobes.h | 5 +++--
kernel/events/uprobes.c | 29 +++++++++++++++++------------
2 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index f548fea2adec8..f3b07753c2f3d 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -25,6 +25,7 @@ struct mm_struct;
struct inode;
struct notifier_block;
struct page;
+struct srcu_ctr;
/*
* Allowed return values from uprobe consumer's handler callback
@@ -106,7 +107,7 @@ enum hprobe_state {
* underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an
* internal marker and is handled transparently by hprobe_fetch() helper.
*
- * When uprobe is SRCU-protected, we also record srcu_idx value, necessary for
+ * When uprobe is SRCU-protected, we also record srcu_scp value, necessary for
* SRCU unlocking.
*
* See hprobe_expire() and hprobe_fetch() for details of race-free uprobe
@@ -115,7 +116,7 @@ enum hprobe_state {
*/
struct hprobe {
enum hprobe_state state;
- int srcu_idx;
+ struct srcu_ctr __percpu *srcu_scp;
struct uprobe *uprobe;
};
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 4084e926e2844..afa491b0bd3f9 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -54,7 +54,7 @@ static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
/* Covers return_instance's uprobe lifetime. */
-DEFINE_STATIC_SRCU(uretprobes_srcu);
+DEFINE_STATIC_SRCU_FAST_UPDOWN(uretprobes_srcu);
/* Have a copy of original instruction */
#define UPROBE_COPY_INSN 0
@@ -707,12 +707,13 @@ static void put_uprobe(struct uprobe *uprobe)
}
/* Initialize hprobe as SRCU-protected "leased" uprobe */
-static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx)
+static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe,
+ struct srcu_ctr __percpu *srcu_scp)
{
WARN_ON(!uprobe);
hprobe->state = HPROBE_LEASED;
hprobe->uprobe = uprobe;
- hprobe->srcu_idx = srcu_idx;
+ hprobe->srcu_scp = srcu_scp;
}
/* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */
@@ -720,7 +721,7 @@ static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe)
{
hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE;
hprobe->uprobe = uprobe;
- hprobe->srcu_idx = -1;
+ hprobe->srcu_scp = NULL;
}
/*
@@ -757,7 +758,7 @@ static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate)
{
switch (hstate) {
case HPROBE_LEASED:
- __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
+ srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp);
break;
case HPROBE_STABLE:
put_uprobe(hprobe->uprobe);
@@ -829,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get)
*/
if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) {
/* We won the race, we are the ones to unlock SRCU */
- __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
+ srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp);
return get ? get_uprobe(uprobe) : uprobe;
}
@@ -2045,7 +2046,7 @@ static void ri_timer(struct timer_list *timer)
struct return_instance *ri;
/* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */
- guard(srcu)(&uretprobes_srcu);
+ guard(srcu_fast_updown)(&uretprobes_srcu);
/* RCU protects return_instance from freeing. */
guard(rcu)();
@@ -2142,7 +2143,7 @@ static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
t->utask = n_utask;
/* protect uprobes from freeing, we'll need try_get_uprobe() them */
- guard(srcu)(&uretprobes_srcu);
+ guard(srcu_fast_updown)(&uretprobes_srcu);
p = &n_utask->return_instances;
for (o = o_utask->return_instances; o; o = o->next) {
@@ -2254,8 +2255,8 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
{
struct uprobe_task *utask = current->utask;
unsigned long orig_ret_vaddr, trampoline_vaddr;
+ struct srcu_ctr __percpu *srcu_scp;
bool chained;
- int srcu_idx;
if (!get_xol_area())
goto free;
@@ -2293,8 +2294,12 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
}
- /* __srcu_read_lock() because SRCU lock survives switch to user space */
- srcu_idx = __srcu_read_lock(&uretprobes_srcu);
+ /*
+ * Use srcu_down_read_fast() because the SRCU lock survives a switch to
+ * user space and can be unlocked from a different context by ri_timer()
+ * or dup_utask().
+ */
+ srcu_scp = srcu_down_read_fast(&uretprobes_srcu);
ri->func = instruction_pointer(regs);
ri->stack = user_stack_pointer(regs);
@@ -2303,7 +2308,7 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs,
utask->depth++;
- hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx);
+ hprobe_init_leased(&ri->hprobe, uprobe, srcu_scp);
ri->next = utask->return_instances;
rcu_assign_pointer(utask->return_instances, ri);
--
2.53.0-Meta
^ permalink raw reply related [flat|nested] 4+ messages in thread