[PATCH v2 09/12] uprobes: batch uprobes_treelock during registration

Andrii Nakryiko
2024-07-01T22:40:08Z

Now that we have a good separate of each registration step, take
uprobes_treelock just once for relevant registration step, and then
process all relevant uprobes in one go.

Even if writer lock introduces a relatively large delay (as might happen
with per-CPU RW semaphore), this will keep overall batch attachment
reasonably fast.

We teach put_uprobe(), though __put_uprobe() helper, to optionally take
or not uprobes_treelock, to accommodate this pattern.

With these changes we don't need insert_uprobe() operation that
unconditionally takes uprobes_treelock, so get rid of it, leaving only
lower-level __insert_uprobe() helper.

Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
---
kernel/events/uprobes.c | 45 +++++++++++++++++++++--------------------
1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 128677ffe662..ced85284bbf4 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -665,7 +665,7 @@ static void uprobe_free_rcu(struct rcu_head *rcu)
kfree(uprobe);
}

-static void put_uprobe(struct uprobe *uprobe)
+static void __put_uprobe(struct uprobe *uprobe, bool tree_locked)
{
s64 v;

@@ -683,7 +683,8 @@ static void put_uprobe(struct uprobe *uprobe)
if (unlikely((u32)v == 0)) {
bool destroy;

- write_lock(&uprobes_treelock);
+ if (!tree_locked)
+ write_lock(&uprobes_treelock);
/*
* We might race with find_uprobe()->__get_uprobe() executed
* from inside read-locked uprobes_treelock, which can bump
@@ -706,7 +707,8 @@ static void put_uprobe(struct uprobe *uprobe)
destroy = atomic64_read(&uprobe->ref) == v;
if (destroy && uprobe_is_active(uprobe))
rb_erase(&uprobe->rb_node, &uprobes_tree);
- write_unlock(&uprobes_treelock);
+ if (!tree_locked)
+ write_unlock(&uprobes_treelock);

/*
* Beyond here we don't need RCU protection, we are either the
@@ -745,6 +747,11 @@ static void put_uprobe(struct uprobe *uprobe)
rcu_read_unlock_trace();
}

+static void put_uprobe(struct uprobe *uprobe)
+{
+ __put_uprobe(uprobe, false);
+}
+
static __always_inline
int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
const struct uprobe *r)
@@ -844,21 +851,6 @@ static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
return u;
}

-/*
- * Acquire uprobes_treelock and insert uprobe into uprobes_tree
- * (or reuse existing one, see __insert_uprobe() comments above).
- */
-static struct uprobe *insert_uprobe(struct uprobe *uprobe)
-{
- struct uprobe *u;
-
- write_lock(&uprobes_treelock);
- u = __insert_uprobe(uprobe);
- write_unlock(&uprobes_treelock);
-
- return u;
-}
-
static void
ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
{
@@ -1318,6 +1310,8 @@ int uprobe_register_batch(struct inode *inode, int cnt,
uc->uprobe = uprobe;
}

+ ret = 0;
+ write_lock(&uprobes_treelock);
for (i = 0; i < cnt; i++) {
struct uprobe *cur_uprobe;

@@ -1325,19 +1319,24 @@ int uprobe_register_batch(struct inode *inode, int cnt,
uprobe = uc->uprobe;

/* add to uprobes_tree, sorted on inode:offset */
- cur_uprobe = insert_uprobe(uprobe);
+ cur_uprobe = __insert_uprobe(uprobe);
/* a uprobe exists for this inode:offset combination */
if (cur_uprobe != uprobe) {
if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
ref_ctr_mismatch_warn(cur_uprobe, uprobe);
- put_uprobe(cur_uprobe);
+
+ __put_uprobe(cur_uprobe, true);
ret = -EINVAL;
- goto cleanup_uprobes;
+ goto unlock_treelock;
}
kfree(uprobe);
uc->uprobe = cur_uprobe;
}
}
+unlock_treelock:
+ write_unlock(&uprobes_treelock);
+ if (ret)
+ goto cleanup_uprobes;

for (i = 0; i < cnt; i++) {
uc = get_uprobe_consumer(i, ctx);
@@ -1367,13 +1366,15 @@ int uprobe_register_batch(struct inode *inode, int cnt,
}
cleanup_uprobes:
/* put all the successfully allocated/reused uprobes */
+ write_lock(&uprobes_treelock);
for (i = 0; i < cnt; i++) {
uc = get_uprobe_consumer(i, ctx);

if (uc->uprobe)
- put_uprobe(uc->uprobe);
+ __put_uprobe(uc->uprobe, true);
uc->uprobe = NULL;
}
+ write_unlock(&uprobes_treelock);
return ret;
}

--
2.43.0