[PATCH bpf-next 2/2] selftests/bpf: Add bpf_call_rcu tests for hash and LRU hash maps

[Puranjay Mohan <puranjay@kernel.org>]

Each of these checks the element a callback was armed on is still the one
it runs on, by the value it sees.

hash_delete arms, deletes, then inserts another key, which would be handed
the freed element straight back off the freelist. hash_np does the same on
a BPF_F_NO_PREALLOC map, which frees to bpf_mem_alloc instead.
hash_replace replaces the key twice, which is how a preallocated map
recycles through its per-CPU spare. reuse arms an element that has already
been deleted and recycled. lru_evict floods a two-entry map and checks the
armed element was not evicted. hash_chain checks a re-arm from the callback
of a deleted element is refused.

stress hammers arm and delete on overlapping keys from several CPUs and
checks every arm ran.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
---
 .../selftests/bpf/prog_tests/call_rcu.c       | 254 +++++++++++++++-
 tools/testing/selftests/bpf/progs/call_rcu.c  | 274 ++++++++++++++++++
 2 files changed, 526 insertions(+), 2 deletions(-)

diff --git a/tools/testing/selftests/bpf/prog_tests/call_rcu.c b/tools/testing/selftests/bpf/prog_tests/call_rcu.c
index e4d271d43d878..b60220cdf318c 100644
--- a/tools/testing/selftests/bpf/prog_tests/call_rcu.c
+++ b/tools/testing/selftests/bpf/prog_tests/call_rcu.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
 #include <test_progs.h>
+#include <pthread.h>
 #include "call_rcu.skel.h"
 #include "call_rcu_fail.skel.h"
 
@@ -163,6 +164,158 @@ static void test_call_rcu_teardown(void)
 	call_rcu__destroy(skel);
 }
 
+/*
+ * A hash element armed and then deleted: the element must stay alive until the
+ * callback has run, and the callback must still see its value.
+ */
+static void test_call_rcu_hash_delete(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	struct call_rcu *skel;
+	int i;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+		return;
+
+	if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_hash_then_delete),
+					      &opts), "test_run"))
+		goto out;
+	if (!ASSERT_EQ(opts.retval, 0, "retval") ||
+	    !ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"))
+		goto out;
+
+	kern_sync_rcu();
+	for (i = 0; i < 3000; i++) {
+		if (__atomic_load_n(&skel->bss->hash_callbacks, __ATOMIC_ACQUIRE) >= 1)
+			break;
+		usleep(10000);
+	}
+
+	ASSERT_EQ(skel->bss->hash_callbacks, 1, "hash_callbacks");
+	ASSERT_EQ(skel->bss->hash_cb_val, 0xbadc0de, "hash_cb_val");
+out:
+	call_rcu__destroy(skel);
+}
+
+/* A hash callback which re-arms, with the element deleted mid-chain. */
+static void test_call_rcu_hash_chain(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	struct call_rcu *skel;
+	int i;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+		return;
+
+	skel->bss->hash_chain = 3;
+	if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_hash_then_delete),
+					      &opts), "test_run") ||
+	    !ASSERT_EQ(opts.retval, 0, "retval"))
+		goto out;
+
+	kern_sync_rcu();
+	for (i = 0; i < 3000; i++) {
+		if (__atomic_load_n(&skel->bss->hash_callbacks, __ATOMIC_ACQUIRE) >= 1)
+			break;
+		usleep(10000);
+	}
+
+	/*
+	 * The delete lands before the first callback, so the chain is cut:
+	 * re-arming a dead head fails and the element is released once.
+	 */
+	ASSERT_EQ(skel->bss->hash_chain_err, -EBUSY, "chain_refused");
+
+	/* A re-arm that slipped through would land within a grace period. */
+	kern_sync_rcu();
+	usleep(100000);
+	ASSERT_EQ(skel->bss->hash_callbacks, 1, "hash_callbacks");
+out:
+	call_rcu__destroy(skel);
+}
+
+/* An LRU element evicted under pressure while its callback is queued. */
+static void test_call_rcu_lru_evict(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	struct call_rcu *skel;
+	int i;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+		return;
+
+	if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_lru_then_evict),
+					      &opts), "test_run") ||
+	    !ASSERT_EQ(opts.retval, 0, "retval") ||
+	    !ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"))
+		goto out;
+
+	kern_sync_rcu();
+	for (i = 0; i < 3000; i++) {
+		if (__atomic_load_n(&skel->bss->lru_callbacks, __ATOMIC_ACQUIRE) >= 1)
+			break;
+		usleep(10000);
+	}
+
+	ASSERT_EQ(skel->bss->lru_still_present, 1, "lru_still_present");
+	ASSERT_EQ(skel->bss->lru_callbacks, 1, "lru_callbacks");
+	ASSERT_EQ(skel->bss->lru_cb_val, 0xfeedface, "lru_cb_val");
+out:
+	call_rcu__destroy(skel);
+}
+
+static void *stress_thread(void *arg)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	int i, err, prog_fd = *(int *)arg;
+
+	for (i = 0; i < 200; i++) {
+		err = bpf_prog_test_run_opts(prog_fd, &opts);
+		if (!ASSERT_OK(err, "test_run"))
+			break;
+	}
+
+	return NULL;
+}
+
+/* Concurrent arm and delete on overlapping keys from several CPUs. */
+static void test_call_rcu_stress(void)
+{
+	pthread_t thread[8];
+	struct call_rcu *skel;
+	int i, err, prog_fd;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+		return;
+
+	prog_fd = bpf_program__fd(skel->progs.stress_hash);
+	for (i = 0; i < ARRAY_SIZE(thread); i++) {
+		err = pthread_create(&thread[i], NULL, stress_thread, &prog_fd);
+		if (!ASSERT_OK(err, "pthread_create"))
+			break;
+	}
+	while (i--)
+		pthread_join(thread[i], NULL);
+
+	for (i = 0; i < 3000; i++) {
+		if (__atomic_load_n(&skel->bss->stress_callbacks, __ATOMIC_ACQUIRE) >=
+		    skel->bss->stress_arms)
+			break;
+		kern_sync_rcu();
+		usleep(10000);
+	}
+
+	/* Every arm must have run, and the run must not have stalled early. */
+	ASSERT_GT(skel->bss->stress_arms, 1000, "stress_arms");
+	ASSERT_EQ(skel->bss->stress_callbacks, skel->bss->stress_arms, "stress_callbacks");
+
+	call_rcu__destroy(skel);
+}
+
 static void test_call_rcu_bad_map(void)
 {
 	LIBBPF_OPTS(bpf_map_create_opts, opts);
@@ -177,9 +330,9 @@ static void test_call_rcu_bad_map(void)
 	opts.btf_key_type_id = bpf_map__btf_key_type_id(skel->maps.arr);
 	opts.btf_value_type_id = bpf_map__btf_value_type_id(skel->maps.arr);
 
-	fd = bpf_map_create(BPF_MAP_TYPE_HASH, "rcu_hash", sizeof(__u32),
+	fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_HASH, "rcu_pcpu", sizeof(__u32),
 			    bpf_map__value_size(skel->maps.arr), 1, &opts);
-	ASSERT_EQ(fd, -EOPNOTSUPP, "hash_rejected");
+	ASSERT_EQ(fd, -EOPNOTSUPP, "percpu_hash_rejected");
 	if (fd >= 0)
 		close(fd);
 
@@ -253,6 +406,89 @@ static void test_call_rcu_two_heads(void)
 	call_rcu__destroy(skel);
 }
 
+static void test_call_rcu_hash_replace(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	struct call_rcu *skel;
+	int i;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel"))
+		return;
+
+	if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_then_replace),
+					      &opts), "run"))
+		goto out;
+	if (!ASSERT_EQ(opts.retval, 0, "retval"))
+		goto out;
+	ASSERT_OK(skel->bss->arm_err, "arm_err");
+
+	kern_sync_rcu();
+	for (i = 0; i < 3000; i++) {
+		if (__atomic_load_n(&skel->bss->hash_callbacks, __ATOMIC_ACQUIRE) >= 1)
+			break;
+		usleep(10000);
+	}
+	if (!ASSERT_EQ(skel->bss->hash_callbacks, 1, "hash_callbacks"))
+		goto out;
+
+	/*
+	 * The value the callback saw tells us which element it ran on: the one
+	 * that was armed, not one handed back out to a later update.
+	 */
+	ASSERT_EQ(skel->bss->hash_cb_val, 0xbadc0de, "hash_cb_val");
+out:
+	call_rcu__destroy(skel);
+}
+
+static void test_call_rcu_reuse(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	struct call_rcu *skel;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel"))
+		return;
+	if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_after_reuse), &opts),
+		       "run"))
+		goto out;
+	ASSERT_EQ(opts.retval, 0, "retval");
+	ASSERT_OK(skel->bss->reuse_arm2, "arm_after_reuse");
+out:
+	call_rcu__destroy(skel);
+}
+
+static void test_call_rcu_hash_np(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	struct call_rcu *skel;
+	int i;
+
+	skel = call_rcu__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel"))
+		return;
+
+	if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_delete_reinsert),
+					      &opts), "run"))
+		goto out;
+	if (!ASSERT_EQ(opts.retval, 0, "retval"))
+		goto out;
+	ASSERT_OK(skel->bss->arm_err, "arm_err");
+
+	kern_sync_rcu();
+	for (i = 0; i < 3000; i++) {
+		if (__atomic_load_n(&skel->bss->hash_callbacks, __ATOMIC_ACQUIRE) >= 1)
+			break;
+		usleep(10000);
+	}
+	if (!ASSERT_EQ(skel->bss->hash_callbacks, 1, "hash_callbacks"))
+		goto out;
+
+	ASSERT_EQ(skel->bss->hash_cb_val, 0xbadc0de, "hash_cb_val");
+out:
+	call_rcu__destroy(skel);
+}
+
 void test_call_rcu(void)
 {
 	if (test__start_subtest("run"))
@@ -265,6 +501,20 @@ void test_call_rcu(void)
 		test_call_rcu_two_heads();
 	if (test__start_subtest("teardown"))
 		test_call_rcu_teardown();
+	if (test__start_subtest("hash_delete"))
+		test_call_rcu_hash_delete();
+	if (test__start_subtest("hash_chain"))
+		test_call_rcu_hash_chain();
+	if (test__start_subtest("lru_evict"))
+		test_call_rcu_lru_evict();
+	if (test__start_subtest("hash_replace"))
+		test_call_rcu_hash_replace();
+	if (test__start_subtest("hash_np"))
+		test_call_rcu_hash_np();
+	if (test__start_subtest("reuse"))
+		test_call_rcu_reuse();
+	if (test__start_subtest("stress"))
+		test_call_rcu_stress();
 	if (test__start_subtest("bad_map"))
 		test_call_rcu_bad_map();
 	if (test__start_subtest("iter"))
diff --git a/tools/testing/selftests/bpf/progs/call_rcu.c b/tools/testing/selftests/bpf/progs/call_rcu.c
index 505008647fd6f..f3ad927298005 100644
--- a/tools/testing/selftests/bpf/progs/call_rcu.c
+++ b/tools/testing/selftests/bpf/progs/call_rcu.c
@@ -20,6 +20,31 @@ struct {
 	__type(value, struct elem);
 } arr SEC(".maps");
 
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 16384);
+	__type(key, __u32);
+	__type(value, struct elem);
+} hash SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 8);
+	__uint(map_flags, BPF_F_NO_PREALLOC);
+	__type(key, __u32);
+	__type(value, struct elem);
+} hash_np SEC(".maps");
+
+int hash_cb_val;
+int hash_callbacks;
+int hash_chain;		/* set by userspace: re-arms from the hash callback */
+int hash_chain_err;
+int lru_callbacks;
+int lru_still_present;
+int lru_cb_val;
+int stress_callbacks;
+int stress_arms;
+
 __u32 cb_key;
 __u64 cb_keys;
 __u64 cb_val;
@@ -103,6 +128,255 @@ int arm_trace(void *ctx)
 	return 0;
 }
 
+static int hash_reclaim(struct bpf_map *map, void *key, void *value)
+{
+	struct elem *e = value;
+
+	hash_cb_val = e->val;
+
+	if (hash_chain > 0) {
+		hash_chain--;
+		hash_chain_err = bpf_call_rcu(&e->rh, &hash, hash_reclaim);
+	}
+
+	__sync_fetch_and_add(&hash_callbacks, 1);
+	return 0;
+}
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(max_entries, 2);
+	__type(key, __u32);
+	__type(value, struct elem);
+} lru SEC(".maps");
+
+static int lru_reclaim(struct bpf_map *map, void *key, void *value)
+{
+	struct elem *e = value;
+
+	lru_cb_val = e->val;
+	__sync_fetch_and_add(&lru_callbacks, 1);
+	return 0;
+}
+
+/*
+ * Arm an LRU element, then push enough traffic through the map to evict it
+ * while the callback is still queued.
+ */
+SEC("syscall")
+int arm_lru_then_evict(void *ctx)
+{
+	struct elem init = {};
+	__u32 key = 100;
+	struct elem *e;
+	int i;
+
+	init.val = 0xfeedface;
+	if (bpf_map_update_elem(&lru, &key, &init, BPF_ANY))
+		return 1;
+
+	e = bpf_map_lookup_elem(&lru, &key);
+	if (!e)
+		return 2;
+
+	arm_err = bpf_call_rcu(&e->rh, &lru, lru_reclaim);
+	if (arm_err)
+		return 3;
+
+	/* max_entries is 2, so this would evict the armed element. */
+	init.val = 0;
+	bpf_for(i, 0, 64) {
+		__u32 k = 200 + i;
+
+		bpf_map_update_elem(&lru, &k, &init, BPF_ANY);
+	}
+
+	/* The callback still needs it, so the eviction must have been declined. */
+	lru_still_present = !!bpf_map_lookup_elem(&lru, &key);
+	return 0;
+}
+
+static int stress_reclaim(struct bpf_map *map, void *key, void *value)
+{
+	__sync_fetch_and_add(&stress_callbacks, 1);
+	return 0;
+}
+
+/*
+ * Hammer arm and delete on overlapping keys from several CPUs at once, to
+ * exercise the window where a delete lands while a callback is running.
+ */
+SEC("syscall")
+int stress_hash(void *ctx)
+{
+	struct elem init = {};
+	struct elem *e;
+	__u32 key;
+	int i;
+
+	init.val = 0xabcd1234;
+
+	bpf_for(i, 0, 16) {
+		key = bpf_get_prandom_u32() & 0xff;
+
+		if (bpf_map_update_elem(&hash, &key, &init, BPF_ANY))
+			continue;
+
+		e = bpf_map_lookup_elem(&hash, &key);
+		if (!e)
+			continue;
+
+		if (!bpf_call_rcu(&e->rh, &hash, stress_reclaim))
+			__sync_fetch_and_add(&stress_arms, 1);
+		bpf_map_delete_elem(&hash, &key);
+	}
+
+	return 0;
+}
+
+/*
+ * Arm a hash element, then replace its key twice. On a preallocated map the
+ * first replace would stash the old element in this CPU's spare and the second
+ * would hand that same element straight back out, so a callback that still
+ * needs it must take it out of circulation instead.
+ */
+SEC("syscall")
+int arm_then_replace(void *ctx)
+{
+	struct elem init = {};
+	__u32 key = 7;
+	struct elem *e;
+
+	init.val = 0xbadc0de;
+	if (bpf_map_update_elem(&hash, &key, &init, BPF_ANY))
+		return 1;
+
+	e = bpf_map_lookup_elem(&hash, &key);
+	if (!e)
+		return 2;
+
+	arm_err = bpf_call_rcu(&e->rh, &hash, hash_reclaim);
+	if (arm_err)
+		return 3;
+
+	init.val = 0xaaaa;
+	if (bpf_map_update_elem(&hash, &key, &init, BPF_ANY))
+		return 4;
+	init.val = 0xbbbb;
+	if (bpf_map_update_elem(&hash, &key, &init, BPF_ANY))
+		return 5;
+
+	return 0;
+}
+
+static int hash_np_reclaim(struct bpf_map *map, void *key, void *value)
+{
+	struct elem *e = value;
+
+	hash_cb_val = e->val;
+	__sync_fetch_and_add(&hash_callbacks, 1);
+	return 0;
+}
+
+/*
+ * Same handover on a map that frees elements to bpf_mem_alloc rather than a
+ * freelist: arm, delete, then insert the key again. The callback must still
+ * see the value it was armed on, not whatever the reinsert wrote.
+ */
+SEC("syscall")
+int arm_delete_reinsert(void *ctx)
+{
+	struct elem init = {};
+	__u32 key = 3;
+	struct elem *e;
+
+	init.val = 0xbadc0de;
+	if (bpf_map_update_elem(&hash_np, &key, &init, BPF_ANY))
+		return 1;
+
+	e = bpf_map_lookup_elem(&hash_np, &key);
+	if (!e)
+		return 2;
+
+	arm_err = bpf_call_rcu(&e->rh, &hash_np, hash_np_reclaim);
+	if (arm_err)
+		return 3;
+
+	if (bpf_map_delete_elem(&hash_np, &key))
+		return 4;
+
+	init.val = 0x1234;
+	if (bpf_map_update_elem(&hash_np, &key, &init, BPF_ANY))
+		return 5;
+
+	return 0;
+}
+
+/* Arm a hash element, then delete it before the callback can run. */
+SEC("syscall")
+int arm_hash_then_delete(void *ctx)
+{
+	struct elem init = {};
+	__u32 key = 7;
+	struct elem *e;
+
+	init.val = 0xbadc0de;
+	if (bpf_map_update_elem(&hash, &key, &init, BPF_ANY))
+		return 1;
+
+	e = bpf_map_lookup_elem(&hash, &key);
+	if (!e)
+		return 2;
+
+	arm_err = bpf_call_rcu(&e->rh, &hash, hash_reclaim);
+	if (arm_err)
+		return 3;
+
+	if (bpf_map_delete_elem(&hash, &key))
+		return 4;
+
+	/* The freelist is LIFO, so this would hand the same element back out. */
+	key = 8;
+	init.val = 0x5678;
+	return bpf_map_update_elem(&hash, &key, &init, BPF_ANY) ? 5 : 0;
+}
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, struct elem);
+} one SEC(".maps");
+
+int reuse_arm1, reuse_arm2;
+
+static int reuse_cb(struct bpf_map *map, void *key, void *value)
+{
+	return 0;
+}
+
+/* Insert, delete without ever arming, reinsert, then arm the recycled element. */
+SEC("syscall")
+int arm_after_reuse(void *ctx)
+{
+	struct elem init = {};
+	__u32 key = 1;
+	struct elem *e;
+
+	if (bpf_map_update_elem(&one, &key, &init, BPF_ANY))
+		return 1;
+	/* No callback is ever armed on this element before the delete. */
+	if (bpf_map_delete_elem(&one, &key))
+		return 3;
+	if (bpf_map_update_elem(&one, &key, &init, BPF_ANY))
+		return 4;
+	e = bpf_map_lookup_elem(&one, &key);
+	if (!e)
+		return 5;
+	reuse_arm2 = bpf_call_rcu(&e->rh, &one, reuse_cb);
+	return 0;
+}
+
 SEC("iter/bpf_map_elem")
 int dump(struct bpf_iter__bpf_map_elem *ctx)
 {
-- 
2.53.0-Meta