From: Emil Tsalapatis <emil@etsalapatis.com>
To: bpf@vger.kernel.org
Cc: ast@kernel.org, andrii@kernel.org, eddyz87@gmail.com,
memxor@gmail.com, daniel@iogearbox.net,
Emil Tsalapatis <emil@etsalapatis.com>
Subject: [PATCH bpf-next v1 4/7] selftests/bpf: libarena: Handle NMI reentrancy
Date: Tue, 20 Oct 2026 18:18:57 +0000 [thread overview]
Message-ID: <20261020181900.6755-5-emil@etsalapatis.com> (raw)
In-Reply-To: <20261020181900.6755-1-emil@etsalapatis.com>
An NMI can interrupt a cache operation and access the same per-CPU array.
BPF programs also only disable migration, so another task can preempt a
cache operation and run on the same CPU. Allowing the nested operation to
proceed would let it overwrite the interrupted operation's state.
Track active cache operations per CPU and reject nested access, following
the model used by the main BPF memory allocator. Claim the CPU with a
cmpxchg so that two tasks cannot both observe it as idle. An allocation
from NMI context succeeds only when the local cache can satisfy it without
taking the buddy lock.
When an NMI free cannot use the local cache, link the object onto an
intrusive deferred-free list. A later allocator operation from a safe
context drains that list while holding the buddy lock.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
.../bpf/libarena/include/libarena/buddy.h | 3 +
.../libarena/include/libarena/buddy_cache.h | 6 +
.../bpf/libarena/selftests/test_buddy.bpf.c | 2 +
.../selftests/bpf/libarena/src/buddy.bpf.c | 188 +++++++++++++++++-
4 files changed, 189 insertions(+), 10 deletions(-)
diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h b/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h
index 9150a9d22cce..655b3f5624bf 100644
--- a/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h
+++ b/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h
@@ -51,6 +51,8 @@ struct buddy_header {
u32 next_index; /* Same for the next allocation. */
};
+struct buddy_deferred_free;
+
/*
* We bring memory into the allocator 1 MiB at a time.
*/
@@ -72,6 +74,7 @@ struct buddy {
arena_spinlock_t lock; /* Allocator lock */
u64 vaddr; /* Allocation into reserved vaddr */
struct buddy_cache_state cache_state;
+ struct buddy_deferred_free __arena *deferred_free;
};
#ifdef __BPF__
diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/buddy_cache.h b/tools/testing/selftests/bpf/libarena/include/libarena/buddy_cache.h
index 769d966686f5..cad198ac732d 100644
--- a/tools/testing/selftests/bpf/libarena/include/libarena/buddy_cache.h
+++ b/tools/testing/selftests/bpf/libarena/include/libarena/buddy_cache.h
@@ -43,3 +43,9 @@ struct buddy_cache_state {
struct buddy_cache __arena *percpu[BUDDY_CACHE_MAX_CPUS];
bool valid;
};
+
+/* Per-CPU guard against nested or preempting cache operations. */
+
+struct __attribute__((aligned(64))) buddy_cache_active {
+ unsigned int value;
+};
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c
index 1a1856befeae..6d123ec052b0 100644
--- a/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c
@@ -44,6 +44,8 @@ __weak int test_buddy_create(void)
ret = buddy_destroy(&buddy);
if (ret)
return ret;
+ if (buddy.deferred_free)
+ return -EINVAL;
if (buddy.cache_state.valid)
return -EINVAL;
}
diff --git a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c
index ce05e3e58eba..85d670b8762b 100644
--- a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c
+++ b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c
@@ -7,6 +7,7 @@
#include <libarena/buddy_cache.h>
#include <bpf_arena_spin_lock.h>
+#include <bpf_experimental.h>
_Static_assert(BUDDY_CACHE_MAX_CPUS == _Q_MAX_CPUS,
"buddy caches must cover every CPU arena spinlocks support");
@@ -48,6 +49,14 @@ static u64 size_to_order(size_t size);
static void __arena *buddy_alloc_memory(struct buddy __arena *buddy, size_t size,
int order, size_t norder);
static int buddy_free_unlocked(struct buddy __arena *buddy, u64 addr);
+static bool buddy_deferred_free_pending(struct buddy __arena *buddy);
+static int buddy_deferred_free_complete(struct buddy __arena *buddy);
+static struct buddy_cache_active active[BUDDY_CACHE_MAX_CPUS];
+
+struct buddy_cache_ctx {
+ struct buddy_cache __arena *cache;
+ struct buddy_cache_active *active;
+};
#define buddy_lock(buddy, flags) (arena_spin_lock_irqsave(&(buddy)->lock, (flags)))
#define buddy_unlock(buddy, flags) (arena_spin_unlock_irqrestore(&(buddy)->lock, (flags)))
@@ -62,6 +71,31 @@ static int buddy_free_unlocked(struct buddy __arena *buddy, u64 addr);
#define DISABLE_CACHE_IF_ASAN(...)
#endif
+static __always_inline int
+buddy_cache_enter(struct buddy_cache_state __arena *state,
+ struct buddy_cache_ctx *ctx)
+{
+ unsigned int cpu;
+
+ /*
+ * No support for cache operations from contexts that interrupted or
+ * preempted an in-flight operation on the same CPU, just like the
+ * kernel's BPF memory allocator.
+ */
+ cpu = bpf_get_smp_processor_id();
+ ctx->cache = state->percpu[cpu];
+ ctx->active = &active[cpu];
+ if (cmpxchg(&ctx->active->value, 0, 1))
+ return -EAGAIN;
+
+ return 0;
+}
+
+static __always_inline void buddy_cache_exit(struct buddy_cache_ctx *ctx)
+{
+ smp_store_release(&ctx->active->value, 0);
+}
+
static __always_inline void __arena *
buddy_cache_pop(struct buddy_cache __arena *cache, int order)
{
@@ -100,18 +134,21 @@ static __always_inline int buddy_cache_push(struct buddy_cache __arena *cache,
void __arena *buddy_cache_alloc(struct buddy __arena *buddy, size_t order)
{
struct buddy_cache_state __arena *state = &buddy->cache_state;
- struct buddy_cache __arena *cache;
- unsigned int cpu;
+ struct buddy_cache_ctx ctx;
+ void __arena *address = NULL;
DISABLE_CACHE_IF_ASAN(NULL);
if (order >= BUDDY_CACHE_NUM_ORDERS)
return NULL;
- cpu = bpf_get_smp_processor_id();
- cache = state->percpu[cpu];
+ if (buddy_cache_enter(state, &ctx))
+ return NULL;
+
+ address = buddy_cache_pop(ctx.cache, order);
+ buddy_cache_exit(&ctx);
- return buddy_cache_pop(cache, order);
+ return address;
}
/* Retrieve the order of an allocation from the address. */
@@ -129,9 +166,9 @@ static int buddy_cache_get_order(void __arena *addr)
int buddy_cache_free(struct buddy __arena *buddy, void __arena *addr)
{
struct buddy_cache_state __arena *state = &buddy->cache_state;
- struct buddy_cache __arena *cache;
- unsigned int cpu;
+ struct buddy_cache_ctx ctx;
int order;
+ int ret = 0;
DISABLE_CACHE_IF_ASAN(-EINVAL);
@@ -139,10 +176,13 @@ int buddy_cache_free(struct buddy __arena *buddy, void __arena *addr)
if (order >= BUDDY_CACHE_NUM_ORDERS)
return -EINVAL;
- cpu = bpf_get_smp_processor_id();
- cache = state->percpu[cpu];
+ if (buddy_cache_enter(state, &ctx))
+ return -EAGAIN;
+
+ ret = buddy_cache_push(ctx.cache, order, addr);
+ buddy_cache_exit(&ctx);
- return buddy_cache_push(cache, order, addr);
+ return ret;
}
enum {
@@ -707,6 +747,7 @@ __weak int buddy_init(struct buddy __arena *buddy)
if (!asan_ready())
return -EINVAL;
+ buddy->deferred_free = NULL;
/* Reserve enough address space to ensure allocations are aligned. */
ret = buddy_reserve_arena_vaddr(buddy);
@@ -777,6 +818,7 @@ __weak int buddy_destroy(struct buddy __arena *buddy)
/* Clear all fields. */
buddy->first_chunk = NULL;
+ buddy->deferred_free = NULL;
return 0;
}
@@ -926,9 +968,19 @@ static void __arena *buddy_alloc_memory(struct buddy __arena *buddy, size_t size
return NULL;
}
+ if (bpf_in_nmi())
+ return NULL;
+
if (buddy_lock(buddy, flags))
return NULL;
+ if (unlikely(buddy_deferred_free_pending(buddy))) {
+ if (buddy_deferred_free_complete(buddy)) {
+ buddy_unlock(buddy, flags);
+ return NULL;
+ }
+ }
+
address = (u8 __arena *)buddy_alloc_from_existing_chunks(buddy, order, norder);
buddy_unlock(buddy, flags);
if (address)
@@ -1084,6 +1136,98 @@ static int buddy_free_unlocked(struct buddy __arena *buddy, u64 addr)
return 0;
}
+static bool buddy_deferred_free_pending(struct buddy __arena *buddy)
+{
+ return buddy->deferred_free != NULL;
+}
+
+static void buddy_deferred_free_start(struct buddy __arena *buddy, void __arena *addr)
+{
+ void __arena *head;
+ void __arena * __arena *new;
+
+ /*
+ * Make a linked list by chaining the allocations into a linked list.
+ * Possible because allocations are 16 bytes minimum and so fit a pointer.
+ */
+ new = (void __arena * __arena *)addr;
+ asan_unpoison(addr, sizeof(void *));
+ do {
+ head = buddy->deferred_free;
+ *new = head;
+ } while (cmpxchg((void __arena * __arena *)&buddy->deferred_free,
+ head, (void __arena *)new) != head && can_loop);
+}
+
+static void buddy_deferred_free_start_bulk(struct buddy __arena *buddy,
+ void __arena * __arena *addrs, size_t naddrs)
+{
+ void __arena *head, *first, *last, *addr;
+ unsigned int i;
+
+ /* Create a linked list locally. */
+ first = NULL;
+ last = NULL;
+ for (i = zero; i < naddrs && can_loop; i++) {
+ addr = addrs[i];
+ if (!addr)
+ continue;
+
+ asan_unpoison(addr, sizeof(void *));
+
+ if (!first)
+ first = addr;
+ else
+ *(void __arena * __arena *)last = addr;
+
+ last = addr;
+ }
+
+ if (!first)
+ return;
+
+ /*
+ * Make a linked list by chaining the allocations into a linked list.
+ * Possible because allocations are 16 bytes minimum and so fit a pointer.
+ */
+ do {
+ head = buddy->deferred_free;
+ *(void __arena * __arena *)last = head;
+ } while (cmpxchg((void __arena * __arena *)&buddy->deferred_free,
+ head, first) != head && can_loop);
+}
+
+static int buddy_deferred_free_complete(struct buddy __arena *buddy)
+{
+ void __arena *head, *old;
+ int ret = 0, err;
+
+ do {
+ head = buddy->deferred_free;
+ if (!head)
+ break;
+ old = head;
+
+ /*
+ * The cmpxchg is safe because we never pop from the list
+ * without holding the buddy lock. The next pointer of
+ * the element then cannot have been gotten invalidated.
+ */
+
+ } while (cmpxchg((void __arena * __arena *)&buddy->deferred_free,
+ old, NULL) != old && can_loop);
+
+ while (head && can_loop) {
+ old = head;
+ head = *(void __arena * __arena *)head;
+
+ err = buddy_free_unlocked(buddy, (u64)old);
+ ret = ret ?: err;
+ }
+
+ return ret;
+}
+
__weak int buddy_free(struct buddy __arena *buddy, void __arena *addr)
{
unsigned long flags;
@@ -1100,12 +1244,25 @@ __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr)
if (!ret)
return 0;
+ if (bpf_in_nmi()) {
+ buddy_deferred_free_start(buddy, addr);
+ return 0;
+ }
+
ret = buddy_lock(buddy, flags);
if (ret)
return ret;
ret = buddy_free_unlocked(buddy, (u64)addr);
+ /* Drain any frees stranded during an NMI. */
+ if (unlikely(buddy_deferred_free_pending(buddy))) {
+ ret = buddy_deferred_free_complete(buddy);
+ if (ret)
+ goto out_unlock;
+ }
+
+out_unlock:
buddy_unlock(buddy, flags);
return ret;
@@ -1127,6 +1284,11 @@ __weak int buddy_free_bulk(struct buddy __arena *buddy, void __arena * __arena *
if (!addrs)
return -EINVAL;
+ if (bpf_in_nmi()) {
+ buddy_deferred_free_start_bulk(buddy, addrs, naddrs);
+ return 0;
+ }
+
ret = buddy_lock(buddy, flags);
if (ret)
return ret;
@@ -1140,6 +1302,12 @@ __weak int buddy_free_bulk(struct buddy __arena *buddy, void __arena * __arena *
ret = ret ?: err;
}
+ /* Drain any frees stranded during an NMI. */
+ if (unlikely(buddy_deferred_free_pending(buddy))) {
+ err = buddy_deferred_free_complete(buddy);
+ ret = ret ?: err;
+ }
+
buddy_unlock(buddy, flags);
return ret;
--
2.52.0
next prev parent reply other threads:[~2026-10-20 18:19 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-20 18:18 [PATCH bpf-next v1 0/7] bpf libarena allocator improvements Emil Tsalapatis
2026-10-20 18:18 ` [PATCH bpf-next v1 1/7] selftests/bpf: libarena: Add bulk buddy allocation helpers Emil Tsalapatis
2026-10-20 18:18 ` [PATCH bpf-next v1 2/7] selftests/bpf: Add a parallel buddy allocator benchmark Emil Tsalapatis
2026-10-20 18:56 ` bot+bpf-ci
2026-10-20 18:18 ` [PATCH bpf-next v1 3/7] selftests/bpf: libarena: Add buddy per-CPU caches Emil Tsalapatis
2026-10-20 18:36 ` sashiko-bot
2026-10-20 18:56 ` bot+bpf-ci
2026-10-20 18:18 ` Emil Tsalapatis [this message]
2026-10-20 18:35 ` [PATCH bpf-next v1 4/7] selftests/bpf: libarena: Handle NMI reentrancy sashiko-bot
2026-10-20 18:56 ` bot+bpf-ci
2026-10-20 18:18 ` [PATCH bpf-next v1 5/7] selftests/bpf: libarena: Refill local caches in batches Emil Tsalapatis
2026-10-20 18:18 ` [PATCH bpf-next v1 6/7] selftests/bpf: libarena: Add a transfer cache Emil Tsalapatis
2026-10-20 18:19 ` [PATCH bpf-next v1 7/7] selftests/bpf: libarena: Cover buddy cache fast paths Emil Tsalapatis
2026-10-20 18:35 ` sashiko-bot
2026-10-20 21:41 ` [PATCH bpf-next v1 0/7] bpf libarena allocator improvements Alexei Starovoitov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261020181900.6755-5-emil@etsalapatis.com \
--to=emil@etsalapatis.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=memxor@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox