From: Emil Tsalapatis <emil@etsalapatis.com>
To: bpf@vger.kernel.org
Cc: ast@kernel.org, andrii@kernel.org, eddyz87@gmail.com,
memxor@gmail.com, daniel@iogearbox.net,
Emil Tsalapatis <emil@etsalapatis.com>,
Puranjay Mohan <puranjay@kernel.org>
Subject: [PATCH bpf-next v2 4/7] bpf: Add explicit state machine for arena free spans
Date: Wed, 23 Sep 2026 06:14:22 +0000 [thread overview]
Message-ID: <20260923061425.7045-5-emil@etsalapatis.com> (raw)
In-Reply-To: <20260923061425.7045-1-emil@etsalapatis.com>
The arena_free_worker() call currently tracks the status
of each arena_free_span by placing it into a separate
list. This requires multiple list manipulation calls
during the freeing operation, complicating the code for
no reason.
Add an explicit state machine for span state. This
avoids encoding the states within temporary list membership,
and also allows for safely reschedulign freeing work for each
span separately.
Suggested-by: Puranjay Mohan <puranjay@kernel.org>
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
kernel/bpf/arena.c | 108 +++++++++++++++++++++++++++------------------
1 file changed, 65 insertions(+), 43 deletions(-)
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 8bb011aae604..ad58aeaa0c87 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -72,11 +72,18 @@ struct bpf_arena {
static void arena_free_worker(struct work_struct *work);
static void arena_free_irq(struct irq_work *iw);
+enum arena_free_span_state {
+ ARENA_FREE_SPAN_NOT_STARTED, /* Freeing not started */
+ ARENA_FREE_SPAN_UNAVAIL, /* Region cleared & unavailable */
+ ARENA_FREE_SPAN_ZAPPED, /* Region zapped and cleared */
+ ARENA_FREE_SPAN_FAILED, /* Freeing operation cannot continue */
+};
+
struct arena_free_span {
struct llist_node node;
unsigned long uaddr;
u32 page_cnt;
- bool release_only;
+ enum arena_free_span_state state;
};
u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena)
@@ -971,7 +978,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
s->page_cnt = page_cnt;
s->uaddr = uaddr;
- s->release_only = release_only;
+ s->state = release_only ? ARENA_FREE_SPAN_ZAPPED : ARENA_FREE_SPAN_NOT_STARTED;
llist_add(&s->node, &arena->free_spans);
irq_work_queue(&arena->free_irq);
}
@@ -1021,7 +1028,7 @@ static void arena_free_worker(struct work_struct *work)
struct llist_node *list, *pos, *t;
struct arena_free_span *s;
u64 arena_vm_start, user_vm_start;
- struct llist_head free_pages, teardown_spans;
+ struct llist_head free_pages;
struct clear_range_data cdata;
struct page *page;
unsigned long full_uaddr;
@@ -1038,7 +1045,6 @@ static void arena_free_worker(struct work_struct *work)
bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
init_llist_head(&free_pages);
- init_llist_head(&teardown_spans);
cdata.arena = arena;
cdata.free_pages = &free_pages;
arena_vm_start = bpf_arena_get_kern_vm_start(arena);
@@ -1047,26 +1053,18 @@ static void arena_free_worker(struct work_struct *work)
list = llist_del_all(&arena->free_spans);
llist_for_each_safe(pos, t, list) {
s = llist_entry(pos, struct arena_free_span, node);
- page_cnt = s->page_cnt;
- pgoff = compute_pgoff(arena, s->uaddr);
-
- if (s->release_only) {
- ret = range_tree_make_avail(&arena->rt, pgoff, page_cnt);
- WARN_ON_ONCE(ret);
- kfree_nolock(s);
+ if (s->state != ARENA_FREE_SPAN_NOT_STARTED)
continue;
- }
+ page_cnt = s->page_cnt;
+ pgoff = compute_pgoff(arena, s->uaddr);
kaddr = arena_vm_start + s->uaddr;
ret = range_tree_set_unavail(&arena->rt, pgoff, page_cnt);
if (ret) {
/* Kick off another attempt at the end of this call. */
- if (ret == -EAGAIN) {
- llist_add(pos, &arena->free_spans);
- retry = true;
+ if (ret == -EAGAIN)
continue;
- }
/*
* An -ENOMEM failure is the same failure mode as in
@@ -1076,20 +1074,24 @@ static void arena_free_worker(struct work_struct *work)
if (ret != -ENOMEM)
WARN_ON_ONCE(ret);
- kfree_nolock(s);
+ s->state = ARENA_FREE_SPAN_FAILED;
continue;
}
+ s->state = ARENA_FREE_SPAN_UNAVAIL;
+
/* clear ptes and collect pages in free_pages llist */
apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,
apply_range_clear_cb, &cdata);
- __llist_add(pos, &teardown_spans);
}
raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
/* Keep ranges unavailable until their stale translations are gone. */
- llist_for_each_safe(pos, t, READ_ONCE(teardown_spans.first)) {
+ llist_for_each_safe(pos, t, list) {
s = llist_entry(pos, struct arena_free_span, node);
+ if (s->state != ARENA_FREE_SPAN_UNAVAIL)
+ continue;
+
page_cnt = s->page_cnt;
full_uaddr = clear_lo32(user_vm_start) + s->uaddr;
kaddr = arena_vm_start + s->uaddr;
@@ -1099,6 +1101,12 @@ static void arena_free_worker(struct work_struct *work)
/* remove pages from user vmas */
zap_pages(arena, full_uaddr, page_cnt);
+
+ /*
+ * Used to avoid zapping twice if we fail the lock acquisition
+ * below and rerun the span through this function.
+ */
+ s->state = ARENA_FREE_SPAN_ZAPPED;
}
/* free all pages collected by apply_to_existing_page_range() in the first loop */
@@ -1107,40 +1115,54 @@ static void arena_free_worker(struct work_struct *work)
__free_page(page);
}
- if (!llist_empty(&teardown_spans)) {
- if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) {
- llist_for_each_safe(pos, t, __llist_del_all(&teardown_spans)) {
- s = llist_entry(pos, struct arena_free_span, node);
- s->release_only = true;
- llist_add(pos, &arena->free_spans);
+ if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) {
+ llist_for_each_safe(pos, t, list) {
+ s = llist_entry(pos, struct arena_free_span, node);
+
+ if (s->state == ARENA_FREE_SPAN_FAILED) {
+ kfree_nolock(s);
+ continue;
}
- schedule_work(work);
- bpf_map_memcg_exit(old_memcg, new_memcg);
- return;
+ llist_add(pos, &arena->free_spans);
+ retry = true;
}
+ goto done;
+ }
- llist_for_each_safe(pos, t, __llist_del_all(&teardown_spans)) {
- s = llist_entry(pos, struct arena_free_span, node);
- page_cnt = s->page_cnt;
- pgoff = compute_pgoff(arena, s->uaddr);
- /*
- * This range tree operation does not allocate memory,
- * and so should never fail regardless of contention
- * or memory pressure. This is in contrast to regular
- * inserts that _can_ fail under memory pressure and
- * force us to defer the free.
- */
- ret = range_tree_make_avail(&arena->rt, pgoff, page_cnt);
- WARN_ON_ONCE(ret);
+ llist_for_each_safe(pos, t, list) {
+ s = llist_entry(pos, struct arena_free_span, node);
+
+ /* Remove the spans of failed allocations. */
+ if (s->state == ARENA_FREE_SPAN_FAILED) {
kfree_nolock(s);
+ continue;
}
- raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
+
+ if (s->state == ARENA_FREE_SPAN_NOT_STARTED) {
+ llist_add(pos, &arena->free_spans);
+ retry = true;
+ continue;
+ }
+
+ page_cnt = s->page_cnt;
+ pgoff = compute_pgoff(arena, s->uaddr);
+ /*
+ * This range tree operation does not allocate memory,
+ * and so should never fail regardless of contention
+ * or memory pressure. This is in contrast to regular
+ * inserts that _can_ fail under memory pressure and
+ * force us to defer the free.
+ */
+ ret = range_tree_make_avail(&arena->rt, pgoff, page_cnt);
+ WARN_ON_ONCE(ret);
+ kfree_nolock(s);
}
+ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
+done:
bpf_map_memcg_exit(old_memcg, new_memcg);
- /* Retry if any region was unavailable for free. */
if (retry)
schedule_work(work);
}
--
2.54.0
next prev parent reply other threads:[~2026-09-23 6:14 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-23 6:14 [PATCH bpf-next v2 0/7] bpf: Fix arena memory incoherence Emil Tsalapatis
2026-09-23 6:14 ` [PATCH bpf-next v2 1/7] bpf: Update is_range_tree_set to work for consecutive ranges Emil Tsalapatis
2026-09-23 6:14 ` [PATCH bpf-next v2 2/7] bpf: Track availability information for ranges in range tree Emil Tsalapatis
2026-09-23 6:14 ` [PATCH bpf-next v2 3/7] bpf: Fix arena race between page free and alloc leading to incoherency Emil Tsalapatis
2026-09-23 6:14 ` Emil Tsalapatis [this message]
2026-09-23 6:14 ` [PATCH bpf-next v2 5/7] bpf: Atomically update PTE and range tree in arena VM fault handler Emil Tsalapatis
2026-09-23 6:14 ` [PATCH bpf-next v2 6/7] bpf: Avoid unavailable range leakage during arena_free_pages() Emil Tsalapatis
2026-09-23 6:14 ` [PATCH bpf-next v2 7/7] selftests/bpf: Add arena allocation race tests Emil Tsalapatis
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260923061425.7045-5-emil@etsalapatis.com \
--to=emil@etsalapatis.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=eddyz87@gmail.com \
--cc=memxor@gmail.com \
--cc=puranjay@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox