* [PATCH bpf-next v5 1/7] bpf: Use an llist for page allocations
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 2/7] bpf: Add sleepable argument to bpf_alloc_pages() Emil Tsalapatis
` (5 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
bpf_map_alloc_pages() does not use its map argument. Storing allocated
pages in an array also forces arena callers to allocate a separate
pointer array.
Expose the single-page allocator as bpf_alloc_page(), rename the bulk
helper to bpf_alloc_pages(), and return bulk allocations through an
llist using page->pcp_llist. Add bpf_free_pages() to safely release all
pages remaining on such a list.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
include/linux/bpf.h | 6 ++-
kernel/bpf/arena.c | 95 +++++++++++++++++++-------------------------
kernel/bpf/syscall.c | 39 ++++++++++--------
3 files changed, 67 insertions(+), 73 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 4bae3796c42f..904b539810c7 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2902,8 +2902,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
-int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
- unsigned long nr_pages, struct page **page_array);
+struct page *bpf_alloc_page(int nid);
+int bpf_alloc_pages(int nid, unsigned long nr_pages,
+ struct llist_head *pages);
+void bpf_free_pages(struct llist_head *pages);
#ifdef CONFIG_MEMCG
void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg,
struct mem_cgroup **new_memcg);
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index c6369ea5e208..de4f7c7f68f5 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -146,7 +146,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr)
struct apply_range_data {
struct bpf_arena *arena;
- struct page **pages;
+ struct llist_head *pages;
int i;
};
@@ -158,13 +158,17 @@ struct clear_range_data {
static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
{
struct apply_range_data *d = data;
+ struct llist_node *node;
struct page *page;
pte_t pteval;
if (!data)
return 0;
- page = d->pages[d->i];
+ node = READ_ONCE(d->pages->first);
+ if (WARN_ON_ONCE(!node))
+ return -EINVAL;
+ page = llist_entry(node, struct page, pcp_llist);
/* paranoia, similar to vmap_pages_pte_range() */
if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page))))
return -EINVAL;
@@ -197,6 +201,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data)
return -EBUSY;
set_pte_at(&init_mm, addr, pte, pteval);
#endif
+ WARN_ON_ONCE(llist_del_first(d->pages) != node);
d->i++;
WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1);
return 0;
@@ -312,8 +317,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
INIT_WORK(&arena->free_work, arena_free_worker);
bpf_map_init_from_attr(&arena->map, attr);
- err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page);
- if (err)
+ arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE);
+ if (!arena->scratch_page)
goto err_free_arena;
range_tree_init(&arena->rt);
@@ -481,7 +486,9 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
struct bpf_map *map = vmf->vma->vm_file->private_data;
struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
struct mem_cgroup *new_memcg, *old_memcg;
+ LLIST_HEAD(pages);
struct page *page, *new_page = NULL;
+ struct apply_range_data data;
vm_fault_t fault_ret;
long kbase, kaddr;
unsigned long flags;
@@ -543,8 +550,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
* The probed page was freed meanwhile or preallocation failed;
* try the non-blocking allocator, we cannot sleep here.
*/
- ret = bpf_map_alloc_pages(map, map->numa_node, 1, &new_page);
- if (ret) {
+ new_page = bpf_alloc_page(map->numa_node);
+ if (!new_page) {
fault_ret = VM_FAULT_SIGBUS;
goto out_err_locked_memcg;
}
@@ -555,12 +562,14 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
fault_ret = VM_FAULT_SIGBUS;
goto out_err_locked_memcg;
}
- struct apply_range_data data = {
- .arena = arena, .pages = &new_page, .i = 0
- };
+ llist_add(&new_page->pcp_llist, &pages);
+ data.arena = arena;
+ data.pages = &pages;
+ data.i = 0;
ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data);
if (ret) {
+ llist_del_first(&pages);
range_tree_set(&arena->rt, vmf->pgoff, 1);
fault_ret = VM_FAULT_SIGBUS;
goto out_err_locked_memcg;
@@ -716,13 +725,12 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena);
struct mem_cgroup *new_memcg, *old_memcg;
struct apply_range_data data;
- struct page **pages = NULL;
- long remaining, mapped = 0;
- long alloc_pages;
+ LLIST_HEAD(pages);
+ long mapped = 0;
unsigned long flags;
long pgoff = 0;
u32 uaddr32;
- int ret, i;
+ int ret;
if (node_id != NUMA_NO_NODE &&
((unsigned int)node_id >= nr_node_ids || !node_online(node_id)))
@@ -741,15 +749,9 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
}
bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
- /* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */
- alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *));
- pages = kmalloc_nolock(alloc_pages * sizeof(struct page *), __GFP_ACCOUNT, NUMA_NO_NODE);
- if (!pages) {
- bpf_map_memcg_exit(old_memcg, new_memcg);
- return 0;
- }
data.arena = arena;
- data.pages = pages;
+ data.pages = &pages;
+ data.i = 0;
if (raw_res_spin_lock_irqsave(&arena->spinlock, flags))
goto out_free_pages;
@@ -767,45 +769,28 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
if (ret)
goto out_unlock_free_pages;
- remaining = page_cnt;
uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE);
- while (remaining) {
- long this_batch = min(remaining, alloc_pages);
-
- /* zeroing is needed, since alloc_pages_bulk() only fills in non-zero entries */
- memset(pages, 0, this_batch * sizeof(struct page *));
-
- ret = bpf_map_alloc_pages(&arena->map, node_id, this_batch, pages);
- if (ret)
- goto out;
+ ret = bpf_alloc_pages(node_id, page_cnt, &pages);
+ if (ret)
+ goto out;
- /*
- * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1
- * will not overflow 32-bit. Lower 32-bit need to represent
- * contiguous user address range.
- * Map these pages at kern_vm_start base.
- * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow
- * lower 32-bit and it's ok.
- */
- data.i = 0;
- ret = apply_to_page_range(&init_mm,
- kern_vm_start + uaddr32 + (mapped << PAGE_SHIFT),
- this_batch << PAGE_SHIFT, apply_range_set_cb, &data);
- if (ret) {
- /* data.i pages were mapped, account them and free the remaining */
- mapped += data.i;
- for (i = data.i; i < this_batch; i++)
- free_pages_nolock(pages[i], 0);
- goto out;
- }
+ /*
+ * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1
+ * will not overflow 32-bit. Lower 32-bit need to represent
+ * contiguous user address range.
+ * Map these pages at kern_vm_start base.
+ * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow
+ * lower 32-bit and it's ok.
+ */
+ ret = apply_to_page_range(&init_mm, kern_vm_start + uaddr32,
+ page_cnt << PAGE_SHIFT, apply_range_set_cb, &data);
+ mapped = data.i;
+ if (ret)
+ goto out;
- mapped += this_batch;
- remaining -= this_batch;
- }
flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT);
raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
- kfree_nolock(pages);
bpf_map_memcg_exit(old_memcg, new_memcg);
return clear_lo32(arena->user_vm_start) + uaddr32;
out:
@@ -819,7 +804,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
out_unlock_free_pages:
raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
out_free_pages:
- kfree_nolock(pages);
+ bpf_free_pages(&pages);
bpf_map_memcg_exit(old_memcg, new_memcg);
return 0;
}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index ac52f4ae414c..a5df15a6cd51 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -602,7 +602,7 @@ static bool can_alloc_pages(void)
!IS_ENABLED(CONFIG_PREEMPT_RT);
}
-static struct page *__bpf_alloc_page(int nid)
+struct page *bpf_alloc_page(int nid)
{
if (!can_alloc_pages())
return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);
@@ -613,27 +613,34 @@ static struct page *__bpf_alloc_page(int nid)
0);
}
-int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
- unsigned long nr_pages, struct page **pages)
+void bpf_free_pages(struct llist_head *pages)
{
- unsigned long i, j;
+ struct llist_node *node;
+ struct page *page, *tmp;
+
+ node = llist_del_all(pages);
+ llist_for_each_entry_safe(page, tmp, node, pcp_llist)
+ free_pages_nolock(page, 0);
+}
+
+int bpf_alloc_pages(int nid, unsigned long nr_pages,
+ struct llist_head *pages)
+{
+ unsigned long i;
struct page *pg;
- int ret = 0;
for (i = 0; i < nr_pages; i++) {
- pg = __bpf_alloc_page(nid);
-
- if (pg) {
- pages[i] = pg;
- continue;
- }
- for (j = 0; j < i; j++)
- free_pages_nolock(pages[j], 0);
- ret = -ENOMEM;
- break;
+ pg = bpf_alloc_page(nid);
+ if (!pg)
+ goto free_pages;
+ llist_add(&pg->pcp_llist, pages);
}
- return ret;
+ return 0;
+
+free_pages:
+ bpf_free_pages(pages);
+ return -ENOMEM;
}
static int btf_field_cmp(const void *a, const void *b)
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH bpf-next v5 2/7] bpf: Add sleepable argument to bpf_alloc_pages()
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 1/7] bpf: Use an llist for page allocations Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 3/7] bpf: Add sleepable arena page allocation path Emil Tsalapatis
` (4 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
bpf_alloc_pages() currently decides whether it may use the
blocking page allocator from the current execution context
alone. Let callers further restrict that choice by passing
whether their context is sleepable.
Use the blocking allocator only when both the caller and
runtime context allow sleeping. Add __GFP_RETRY_MAYFAIL so
this path can reclaim without invoking the OOM killer when
the allocation is charged to another memcg. Existing
non-sleepable callers retain the no-lock allocation behavior.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
include/linux/bpf.h | 4 ++--
kernel/bpf/arena.c | 6 +++---
kernel/bpf/syscall.c | 10 +++++-----
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 904b539810c7..670eb9f3f20a 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2902,9 +2902,9 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
-struct page *bpf_alloc_page(int nid);
+struct page *bpf_alloc_page(int nid, bool sleepable);
int bpf_alloc_pages(int nid, unsigned long nr_pages,
- struct llist_head *pages);
+ struct llist_head *pages, bool sleepable);
void bpf_free_pages(struct llist_head *pages);
#ifdef CONFIG_MEMCG
void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg,
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index de4f7c7f68f5..c556df7730c4 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -317,7 +317,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
INIT_WORK(&arena->free_work, arena_free_worker);
bpf_map_init_from_attr(&arena->map, attr);
- arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE);
+ arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE, true);
if (!arena->scratch_page)
goto err_free_arena;
@@ -550,7 +550,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
* The probed page was freed meanwhile or preallocation failed;
* try the non-blocking allocator, we cannot sleep here.
*/
- new_page = bpf_alloc_page(map->numa_node);
+ new_page = bpf_alloc_page(map->numa_node, false);
if (!new_page) {
fault_ret = VM_FAULT_SIGBUS;
goto out_err_locked_memcg;
@@ -771,7 +771,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE);
- ret = bpf_alloc_pages(node_id, page_cnt, &pages);
+ ret = bpf_alloc_pages(node_id, page_cnt, &pages, false);
if (ret)
goto out;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a5df15a6cd51..ef8fb2f6e6e3 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -602,14 +602,14 @@ static bool can_alloc_pages(void)
!IS_ENABLED(CONFIG_PREEMPT_RT);
}
-struct page *bpf_alloc_page(int nid)
+struct page *bpf_alloc_page(int nid, bool sleepable)
{
- if (!can_alloc_pages())
+ if (!sleepable || !can_alloc_pages())
return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);
return alloc_pages_node(nid,
GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT
- | __GFP_NOWARN,
+ | __GFP_NOWARN | __GFP_RETRY_MAYFAIL,
0);
}
@@ -624,13 +624,13 @@ void bpf_free_pages(struct llist_head *pages)
}
int bpf_alloc_pages(int nid, unsigned long nr_pages,
- struct llist_head *pages)
+ struct llist_head *pages, bool sleepable)
{
unsigned long i;
struct page *pg;
for (i = 0; i < nr_pages; i++) {
- pg = bpf_alloc_page(nid);
+ pg = bpf_alloc_page(nid, sleepable);
if (!pg)
goto free_pages;
llist_add(&pg->pcp_llist, pages);
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH bpf-next v5 3/7] bpf: Add sleepable arena page allocation path
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 1/7] bpf: Use an llist for page allocations Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 2/7] bpf: Add sleepable argument to bpf_alloc_pages() Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 4/7] selftests/bpf: Test large allocations for both sleepable/nonsleepable arena users Emil Tsalapatis
` (3 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
The bpf_arena_alloc_pages() function currently only allocates pages
inside a spinlock critical section with IRQs off. This forces the use of
alloc_pages_nolock() in the BPF allocator, even when the caller is a
sleepable BPF function. This in turn causes allocation failures even in
cases where falling into the allocator slow path and possibly sleeping
would eventually succeed. This can be triggered consistently by heavy
BPF arena users like scx.
Allocate the arena pages before taking the critical section and pass
whether the caller can sleep to bpf_alloc_pages(). This lets sleepable
callers use the blocking allocator while non-sleepable callers retain
the no-lock allocation behavior.
Fixes: b8467290edab ("bpf: arena: make arena kfuncs any context safe")
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
kernel/bpf/arena.c | 73 +++++++++++++++++++++++++++++-----------------
1 file changed, 46 insertions(+), 27 deletions(-)
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index c556df7730c4..0ff707707da0 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -713,6 +713,27 @@ static u64 clear_lo32(u64 val)
return val & ~(u64)~0U;
}
+static int arena_adjust_tree(struct bpf_arena *arena, long uaddr, long page_cnt, long *pgoff)
+{
+ int ret;
+
+ /* Special case where user is requesting specific range. */
+ if (uaddr) {
+ ret = is_range_tree_set(&arena->rt, *pgoff, page_cnt);
+ if (ret)
+ return ret;
+ return range_tree_clear(&arena->rt, *pgoff, page_cnt);
+ }
+
+ ret = range_tree_find(&arena->rt, page_cnt);
+ if (ret < 0)
+ return ret;
+
+ *pgoff = ret;
+
+ return range_tree_clear(&arena->rt, *pgoff, page_cnt);
+}
+
/*
* Allocate pages and vmap them into kernel vmalloc area.
* Later the pages will be mmaped into user space vma.
@@ -730,6 +751,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
unsigned long flags;
long pgoff = 0;
u32 uaddr32;
+ long addr = 0;
int ret;
if (node_id != NUMA_NO_NODE &&
@@ -747,8 +769,12 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
/* requested address will be outside of user VMA */
return 0;
}
-
bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
+
+ ret = bpf_alloc_pages(node_id, page_cnt, &pages, sleepable);
+ if (ret)
+ goto out_memcg;
+
data.arena = arena;
data.pages = &pages;
data.i = 0;
@@ -756,25 +782,14 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
if (raw_res_spin_lock_irqsave(&arena->spinlock, flags))
goto out_free_pages;
- if (uaddr) {
- ret = is_range_tree_set(&arena->rt, pgoff, page_cnt);
- if (ret)
- goto out_unlock_free_pages;
- ret = range_tree_clear(&arena->rt, pgoff, page_cnt);
- } else {
- ret = pgoff = range_tree_find(&arena->rt, page_cnt);
- if (pgoff >= 0)
- ret = range_tree_clear(&arena->rt, pgoff, page_cnt);
+ ret = arena_adjust_tree(arena, uaddr, page_cnt, &pgoff);
+ if (ret) {
+ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
+ goto out_free_pages;
}
- if (ret)
- goto out_unlock_free_pages;
uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE);
- ret = bpf_alloc_pages(node_id, page_cnt, &pages, false);
- if (ret)
- goto out;
-
/*
* Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1
* will not overflow 32-bit. Lower 32-bit need to represent
@@ -787,26 +802,30 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
page_cnt << PAGE_SHIFT, apply_range_set_cb, &data);
mapped = data.i;
if (ret)
- goto out;
+ goto out_unmap;
flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT);
raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
- bpf_map_memcg_exit(old_memcg, new_memcg);
- return clear_lo32(arena->user_vm_start) + uaddr32;
-out:
+
+ addr = clear_lo32(arena->user_vm_start) + uaddr32;
+ goto out_memcg;
+
+out_unmap:
+ /* Error handling: Undo partial mappings. */
+ flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT);
range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped);
raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
- if (mapped) {
- flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT);
+ if (mapped)
arena_free_pages(arena, uaddr32, mapped, sleepable);
- }
- goto out_free_pages;
-out_unlock_free_pages:
- raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
+
out_free_pages:
+ /* Error handling: Free back any unmapped pages. */
bpf_free_pages(&pages);
+
+out_memcg:
bpf_map_memcg_exit(old_memcg, new_memcg);
- return 0;
+
+ return addr;
}
/*
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH bpf-next v5 4/7] selftests/bpf: Test large allocations for both sleepable/nonsleepable arena users
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
` (2 preceding siblings ...)
2026-09-28 20:26 ` [PATCH bpf-next v5 3/7] bpf: Add sleepable arena page allocation path Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 5/7] bpf: Directly store kfunc desc index in instruction off field Emil Tsalapatis
` (2 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
We have relaxed the limitation of 1024-page batches for both sleepable
and nonsleepable arena allocations by removing the intermediate page
array. Add a test to confirm that large allocations succeed for both.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
.../bpf/progs/verifier_arena_large.c | 64 +++++++++++++++----
1 file changed, 50 insertions(+), 14 deletions(-)
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c
index 6ab8730d4878..dbdea14ca76f 100644
--- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c
+++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c
@@ -10,6 +10,9 @@
#include <bpf_arena_common.h>
#define ARENA_SIZE (1ull << 32)
+#define LARGE_PAGE_CNT 1025
+
+volatile int zero = 0;
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
@@ -284,6 +287,7 @@ int big_alloc2(void *ctx)
return 0;
}
+/* Nonsleepable because it binds to a socket program. */
SEC("socket")
__success __retval(0)
int big_alloc3(void *ctx)
@@ -291,24 +295,56 @@ int big_alloc3(void *ctx)
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
char __arena *pages;
u64 i;
+ int err = 0;
- /*
- * Allocate 2051 pages in one go to check how kmalloc_nolock() handles large requests.
- * Since kmalloc_nolock() can allocate up to 1024 struct page * at a time, this call should
- * result in three batches: two batches of 1024 pages each, followed by a final batch of 3
- * pages.
- */
- pages = bpf_arena_alloc_pages(&arena, NULL, 2051, NUMA_NO_NODE, 0);
+ /* Verify that a nonsleepable allocation larger than 1024 pages succeeds. */
+ pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0);
if (!pages)
- return 0;
+ return 1;
+
+ for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++)
+ pages[i * PAGE_SIZE] = 123;
+
+ for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) {
+ if (pages[i * PAGE_SIZE] == 123)
+ continue;
+ err = 2;
+ break;
+ }
+
+ bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT);
+ return err;
+#endif
+ return 0;
+}
- bpf_for(i, 0, 2051)
- pages[i * PAGE_SIZE] = 123;
- bpf_for(i, 0, 2051)
- if (pages[i * PAGE_SIZE] != 123)
- return i;
+/* SYSCALL programs are always sleepable. */
+SEC("syscall")
+__success __retval(0)
+int big_alloc4(void *ctx)
+{
+#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
+ char __arena *pages;
+ u64 i;
+ int err = 0;
+
+ /* Verify that a sleepable allocation larger than 1024 pages succeeds. */
+ pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0);
+ if (!pages)
+ return 1;
+
+ for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++)
+ pages[i * PAGE_SIZE] = 123;
+
+ for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) {
+ if (pages[i * PAGE_SIZE] == 123)
+ continue;
+ err = 2;
+ break;
+ }
- bpf_arena_free_pages(&arena, pages, 2051);
+ bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT);
+ return err;
#endif
return 0;
}
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH bpf-next v5 5/7] bpf: Directly store kfunc desc index in instruction off field
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
` (3 preceding siblings ...)
2026-09-28 20:26 ` [PATCH bpf-next v5 4/7] selftests/bpf: Test large allocations for both sleepable/nonsleepable arena users Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 6/7] bpf: Support per-call-site kfunc specialization Emil Tsalapatis
2026-09-28 20:26 ` [PATCH bpf-next v5 7/7] selftests/bpf: Test per-call site function specialization Emil Tsalapatis
6 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
Currently, the verifier sort the kfunc desc table twice: Once during
kfunc collection by function ID, useful during verification, and
once by immediate address, useful during JIT bytecode lowering
time. The latter sort can be removed by storing in the instruction
the kfunc desc array index the desc is in and using that instead.
Modify the JITs to follow the same convention.
This change simplifies the subsequent patch that adds per-call site
function specialization.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
include/linux/bpf.h | 4 +--
include/linux/bpf_verifier.h | 7 ++--
kernel/bpf/fixups.c | 70 +++++-------------------------------
kernel/bpf/verifier.c | 25 ++++++++++---
4 files changed, 33 insertions(+), 73 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 670eb9f3f20a..eed7f8f1e417 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -3301,7 +3301,7 @@ const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
const struct bpf_insn *insn);
int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
- u16 btf_fd_idx, u8 **func_addr);
+ u16 desc_idx, u8 **func_addr);
struct bpf_core_ctx {
struct bpf_verifier_log *log;
@@ -3644,7 +3644,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
static inline int
bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
- u16 btf_fd_idx, u8 **func_addr)
+ u16 desc_idx, u8 **func_addr)
{
return -ENOTSUPP;
}
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index c775bd757706..5cbae5d05fe7 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1784,12 +1784,12 @@ enum bpf_reg_arg_type {
};
#define MAX_KFUNC_DESCS 256
+static_assert(MAX_KFUNC_DESCS <= S16_MAX + 1);
struct bpf_kfunc_desc {
struct btf_func_model func_model;
struct bpf_func_proto proto;
u32 func_id;
- s32 imm;
u16 offset;
unsigned long addr;
};
@@ -1797,9 +1797,8 @@ struct bpf_kfunc_desc {
struct bpf_kfunc_desc_tab {
u32 nr_descs;
/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
- * verification. JITs do lookups by bpf_insn, where func_id may not be
- * available, therefore at the end of verification do_misc_fixups()
- * sorts this by imm and offset.
+ * verification. JITs use the descriptor index stored in the finalized
+ * call's off field.
*
* Grown one entry at a time by bpf_add_kfunc_call().
*/
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 37cf130ebb57..cb7219ff68bd 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -5,8 +5,6 @@
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/vmalloc.h>
-#include <linux/bsearch.h>
-#include <linux/sort.h>
#include <linux/perf_event.h>
#include <linux/sched/signal.h>
#include <net/xdp.h>
@@ -117,73 +115,26 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn)
return dst_reg;
}
-static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b)
-{
- const struct bpf_kfunc_desc *d0 = a;
- const struct bpf_kfunc_desc *d1 = b;
-
- if (d0->imm != d1->imm)
- return d0->imm < d1->imm ? -1 : 1;
- if (d0->offset != d1->offset)
- return d0->offset < d1->offset ? -1 : 1;
- return 0;
-}
-
const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
const struct bpf_insn *insn)
{
- const struct bpf_kfunc_desc desc = {
- .imm = insn->imm,
- .offset = insn->off,
- };
const struct bpf_kfunc_desc *res;
struct bpf_kfunc_desc_tab *tab;
tab = prog->aux->kfunc_tab;
- res = bsearch(&desc, tab->descs, tab->nr_descs,
- sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off);
-
- return res ? &res->func_model : NULL;
-}
-
-static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)
-{
- unsigned long call_imm;
+ if (insn->off < 0 || insn->off >= tab->nr_descs)
+ return NULL;
+ res = &tab->descs[insn->off];
if (bpf_jit_supports_far_kfunc_call()) {
- call_imm = desc->func_id;
- } else {
- call_imm = BPF_CALL_IMM(desc->addr);
- /* Check whether the relative offset overflows desc->imm */
- if ((unsigned long)(s32)call_imm != call_imm) {
- verbose(env, "address of kernel func_id %u is out of range\n",
- desc->func_id);
- return -EINVAL;
- }
- }
- desc->imm = call_imm;
- return 0;
-}
-
-static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env)
-{
- struct bpf_kfunc_desc_tab *tab;
- int i, err;
-
- tab = env->prog->aux->kfunc_tab;
- if (!tab)
- return 0;
-
- for (i = 0; i < tab->nr_descs; i++) {
- err = set_kfunc_desc_imm(env, &tab->descs[i]);
- if (err)
- return err;
+ if (res->func_id != insn->imm)
+ return NULL;
+ } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) {
+ return NULL;
}
- sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
- kfunc_desc_cmp_by_imm_off, NULL);
- return 0;
+ return &res->func_model;
}
static int add_kfunc_in_insns(struct bpf_verifier_env *env,
@@ -2720,10 +2671,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
}
}
- ret = sort_kfunc_descs_by_imm_off(env);
- if (ret)
- return ret;
-
return 0;
}
@@ -2897,4 +2844,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env)
return 0;
}
-
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 03dbc0e00398..8183a8f22ed5 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2584,12 +2584,16 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
}
int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
- u16 btf_fd_idx, u8 **func_addr)
+ u16 desc_idx, u8 **func_addr)
{
+ struct bpf_kfunc_desc_tab *tab;
const struct bpf_kfunc_desc *desc;
- desc = find_kfunc_desc(prog, func_id, btf_fd_idx);
- if (!desc)
+ tab = prog->aux->kfunc_tab;
+ if (desc_idx >= tab->nr_descs)
+ return -EFAULT;
+ desc = &tab->descs[desc_idx];
+ if (desc->func_id != func_id)
return -EFAULT;
*func_addr = (u8 *)desc->addr;
@@ -22079,6 +22083,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_insn *insn_buf, int insn_idx, int *cnt)
{
struct bpf_kfunc_desc *desc;
+ unsigned long call_imm;
+ u16 desc_idx;
int err;
if (!insn->imm) {
@@ -22098,13 +22104,22 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
insn->imm);
return -EFAULT;
}
+ desc_idx = desc - env->prog->aux->kfunc_tab->descs;
err = specialize_kfunc(env, desc, insn_idx);
if (err)
return err;
- if (!bpf_jit_supports_far_kfunc_call())
- insn->imm = BPF_CALL_IMM(desc->addr);
+ if (!bpf_jit_supports_far_kfunc_call()) {
+ call_imm = BPF_CALL_IMM(desc->addr);
+ if ((unsigned long)(s32)call_imm != call_imm) {
+ verbose(env, "address of kernel func_id %u is out of range\n",
+ desc->func_id);
+ return -EINVAL;
+ }
+ insn->imm = call_imm;
+ }
+ insn->off = desc_idx;
if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) {
struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH bpf-next v5 6/7] bpf: Support per-call-site kfunc specialization
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
` (4 preceding siblings ...)
2026-09-28 20:26 ` [PATCH bpf-next v5 5/7] bpf: Directly store kfunc desc index in instruction off field Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
2026-09-28 21:11 ` bot+bpf-ci
2026-09-28 20:26 ` [PATCH bpf-next v5 7/7] selftests/bpf: Test per-call site function specialization Emil Tsalapatis
6 siblings, 1 reply; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
specialize_kfunc() currently updates the canonical kfunc descriptor in
place. Different call sites therefore cannot select different
specializations of the same kfunc, and the selected target depends on
verification order.
Keep canonical descriptors unchanged for verifier lookups. Specialize a
copy for each call site, then reuse or append an immutable target
descriptor and store its index in the finalized call instruction.
Allow space for one canonical and one specialized target per kfunc. This
is conservative because most kfuncs do not specialize. Adding specialized
kfunc versions does not require resorting the array because lookups are
only used for deduplication and func_model lookup. Deduplication for
specialized kfuncs is handled by the specialization process itself,
while all specializations of a function share the same proto and
func_model.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
include/linux/bpf_verifier.h | 12 ++++--
kernel/bpf/verifier.c | 75 +++++++++++++++++++++++++++++++++---
2 files changed, 78 insertions(+), 9 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 5cbae5d05fe7..7bdbda7764c7 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -1784,7 +1784,9 @@ enum bpf_reg_arg_type {
};
#define MAX_KFUNC_DESCS 256
-static_assert(MAX_KFUNC_DESCS <= S16_MAX + 1);
+/* Each kfunc can have its canonical and one specialized call target. */
+#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2)
+static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1);
struct bpf_kfunc_desc {
struct btf_func_model func_model;
@@ -1796,11 +1798,15 @@ struct bpf_kfunc_desc {
struct bpf_kfunc_desc_tab {
u32 nr_descs;
+ u32 nr_base_descs;
/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
* verification. JITs use the descriptor index stored in the finalized
- * call's off field.
+ * call's off field. The first nr_base_descs entries are the canonical
+ * descriptors used for verifier lookups. Call specialization may append
+ * immutable descriptors for additional targets.
*
- * Grown one entry at a time by bpf_add_kfunc_call().
+ * Grown one entry at a time by bpf_add_kfunc_call() and during
+ * call specialization.
*/
struct bpf_kfunc_desc descs[];
};
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 8183a8f22ed5..8ba831329ab7 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2579,7 +2579,7 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
struct bpf_kfunc_desc_tab *tab;
tab = prog->aux->kfunc_tab;
- return bsearch(&desc, tab->descs, tab->nr_descs,
+ return bsearch(&desc, tab->descs, tab->nr_base_descs,
sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
}
@@ -2933,10 +2933,12 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
if (find_kfunc_desc(env->prog, func_id, offset))
return 0;
- if (tab->nr_descs == MAX_KFUNC_DESCS) {
+ if (tab->nr_base_descs == MAX_KFUNC_DESCS) {
verbose(env, "too many different kernel function calls\n");
return -E2BIG;
}
+ if (WARN_ON_ONCE(tab->nr_descs != tab->nr_base_descs))
+ return -EFAULT;
err = fetch_kfunc_meta(env, func_id, offset, &kfunc);
if (err)
@@ -2994,7 +2996,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
desc->addr = addr;
desc->func_model = func_model;
tab->nr_descs++;
- sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
+ tab->nr_base_descs++;
+ sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]),
kfunc_desc_cmp_by_id_off, NULL);
return 0;
}
@@ -22062,6 +22065,56 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc
return 0;
}
+static int add_kfunc_desc_target(struct bpf_verifier_env *env,
+ const struct bpf_kfunc_desc *target_desc,
+ u16 base_desc_idx, u16 *desc_idx)
+{
+ struct bpf_kfunc_desc desc = *target_desc;
+ struct bpf_kfunc_desc_tab *new_tab;
+ struct bpf_kfunc_desc_tab *tab;
+ struct bpf_prog_aux *prog_aux;
+ u32 i;
+
+ prog_aux = env->prog->aux;
+ tab = prog_aux->kfunc_tab;
+
+ if (WARN_ON_ONCE(base_desc_idx >= tab->nr_base_descs))
+ return -EFAULT;
+ if (WARN_ON_ONCE(tab->descs[base_desc_idx].func_id != desc.func_id ||
+ tab->descs[base_desc_idx].offset != desc.offset))
+ return -EFAULT;
+
+ if (tab->descs[base_desc_idx].addr == desc.addr) {
+ *desc_idx = base_desc_idx;
+ return 0;
+ }
+
+ for (i = tab->nr_base_descs; i < tab->nr_descs; i++) {
+ if (tab->descs[i].func_id == desc.func_id &&
+ tab->descs[i].offset == desc.offset &&
+ tab->descs[i].addr == desc.addr) {
+ *desc_idx = i;
+ return 0;
+ }
+ }
+
+ if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) {
+ verbose(env, "too many different kernel function call targets\n");
+ return -E2BIG;
+ }
+
+ new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1),
+ GFP_KERNEL_ACCOUNT);
+ if (!new_tab)
+ return -ENOMEM;
+ tab = new_tab;
+ prog_aux->kfunc_tab = tab;
+
+ *desc_idx = tab->nr_descs;
+ tab->descs[tab->nr_descs++] = desc;
+ return 0;
+}
+
static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
u16 struct_meta_reg,
u16 node_offset_reg,
@@ -22082,9 +22135,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_insn *insn_buf, int insn_idx, int *cnt)
{
+ struct bpf_kfunc_desc desc_copy;
struct bpf_kfunc_desc *desc;
unsigned long call_imm;
- u16 desc_idx;
+ u16 base_desc_idx, desc_idx;
+ bool near_call;
int err;
if (!insn->imm) {
@@ -22104,13 +22159,17 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
insn->imm);
return -EFAULT;
}
- desc_idx = desc - env->prog->aux->kfunc_tab->descs;
+ base_desc_idx = desc - env->prog->aux->kfunc_tab->descs;
+
+ near_call = !bpf_jit_supports_far_kfunc_call();
+ desc_copy = *desc;
+ desc = &desc_copy;
err = specialize_kfunc(env, desc, insn_idx);
if (err)
return err;
- if (!bpf_jit_supports_far_kfunc_call()) {
+ if (near_call) {
call_imm = BPF_CALL_IMM(desc->addr);
if ((unsigned long)(s32)call_imm != call_imm) {
verbose(env, "address of kernel func_id %u is out of range\n",
@@ -22119,6 +22178,10 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}
insn->imm = call_imm;
}
+
+ err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx);
+ if (err)
+ return err;
insn->off = desc_idx;
if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) {
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH bpf-next v5 6/7] bpf: Support per-call-site kfunc specialization
2026-09-28 20:26 ` [PATCH bpf-next v5 6/7] bpf: Support per-call-site kfunc specialization Emil Tsalapatis
@ 2026-09-28 21:11 ` bot+bpf-ci
2026-09-28 21:48 ` Emil Tsalapatis
0 siblings, 1 reply; 10+ messages in thread
From: bot+bpf-ci @ 2026-09-28 21:11 UTC (permalink / raw)
To: emil, bpf
Cc: ast, andrii, eddyz87, memxor, daniel, emil, ast, andrii, daniel,
martin.lau, eddyz87, yonghong.song, mason, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 3964 bytes --]
> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
> index 5cbae5d05fe7..7bdbda7764c7 100644
> --- a/include/linux/bpf_verifier.h
> +++ b/include/linux/bpf_verifier.h
[ ... ]
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 8183a8f22ed5..8ba831329ab7 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
[ ... ]
> @@ -22062,6 +22065,56 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc
> return 0;
> }
>
> +static int add_kfunc_desc_target(struct bpf_verifier_env *env,
> + const struct bpf_kfunc_desc *target_desc,
> + u16 base_desc_idx, u16 *desc_idx)
> +{
> + struct bpf_kfunc_desc desc = *target_desc;
> + struct bpf_kfunc_desc_tab *new_tab;
> + struct bpf_kfunc_desc_tab *tab;
> + struct bpf_prog_aux *prog_aux;
> + u32 i;
> +
> + prog_aux = env->prog->aux;
> + tab = prog_aux->kfunc_tab;
> +
> + if (WARN_ON_ONCE(base_desc_idx >= tab->nr_base_descs))
> + return -EFAULT;
> + if (WARN_ON_ONCE(tab->descs[base_desc_idx].func_id != desc.func_id ||
> + tab->descs[base_desc_idx].offset != desc.offset))
> + return -EFAULT;
This isn't a bug, but would verifier_bug_if() be a better fit for these
sanity checks?
Looking at bpf_fixup_kfunc_call() a few lines below, the
descriptor-not-found case uses:
verifier_bug(env, "kernel function descriptor not found for func_id %u",
insn->imm);
return -EFAULT;
verifier_bug() writes the reason to the verifier log and only WARNs under
CONFIG_DEBUG_KERNEL. A bare WARN_ON_ONCE() warns on every kernel and leaves
the log with no explanation for the -EFAULT.
> +
> + if (tab->descs[base_desc_idx].addr == desc.addr) {
> + *desc_idx = base_desc_idx;
> + return 0;
> + }
> +
> + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) {
> + if (tab->descs[i].func_id == desc.func_id &&
> + tab->descs[i].offset == desc.offset &&
> + tab->descs[i].addr == desc.addr) {
> + *desc_idx = i;
> + return 0;
> + }
> + }
> +
> + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) {
> + verbose(env, "too many different kernel function call targets\n");
> + return -E2BIG;
> + }
> +
> + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1),
> + GFP_KERNEL_ACCOUNT);
> + if (!new_tab)
> + return -ENOMEM;
> + tab = new_tab;
> + prog_aux->kfunc_tab = tab;
> +
> + *desc_idx = tab->nr_descs;
> + tab->descs[tab->nr_descs++] = desc;
> + return 0;
> +}
[ ... ]
> @@ -22082,9 +22135,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
> int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> struct bpf_insn *insn_buf, int insn_idx, int *cnt)
> {
> + struct bpf_kfunc_desc desc_copy;
> struct bpf_kfunc_desc *desc;
> unsigned long call_imm;
> - u16 desc_idx;
> + u16 base_desc_idx, desc_idx;
> + bool near_call;
> int err;
>
> if (!insn->imm) {
> @@ -22104,13 +22159,17 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> insn->imm);
> return -EFAULT;
> }
> - desc_idx = desc - env->prog->aux->kfunc_tab->descs;
> + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs;
> +
> + near_call = !bpf_jit_supports_far_kfunc_call();
> + desc_copy = *desc;
> + desc = &desc_copy;
>
> err = specialize_kfunc(env, desc, insn_idx);
> if (err)
> return err;
>
> - if (!bpf_jit_supports_far_kfunc_call()) {
> + if (near_call) {
> call_imm = BPF_CALL_IMM(desc->addr);
> if ((unsigned long)(s32)call_imm != call_imm) {
> verbose(env, "address of kernel func_id %u is out of range\n",
The same question applies to the WARN_ON_ONCE() in bpf_add_kfunc_call()
which checks `tab->nr_descs != tab->nr_base_descs`.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/36480760039
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH bpf-next v5 6/7] bpf: Support per-call-site kfunc specialization
2026-09-28 21:11 ` bot+bpf-ci
@ 2026-09-28 21:48 ` Emil Tsalapatis
0 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 21:48 UTC (permalink / raw)
To: bot+bpf-ci, emil, bpf
Cc: ast, andrii, eddyz87, memxor, daniel, martin.lau, yonghong.song,
mason, ihor.solodrai
On Mon Sep 28, 2026 at 9:11 PM UTC, bot+bpf-ci wrote:
>> diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
>> index 5cbae5d05fe7..7bdbda7764c7 100644
>> --- a/include/linux/bpf_verifier.h
>> +++ b/include/linux/bpf_verifier.h
>
> [ ... ]
>
>> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
>> index 8183a8f22ed5..8ba831329ab7 100644
>> --- a/kernel/bpf/verifier.c
>> +++ b/kernel/bpf/verifier.c
>
> [ ... ]
>
>> @@ -22062,6 +22065,56 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc
>> return 0;
>> }
>>
>> +static int add_kfunc_desc_target(struct bpf_verifier_env *env,
>> + const struct bpf_kfunc_desc *target_desc,
>> + u16 base_desc_idx, u16 *desc_idx)
>> +{
>> + struct bpf_kfunc_desc desc = *target_desc;
>> + struct bpf_kfunc_desc_tab *new_tab;
>> + struct bpf_kfunc_desc_tab *tab;
>> + struct bpf_prog_aux *prog_aux;
>> + u32 i;
>> +
>> + prog_aux = env->prog->aux;
>> + tab = prog_aux->kfunc_tab;
>> +
>> + if (WARN_ON_ONCE(base_desc_idx >= tab->nr_base_descs))
>> + return -EFAULT;
>> + if (WARN_ON_ONCE(tab->descs[base_desc_idx].func_id != desc.func_id ||
>> + tab->descs[base_desc_idx].offset != desc.offset))
>> + return -EFAULT;
>
> This isn't a bug, but would verifier_bug_if() be a better fit for these
> sanity checks?
>
> Looking at bpf_fixup_kfunc_call() a few lines below, the
> descriptor-not-found case uses:
>
> verifier_bug(env, "kernel function descriptor not found for func_id %u",
> insn->imm);
> return -EFAULT;
>
> verifier_bug() writes the reason to the verifier log and only WARNs under
> CONFIG_DEBUG_KERNEL. A bare WARN_ON_ONCE() warns on every kernel and leaves
> the log with no explanation for the -EFAULT.
>
Seems like a reasonable change, I will leave this version up for a bit before
resending in case anything else comes up that we need to change in the next
respin. Not applying Changes Requested for now since patches 5/6 could use a
review.
>> +
>> + if (tab->descs[base_desc_idx].addr == desc.addr) {
>> + *desc_idx = base_desc_idx;
>> + return 0;
>> + }
>> +
>> + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) {
>> + if (tab->descs[i].func_id == desc.func_id &&
>> + tab->descs[i].offset == desc.offset &&
>> + tab->descs[i].addr == desc.addr) {
>> + *desc_idx = i;
>> + return 0;
>> + }
>> + }
>> +
>> + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) {
>> + verbose(env, "too many different kernel function call targets\n");
>> + return -E2BIG;
>> + }
>> +
>> + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1),
>> + GFP_KERNEL_ACCOUNT);
>> + if (!new_tab)
>> + return -ENOMEM;
>> + tab = new_tab;
>> + prog_aux->kfunc_tab = tab;
>> +
>> + *desc_idx = tab->nr_descs;
>> + tab->descs[tab->nr_descs++] = desc;
>> + return 0;
>> +}
>
> [ ... ]
>
>> @@ -22082,9 +22135,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
>> int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>> struct bpf_insn *insn_buf, int insn_idx, int *cnt)
>> {
>> + struct bpf_kfunc_desc desc_copy;
>> struct bpf_kfunc_desc *desc;
>> unsigned long call_imm;
>> - u16 desc_idx;
>> + u16 base_desc_idx, desc_idx;
>> + bool near_call;
>> int err;
>>
>> if (!insn->imm) {
>> @@ -22104,13 +22159,17 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>> insn->imm);
>> return -EFAULT;
>> }
>> - desc_idx = desc - env->prog->aux->kfunc_tab->descs;
>> + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs;
>> +
>> + near_call = !bpf_jit_supports_far_kfunc_call();
>> + desc_copy = *desc;
>> + desc = &desc_copy;
>>
>> err = specialize_kfunc(env, desc, insn_idx);
>> if (err)
>> return err;
>>
>> - if (!bpf_jit_supports_far_kfunc_call()) {
>> + if (near_call) {
>> call_imm = BPF_CALL_IMM(desc->addr);
>> if ((unsigned long)(s32)call_imm != call_imm) {
>> verbose(env, "address of kernel func_id %u is out of range\n",
>
> The same question applies to the WARN_ON_ONCE() in bpf_add_kfunc_call()
> which checks `tab->nr_descs != tab->nr_base_descs`.
>
>
> ---
> AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
> See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
>
> CI run summary: https://github.com/kernel-patches/bpf/actions/runs/36480760039
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH bpf-next v5 7/7] selftests/bpf: Test per-call site function specialization
2026-09-28 20:26 [PATCH bpf-next v5 0/7] Make sleepable arena paths use sleepable alloc_pages Emil Tsalapatis
` (5 preceding siblings ...)
2026-09-28 20:26 ` [PATCH bpf-next v5 6/7] bpf: Support per-call-site kfunc specialization Emil Tsalapatis
@ 2026-09-28 20:26 ` Emil Tsalapatis
6 siblings, 0 replies; 10+ messages in thread
From: Emil Tsalapatis @ 2026-09-28 20:26 UTC (permalink / raw)
To: bpf; +Cc: ast, andrii, eddyz87, memxor, daniel, Emil Tsalapatis
Add a test to ensure function call specialization is
done per-call site. Use bpf_dynptr_from_file that has
observably different behavior between its sleepable and
nonsleepable versions. The sleepable path fails in the
sleepable __kernel_read() call with -EIO, while the
nonsleepable fails in the page-cache lookup path with
-EFAULT. Test that whatever the order the nonsleepable
and sleepable calls are made in the program, both
call sites use the correct specialized kfunc.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
.../selftests/bpf/prog_tests/file_reader.c | 15 ++
.../testing/selftests/bpf/progs/file_reader.c | 129 ++++++++++++++++++
2 files changed, 144 insertions(+)
diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c
index 48aae7ea0e4b..e59c6c87e9d5 100644
--- a/tools/testing/selftests/bpf/prog_tests/file_reader.c
+++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c
@@ -7,10 +7,12 @@
#include "file_reader_fail.skel.h"
#include <dlfcn.h>
#include <sys/mman.h>
+#include <sys/stat.h>
const char *user_ptr = "hello world";
char file_contents[256000];
void *addr;
+__u64 beyond_eof_offset;
void *get_executable_base_addr(void)
{
@@ -26,12 +28,18 @@ void *get_executable_base_addr(void)
static int initialize_file_contents(void)
{
+ struct stat st;
int fd, page_sz = sysconf(_SC_PAGESIZE);
ssize_t n = 0, cur;
fd = open("/proc/self/exe", O_RDONLY);
if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n"))
return 1;
+ if (!ASSERT_OK(fstat(fd, &st), "fstat /proc/self/exe")) {
+ close(fd);
+ return 1;
+ }
+ beyond_eof_offset = st.st_size + (1ULL << 30);
do {
cur = read(fd, file_contents + n, sizeof(file_contents) - n);
@@ -75,6 +83,7 @@ static void run_test(const char *prog_name)
memcpy(skel->bss->user_buf, file_contents, sizeof(file_contents));
skel->bss->pid = getpid();
+ skel->bss->beyond_eof_offset = beyond_eof_offset;
err = file_reader__load(skel);
if (!ASSERT_OK(err, "file_reader__load"))
@@ -110,6 +119,12 @@ void test_file_reader(void)
if (test__start_subtest("on_open_validate_file_read"))
run_test("on_open_validate_file_read");
+ if (test__start_subtest("on_open_non_sleepable_first"))
+ run_test("on_open_non_sleepable_first");
+
+ if (test__start_subtest("on_open_sleepable_first"))
+ run_test("on_open_sleepable_first");
+
if (test__start_subtest("negative"))
RUN_TESTS(file_reader_fail);
}
diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c
index aa2c05cce2b3..8b972fd26d73 100644
--- a/tools/testing/selftests/bpf/progs/file_reader.c
+++ b/tools/testing/selftests/bpf/progs/file_reader.c
@@ -27,9 +27,14 @@ char tmp_buf[256000];
int pid = 0;
int err, run_success = 0;
+__u64 beyond_eof_offset;
static int validate_file_read(struct file *file);
static int task_work_callback(struct bpf_map *map, void *key, void *value);
+static int sleepable_second_callback(struct bpf_map *map, void *key, void *value);
+
+void bpf_rcu_read_lock(void) __ksym;
+void bpf_rcu_read_unlock(void) __ksym;
SEC("lsm/file_open")
int on_open_expect_fault(void *c)
@@ -81,6 +86,101 @@ int on_open_validate_file_read(void *c)
return 0;
}
+/*
+ * Exercise bpf_dynptr_from_file() first from a non-sleepable LSM program and
+ * then from its sleepable task-work callback. Reading beyond EOF makes the two
+ * backing implementations return different errors.
+ */
+SEC("lsm/file_open")
+int on_open_non_sleepable_first(void *c)
+{
+ struct task_struct *task = bpf_get_current_task_btf();
+ struct bpf_dynptr dynptr;
+ struct elem *work;
+ struct file *file;
+ int key = 0;
+ int ret;
+
+ if (bpf_get_current_pid_tgid() >> 32 != pid)
+ return 0;
+
+ file = bpf_get_task_exe_file(task);
+ if (!file) {
+ err = 1;
+ return 0;
+ }
+
+ /* The non-sleepable reader cannot fault in an uncached folio. */
+ ret = bpf_dynptr_from_file(file, 0, &dynptr);
+ if (!ret)
+ ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0);
+ bpf_dynptr_file_discard(&dynptr);
+ bpf_put_file(file);
+ if (ret != -EFAULT) {
+ err = 2;
+ return 0;
+ }
+
+ work = bpf_map_lookup_elem(&arrmap, &key);
+ if (!work) {
+ err = 3;
+ return 0;
+ }
+
+ ret = bpf_task_work_schedule_signal(task, &work->tw, &arrmap,
+ sleepable_second_callback);
+ if (ret)
+ err = 4;
+ return 0;
+}
+
+/*
+ * Exercise the opposite fixup order: the first call is made from a sleepable
+ * LSM program, while the RCU read-side section makes the second non-sleepable.
+ */
+SEC("lsm.s/file_open")
+int on_open_sleepable_first(void *c)
+{
+ struct task_struct *task = bpf_get_current_task_btf();
+ struct bpf_dynptr dynptr;
+ struct file *file;
+ int ret;
+
+ if (bpf_get_current_pid_tgid() >> 32 != pid)
+ return 0;
+
+ file = bpf_get_task_exe_file(task);
+ if (!file) {
+ err = 7;
+ return 0;
+ }
+
+ ret = bpf_dynptr_from_file(file, 0, &dynptr);
+ if (!ret)
+ ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0);
+ bpf_dynptr_file_discard(&dynptr);
+ if (ret != -EIO) {
+ err = 8;
+ goto out;
+ }
+
+ bpf_rcu_read_lock();
+ ret = bpf_dynptr_from_file(file, 0, &dynptr);
+ bpf_rcu_read_unlock();
+ if (!ret)
+ ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0);
+ bpf_dynptr_file_discard(&dynptr);
+ if (ret != -EFAULT) {
+ err = 9;
+ goto out;
+ }
+
+ run_success = 1;
+out:
+ bpf_put_file(file);
+ return 0;
+}
+
/* Called in a sleepable context, read 256K bytes, cross check with user space read data */
static int task_work_callback(struct bpf_map *map, void *key, void *value)
{
@@ -97,6 +197,35 @@ static int task_work_callback(struct bpf_map *map, void *key, void *value)
return 0;
}
+/* Task-work callbacks are verified as sleepable. */
+static int sleepable_second_callback(struct bpf_map *map, void *key, void *value)
+{
+ struct task_struct *task = bpf_get_current_task_btf();
+ struct bpf_dynptr dynptr;
+ struct file *file;
+ int ret;
+
+ file = bpf_get_task_exe_file(task);
+ if (!file) {
+ err = 5;
+ return 0;
+ }
+
+ /* freader_fetch() converts __kernel_read()'s short read at EOF to -EIO. */
+ ret = bpf_dynptr_from_file(file, 0, &dynptr);
+ if (!ret)
+ ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0);
+ bpf_dynptr_file_discard(&dynptr);
+ bpf_put_file(file);
+ if (ret != -EIO) {
+ err = 6;
+ return 0;
+ }
+
+ run_success = 1;
+ return 0;
+}
+
static int verify_dynptr_read(struct bpf_dynptr *ptr, u32 off, char *user_buf, u32 len)
{
int i;
--
2.52.0
^ permalink raw reply related [flat|nested] 10+ messages in thread