* [PATCHv3 bpf-next 01/12] bpf: Factor stackid_init function from __bpf_get_stackid
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 02/12] bpf: Factor stackid_fastpath " Jiri Olsa
` (10 subsequent siblings)
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
The new stackid_init function stores all the necessary bits for stackid
trace and it will be used by other functions in following changes.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 95 +++++++++++++++++++++++++++----------------
1 file changed, 59 insertions(+), 36 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 463f94ba1cc4..19f9ea605a3e 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -504,33 +504,54 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
#endif
}
-static long __bpf_get_stackid(struct bpf_map *map,
- struct perf_callchain_entry *trace, u64 flags)
+struct stackid {
+ struct stack_map_bucket *bucket;
+ u64 *ips;
+ u32 nr;
+ u32 len;
+ u32 hash;
+ u32 id;
+};
+
+static int stackid_init(struct stackid *stackid, struct bpf_map *map,
+ struct perf_callchain_entry *trace, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
- struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
- u32 hash, id, trace_nr, trace_len, i, max_depth;
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
- bool user = flags & BPF_F_USER_STACK;
- u64 *ips;
- bool hash_matches;
+ u32 max_depth;
if (trace->nr <= skip)
/* skipping more than usable stack trace */
return -EFAULT;
max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
- trace_nr = min_t(u32, trace->nr - skip, max_depth - skip);
- trace_len = trace_nr * sizeof(u64);
- ips = trace->ip + skip;
- hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
- id = hash & (smap->n_buckets - 1);
- bucket = READ_ONCE(smap->buckets[id]);
+ stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip);
+ stackid->len = stackid->nr * sizeof(u64);
+ stackid->ips = trace->ip + skip;
+ stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
+ stackid->id = stackid->hash & (smap->n_buckets - 1);
+ stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
+ return 0;
+}
- hash_matches = bucket && bucket->hash == hash;
+static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
+ struct perf_callchain_entry *trace, u64 flags)
+{
+ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+ struct stack_map_bucket *new_bucket, *old_bucket;
+ bool user = flags & BPF_F_USER_STACK;
+ bool hash_matches;
+ u32 trace_len, i;
+ int err;
+
+ err = stackid_init(stackid, map, trace, flags);
+ if (err)
+ return err;
+
+ hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
/* fast cmp */
if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
- return id;
+ return stackid->id;
if (stack_map_use_build_id(map)) {
struct bpf_stack_build_id *id_offs;
@@ -540,42 +561,42 @@ static long __bpf_get_stackid(struct bpf_map *map,
pcpu_freelist_pop(&smap->freelist);
if (unlikely(!new_bucket))
return -ENOMEM;
- new_bucket->nr = trace_nr;
+ new_bucket->nr = stackid->nr;
id_offs = (struct bpf_stack_build_id *)new_bucket->data;
- for (i = 0; i < trace_nr; i++)
- id_offs[i].ip = ips[i];
- stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */);
- trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
- if (hash_matches && bucket->nr == trace_nr &&
- memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
+ for (i = 0; i < stackid->nr; i++)
+ id_offs[i].ip = stackid->ips[i];
+ stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
+ trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
+ if (hash_matches && stackid->bucket->nr == stackid->nr &&
+ memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) {
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
- return id;
+ return stackid->id;
}
- if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
+ if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) {
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
return -EEXIST;
}
} else {
- if (hash_matches && bucket->nr == trace_nr &&
- memcmp(bucket->data, ips, trace_len) == 0)
- return id;
- if (bucket && !(flags & BPF_F_REUSE_STACKID))
+ if (hash_matches && stackid->bucket->nr == stackid->nr &&
+ memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
+ return stackid->id;
+ if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
return -EEXIST;
new_bucket = (struct stack_map_bucket *)
pcpu_freelist_pop(&smap->freelist);
if (unlikely(!new_bucket))
return -ENOMEM;
- memcpy(new_bucket->data, ips, trace_len);
+ memcpy(new_bucket->data, stackid->ips, stackid->len);
}
- new_bucket->hash = hash;
- new_bucket->nr = trace_nr;
+ new_bucket->hash = stackid->hash;
+ new_bucket->nr = stackid->nr;
- old_bucket = xchg(&smap->buckets[id], new_bucket);
+ old_bucket = xchg(&smap->buckets[stackid->id], new_bucket);
if (old_bucket)
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
- return id;
+ return stackid->id;
}
BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
@@ -584,6 +605,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
u32 elem_size = stack_map_data_size(map);
bool user = flags & BPF_F_USER_STACK;
struct perf_callchain_entry *trace;
+ struct stackid stackid;
bool kernel = !user;
u32 max_depth;
@@ -599,7 +621,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
/* couldn't fetch the stack trace */
return -EFAULT;
- return __bpf_get_stackid(map, trace, flags);
+ return __bpf_get_stackid(&stackid, map, trace, flags);
}
const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -628,6 +650,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
{
struct perf_event *event = ctx->event;
struct perf_callchain_entry *trace;
+ struct stackid stackid;
bool kernel, user;
__u64 nr_kernel;
int ret;
@@ -653,7 +676,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
if (kernel) {
trace->nr = nr_kernel;
- ret = __bpf_get_stackid(map, trace, flags);
+ ret = __bpf_get_stackid(&stackid, map, trace, flags);
} else { /* user */
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
@@ -662,7 +685,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
return -EFAULT;
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
- ret = __bpf_get_stackid(map, trace, flags);
+ ret = __bpf_get_stackid(&stackid, map, trace, flags);
}
/* restore nr */
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 02/12] bpf: Factor stackid_fastpath function from __bpf_get_stackid
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 01/12] bpf: Factor stackid_init function from __bpf_get_stackid Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 22:21 ` bot+bpf-ci
2026-08-03 21:01 ` [PATCHv3 bpf-next 03/12] bpf: Factor stackid_new_bucket " Jiri Olsa
` (9 subsequent siblings)
11 siblings, 1 reply; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
The new stackid_fastpath does the fast stack hash and trace check, that
does not need new bucket allocation. It covers both just-ip and buildid
code paths.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 43 ++++++++++++++++++++++++++++---------------
1 file changed, 28 insertions(+), 15 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 19f9ea605a3e..27210b5d16fc 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -511,6 +511,7 @@ struct stackid {
u32 len;
u32 hash;
u32 id;
+ bool hash_matches;
};
static int stackid_init(struct stackid *stackid, struct bpf_map *map,
@@ -531,28 +532,46 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
stackid->id = stackid->hash & (smap->n_buckets - 1);
stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
+ stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
return 0;
}
+static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
+ struct perf_callchain_entry *trace, u64 flags)
+{
+ int err;
+
+ err = stackid_init(stackid, map, trace, flags);
+ if (err)
+ return err;
+
+ /* fast cmp */
+ if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP)
+ return stackid->id;
+
+ if (stack_map_use_build_id(map))
+ return -ENOENT;
+ if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
+ memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
+ return stackid->id;
+ if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
+ return -EEXIST;
+ return -ENOENT;
+}
+
static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
struct perf_callchain_entry *trace, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
struct stack_map_bucket *new_bucket, *old_bucket;
bool user = flags & BPF_F_USER_STACK;
- bool hash_matches;
u32 trace_len, i;
int err;
- err = stackid_init(stackid, map, trace, flags);
- if (err)
+ err = stackid_fastpath(stackid, map, trace, flags);
+ if (err != -ENOENT)
return err;
- hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
- /* fast cmp */
- if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
- return stackid->id;
-
if (stack_map_use_build_id(map)) {
struct bpf_stack_build_id *id_offs;
@@ -567,7 +586,7 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
id_offs[i].ip = stackid->ips[i];
stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
- if (hash_matches && stackid->bucket->nr == stackid->nr &&
+ if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) {
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
return stackid->id;
@@ -577,12 +596,6 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
return -EEXIST;
}
} else {
- if (hash_matches && stackid->bucket->nr == stackid->nr &&
- memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0)
- return stackid->id;
- if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID))
- return -EEXIST;
-
new_bucket = (struct stack_map_bucket *)
pcpu_freelist_pop(&smap->freelist);
if (unlikely(!new_bucket))
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCHv3 bpf-next 02/12] bpf: Factor stackid_fastpath function from __bpf_get_stackid
2026-08-03 21:01 ` [PATCHv3 bpf-next 02/12] bpf: Factor stackid_fastpath " Jiri Olsa
@ 2026-08-03 22:21 ` bot+bpf-ci
0 siblings, 0 replies; 21+ messages in thread
From: bot+bpf-ci @ 2026-08-03 22:21 UTC (permalink / raw)
To: jolsa, ast, daniel, andrii
Cc: bpf, martin.lau, eddyz87, songliubraving, yhs, qmo, chen.dylane,
info, contact, ast, andrii, daniel, martin.lau, eddyz87,
yonghong.song, clm, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 1227 bytes --]
> bpf: Factor stackid_fastpath function from __bpf_get_stackid
>
> The new stackid_fastpath does the fast stack hash and trace check, that
> does not need new bucket allocation. It covers both just-ip and buildid
> code paths.
>
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
This isn't a bug, but would it help to note in the changelog that -ENOENT
is the 'fall through to the slow path' return, and that hash_matches moves
into struct stackid so the slow path can still see it?
The caller depends on exactly that in:
> diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
> @@ -556,57 +575,45 @@ static long __bpf_get_stackid
>
> - err = stackid_init(stackid, map, trace, flags);
> - if (err)
> + err = stackid_fastpath(stackid, map, trace, flags);
> + if (err != -ENOENT)
> return err;
and the new bool hash_matches field exists so the predicate survives across
the function boundary. A 'no functional change intended' line would also
make the refactor easier to review.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/30854471987
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCHv3 bpf-next 03/12] bpf: Factor stackid_new_bucket from __bpf_get_stackid
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 01/12] bpf: Factor stackid_init function from __bpf_get_stackid Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 02/12] bpf: Factor stackid_fastpath " Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 04/12] bpf: Use stack id functions instead of __bpf_get_stackid Jiri Olsa
` (8 subsequent siblings)
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
The new stackid_new_bucket allocates the new bucket and initializes it
with the trace data.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 48 +++++++++++++++++++++++++++----------------
1 file changed, 30 insertions(+), 18 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 27210b5d16fc..d930d9754d7d 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -559,31 +559,52 @@ static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
return -ENOENT;
}
+static struct stack_map_bucket *
+stackid_new_bucket(struct stackid *stackid, struct bpf_map *map)
+{
+ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
+ struct bpf_stack_build_id *id_offs;
+ struct stack_map_bucket *bucket;
+ u32 i;
+
+ bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist);
+ if (unlikely(!bucket))
+ return NULL;
+
+ if (stack_map_use_build_id(map)) {
+ id_offs = (struct bpf_stack_build_id *)bucket->data;
+ for (i = 0; i < stackid->nr; i++)
+ id_offs[i].ip = stackid->ips[i];
+ } else {
+ memcpy(bucket->data, stackid->ips, stackid->len);
+ }
+
+ bucket->hash = stackid->hash;
+ bucket->nr = stackid->nr;
+ return bucket;
+}
+
static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
struct perf_callchain_entry *trace, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
struct stack_map_bucket *new_bucket, *old_bucket;
bool user = flags & BPF_F_USER_STACK;
- u32 trace_len, i;
+ u32 trace_len;
int err;
err = stackid_fastpath(stackid, map, trace, flags);
if (err != -ENOENT)
return err;
+ new_bucket = stackid_new_bucket(stackid, map);
+ if (!new_bucket)
+ return -ENOMEM;
+
if (stack_map_use_build_id(map)) {
struct bpf_stack_build_id *id_offs;
- /* for build_id+offset, pop a bucket before slow cmp */
- new_bucket = (struct stack_map_bucket *)
- pcpu_freelist_pop(&smap->freelist);
- if (unlikely(!new_bucket))
- return -ENOMEM;
- new_bucket->nr = stackid->nr;
id_offs = (struct bpf_stack_build_id *)new_bucket->data;
- for (i = 0; i < stackid->nr; i++)
- id_offs[i].ip = stackid->ips[i];
stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */);
trace_len = stackid->nr * sizeof(struct bpf_stack_build_id);
if (stackid->hash_matches && stackid->bucket->nr == stackid->nr &&
@@ -595,17 +616,8 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
return -EEXIST;
}
- } else {
- new_bucket = (struct stack_map_bucket *)
- pcpu_freelist_pop(&smap->freelist);
- if (unlikely(!new_bucket))
- return -ENOMEM;
- memcpy(new_bucket->data, stackid->ips, stackid->len);
}
- new_bucket->hash = stackid->hash;
- new_bucket->nr = stackid->nr;
-
old_bucket = xchg(&smap->buckets[stackid->id], new_bucket);
if (old_bucket)
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 04/12] bpf: Use stack id functions instead of __bpf_get_stackid
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (2 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 03/12] bpf: Factor stackid_new_bucket " Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 05/12] bpf: Disable preemption in bpf_get_stackid Jiri Olsa
` (7 subsequent siblings)
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
Replacing __bpf_get_stackid calls with sequence of following functions:
stackid_fastpath
stackid_new_bucket
stackid_install
This makes code more structured and allows us to easily disable
preemption only in bpf_get_stackid in following changes.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 43 +++++++++++++++++++++++++++----------------
1 file changed, 27 insertions(+), 16 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index d930d9754d7d..3ee0034daf52 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -584,22 +584,13 @@ stackid_new_bucket(struct stackid *stackid, struct bpf_map *map)
return bucket;
}
-static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map,
- struct perf_callchain_entry *trace, u64 flags)
+static long stackid_install(struct stackid *stackid, struct bpf_map *map,
+ struct stack_map_bucket *new_bucket, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
- struct stack_map_bucket *new_bucket, *old_bucket;
bool user = flags & BPF_F_USER_STACK;
+ struct stack_map_bucket *old_bucket;
u32 trace_len;
- int err;
-
- err = stackid_fastpath(stackid, map, trace, flags);
- if (err != -ENOENT)
- return err;
-
- new_bucket = stackid_new_bucket(stackid, map);
- if (!new_bucket)
- return -ENOMEM;
if (stack_map_use_build_id(map)) {
struct bpf_stack_build_id *id_offs;
@@ -629,10 +620,12 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
{
u32 elem_size = stack_map_data_size(map);
bool user = flags & BPF_F_USER_STACK;
+ struct stack_map_bucket *new_bucket;
struct perf_callchain_entry *trace;
struct stackid stackid;
bool kernel = !user;
u32 max_depth;
+ int err;
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
@@ -646,7 +639,15 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
/* couldn't fetch the stack trace */
return -EFAULT;
- return __bpf_get_stackid(&stackid, map, trace, flags);
+ err = stackid_fastpath(&stackid, map, trace, flags);
+ if (err != -ENOENT)
+ return err;
+
+ new_bucket = stackid_new_bucket(&stackid, map);
+ if (!new_bucket)
+ return -ENOMEM;
+
+ return stackid_install(&stackid, map, new_bucket, flags);
}
const struct bpf_func_proto bpf_get_stackid_proto = {
@@ -674,6 +675,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
struct bpf_map *, map, u64, flags)
{
struct perf_event *event = ctx->event;
+ struct stack_map_bucket *new_bucket;
struct perf_callchain_entry *trace;
struct stackid stackid;
bool kernel, user;
@@ -701,7 +703,6 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
if (kernel) {
trace->nr = nr_kernel;
- ret = __bpf_get_stackid(&stackid, map, trace, flags);
} else { /* user */
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
@@ -710,12 +711,22 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
return -EFAULT;
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
- ret = __bpf_get_stackid(&stackid, map, trace, flags);
}
+ ret = stackid_fastpath(&stackid, map, trace, flags);
+ if (ret != -ENOENT)
+ goto out;
+
+ new_bucket = stackid_new_bucket(&stackid, map);
+ if (new_bucket) {
+ trace->nr = nr;
+ return stackid_install(&stackid, map, new_bucket, flags);
+ }
+ ret = -ENOMEM;
+
+out:
/* restore nr */
trace->nr = nr;
-
return ret;
}
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 05/12] bpf: Disable preemption in bpf_get_stackid
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (3 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 04/12] bpf: Use stack id functions instead of __bpf_get_stackid Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack Jiri Olsa
` (6 subsequent siblings)
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: stable, Tao Chen, bpf, Martin KaFai Lau, Eduard Zingerman,
Song Liu, Yonghong Song, Quentin Monnet, STAR Labs SG,
Arnaud Lecomte
The get_perf_callchain call needs disabled preemption plus we need
it disabled as long as we access its returned trace entries buffer.
Note the bpf_get_stackid_pe function is executed already with
preemption disabled.
Cc: stable@vger.kernel.org
Fixes: d5a3b1f69186 ("bpf: introduce BPF_MAP_TYPE_STACK_TRACE")
Reported-by: Tao Chen <chen.dylane@linux.dev>
Closes: https://lore.kernel.org/bpf/20260206090653.1336687-2-chen.dylane@linux.dev/
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 24 +++++++++++++-----------
1 file changed, 13 insertions(+), 11 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 3ee0034daf52..5b18d728f4b8 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -632,20 +632,22 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
return -EINVAL;
max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags);
- trace = get_perf_callchain(regs, kernel, user, max_depth,
- false, false, 0);
- if (unlikely(!trace))
- /* couldn't fetch the stack trace */
- return -EFAULT;
+ scoped_guard(preempt) {
+ trace = get_perf_callchain(regs, kernel, user, max_depth,
+ false, false, 0);
+ if (unlikely(!trace))
+ /* couldn't fetch the stack trace */
+ return -EFAULT;
- err = stackid_fastpath(&stackid, map, trace, flags);
- if (err != -ENOENT)
- return err;
+ err = stackid_fastpath(&stackid, map, trace, flags);
+ if (err != -ENOENT)
+ return err;
- new_bucket = stackid_new_bucket(&stackid, map);
- if (!new_bucket)
- return -ENOMEM;
+ new_bucket = stackid_new_bucket(&stackid, map);
+ if (!new_bucket)
+ return -ENOMEM;
+ }
return stackid_install(&stackid, map, new_bucket, flags);
}
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (4 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 05/12] bpf: Disable preemption in bpf_get_stackid Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:30 ` sashiko-bot
2026-08-03 22:06 ` bot+bpf-ci
2026-08-03 21:01 ` [PATCHv3 bpf-next 07/12] bpf: Factor callchain_finalize " Jiri Olsa
` (5 subsequent siblings)
11 siblings, 2 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
The new callchain_store function stores trace entries buffer into
user supplied buffer. It covers both just-ip and buildid data.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 39 +++++++++++++++++++++++++--------------
1 file changed, 25 insertions(+), 14 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 5b18d728f4b8..ee9905d67b8e 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -741,6 +741,29 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
.arg3_type = ARG_ANYTHING,
};
+static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
+ u32 elem_size, u64 flags)
+{
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ u32 trace_nr, copy_len;
+ u64 *ips;
+
+ trace_nr = trace->nr - skip;
+ copy_len = trace_nr * elem_size;
+
+ ips = trace->ip + skip;
+ if (user_build_id) {
+ struct bpf_stack_build_id *id_offs = buf;
+
+ for (u32 i = 0; i < trace_nr; i++)
+ id_offs[i].ip = ips[i];
+ } else {
+ memcpy(buf, ips, copy_len);
+ }
+ return trace_nr;
+}
+
static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
struct perf_callchain_entry *trace_in,
void *buf, u32 size, u64 flags, bool may_fault)
@@ -753,7 +776,6 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
struct perf_callchain_entry *trace;
bool kernel = !user;
int err = -EINVAL;
- u64 *ips;
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_USER_BUILD_ID)))
@@ -798,21 +820,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
goto err_fault;
}
- trace_nr = trace->nr - skip;
+ trace_nr = callchain_store(trace, buf, elem_size, flags);
copy_len = trace_nr * elem_size;
- ips = trace->ip + skip;
- if (user_build_id) {
- struct bpf_stack_build_id *id_offs = buf;
- u32 i;
-
- for (i = 0; i < trace_nr; i++)
- id_offs[i].ip = ips[i];
- } else {
- memcpy(buf, ips, copy_len);
- }
-
- /* trace/ips should not be dereferenced after this point */
+ /* trace should not be dereferenced after this point */
if (may_fault)
rcu_read_unlock();
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack
2026-08-03 21:01 ` [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack Jiri Olsa
@ 2026-08-03 21:30 ` sashiko-bot
2026-08-04 12:02 ` Jiri Olsa
2026-08-03 22:06 ` bot+bpf-ci
1 sibling, 1 reply; 21+ messages in thread
From: sashiko-bot @ 2026-08-03 21:30 UTC (permalink / raw)
To: Jiri Olsa; +Cc: bpf
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
Pre-existing issues:
- [Critical] A Time-of-Check to Time-of-Use (TOCTOU) data race exists in `__bpf_get_stack` when used from a sleepable BPF program, allowing concurrent modification of `trace->nr` that leads to a massive Out-of-Bounds (OOB) memory read and write.
--
--- Patch [1]: [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack ---
commit 1569be7d337e247b4d7eb8620dff965cc57a3fc6
Author: Jiri Olsa <jolsa@kernel.org>
bpf: Factor callchain_store function from __bpf_get_stack
This commit factors out the callchain storage logic into a new
callchain_store() function. It handles extracting both just-ip and buildid
data into a user-supplied buffer.
> diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
> index 463f94ba1cc47..e51e03c4b60ed 100644
> --- a/kernel/bpf/stackmap.c
> +++ b/kernel/bpf/stackmap.c
[ ... ]
> @@ -680,6 +680,29 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
> .arg3_type = ARG_ANYTHING,
> };
>
> +static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
> + u32 elem_size, u64 flags)
> +{
> + bool user_build_id = flags & BPF_F_USER_BUILD_ID;
> + u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
> + u32 trace_nr, copy_len;
> + u64 *ips;
> +
> + trace_nr = trace->nr - skip;
[Severity: Critical]
This is a pre-existing issue, but could this unprotected re-read of
trace->nr lead to an integer underflow and out-of-bounds memory access?
If __bpf_get_stack() is called from a sleepable BPF program, preemption
is enabled. Between the bounds check in __bpf_get_stack() and this read,
the task could be preempted. If a preempting task running on the same CPU
executes a BPF program and reallocates the identical per-CPU buffer, it
might overwrite trace->nr with a smaller value.
When the sleepable task resumes, this re-read of trace->nr could result in
a value smaller than skip, causing trace_nr to underflow and resulting in
a massive copy_len during the memcpy() below.
> + copy_len = trace_nr * elem_size;
> +
> + ips = trace->ip + skip;
> + if (user_build_id) {
> + struct bpf_stack_build_id *id_offs = buf;
> +
> + for (u32 i = 0; i < trace_nr; i++)
> + id_offs[i].ip = ips[i];
> + } else {
> + memcpy(buf, ips, copy_len);
> + }
> + return trace_nr;
> +}
> +
> static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
[ ... ]
> @@ -737,21 +759,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
> goto err_fault;
> }
>
> - trace_nr = trace->nr - skip;
> + trace_nr = callchain_store(trace, buf, elem_size, flags);
[Severity: Critical]
This isn't a bug introduced by this patch, but this is the preemption
window mentioned above.
The check if (unlikely(!trace) || trace->nr < skip) happens right before
this call. For sleepable BPF programs, preemption can occur between the
bounds check and the re-reading of trace->nr inside callchain_store().
Could we cache the validated trace->nr value (e.g., using READ_ONCE())
and pass it to callchain_store() to avoid this Time-of-Check to
Time-of-Use race?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260803210149.296496-7-jolsa@kernel.org?part=1
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack
2026-08-03 21:30 ` sashiko-bot
@ 2026-08-04 12:02 ` Jiri Olsa
0 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-04 12:02 UTC (permalink / raw)
To: sashiko-reviews; +Cc: bpf
On Mon, Aug 03, 2026 at 09:30:50PM +0000, sashiko-bot@kernel.org wrote:
> Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
>
> Pre-existing issues:
> - [Critical] A Time-of-Check to Time-of-Use (TOCTOU) data race exists in `__bpf_get_stack` when used from a sleepable BPF program, allowing concurrent modification of `trace->nr` that leads to a massive Out-of-Bounds (OOB) memory read and write.
> --
>
> --- Patch [1]: [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack ---
> commit 1569be7d337e247b4d7eb8620dff965cc57a3fc6
> Author: Jiri Olsa <jolsa@kernel.org>
> bpf: Factor callchain_store function from __bpf_get_stack
>
> This commit factors out the callchain storage logic into a new
> callchain_store() function. It handles extracting both just-ip and buildid
> data into a user-supplied buffer.
>
> > diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
> > index 463f94ba1cc47..e51e03c4b60ed 100644
> > --- a/kernel/bpf/stackmap.c
> > +++ b/kernel/bpf/stackmap.c
> [ ... ]
> > @@ -680,6 +680,29 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
> > .arg3_type = ARG_ANYTHING,
> > };
> >
> > +static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
> > + u32 elem_size, u64 flags)
> > +{
> > + bool user_build_id = flags & BPF_F_USER_BUILD_ID;
> > + u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
> > + u32 trace_nr, copy_len;
> > + u64 *ips;
> > +
> > + trace_nr = trace->nr - skip;
>
> [Severity: Critical]
> This is a pre-existing issue, but could this unprotected re-read of
> trace->nr lead to an integer underflow and out-of-bounds memory access?
>
> If __bpf_get_stack() is called from a sleepable BPF program, preemption
> is enabled. Between the bounds check in __bpf_get_stack() and this read,
> the task could be preempted. If a preempting task running on the same CPU
> executes a BPF program and reallocates the identical per-CPU buffer, it
> might overwrite trace->nr with a smaller value.
>
> When the sleepable task resumes, this re-read of trace->nr could result in
> a value smaller than skip, causing trace_nr to underflow and resulting in
> a massive copy_len during the memcpy() below.
>
> > + copy_len = trace_nr * elem_size;
> > +
> > + ips = trace->ip + skip;
> > + if (user_build_id) {
> > + struct bpf_stack_build_id *id_offs = buf;
> > +
> > + for (u32 i = 0; i < trace_nr; i++)
> > + id_offs[i].ip = ips[i];
> > + } else {
> > + memcpy(buf, ips, copy_len);
> > + }
> > + return trace_nr;
> > +}
> > +
> > static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
> [ ... ]
> > @@ -737,21 +759,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
> > goto err_fault;
> > }
> >
> > - trace_nr = trace->nr - skip;
> > + trace_nr = callchain_store(trace, buf, elem_size, flags);
>
> [Severity: Critical]
> This isn't a bug introduced by this patch, but this is the preemption
> window mentioned above.
>
> The check if (unlikely(!trace) || trace->nr < skip) happens right before
> this call. For sleepable BPF programs, preemption can occur between the
> bounds check and the re-reading of trace->nr inside callchain_store().
>
> Could we cache the validated trace->nr value (e.g., using READ_ONCE())
> and pass it to callchain_store() to avoid this Time-of-Check to
> Time-of-Use race?
there's preemption fix for this later in the patchset
jirka
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack
2026-08-03 21:01 ` [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack Jiri Olsa
2026-08-03 21:30 ` sashiko-bot
@ 2026-08-03 22:06 ` bot+bpf-ci
1 sibling, 0 replies; 21+ messages in thread
From: bot+bpf-ci @ 2026-08-03 22:06 UTC (permalink / raw)
To: jolsa, ast, daniel, andrii
Cc: bpf, martin.lau, eddyz87, songliubraving, yhs, qmo, chen.dylane,
info, contact, ast, andrii, daniel, martin.lau, eddyz87,
yonghong.song, clm, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2729 bytes --]
> bpf: Factor callchain_store function from __bpf_get_stack
>
> The new callchain_store function stores trace entries buffer into
> user supplied buffer. It covers both just-ip and buildid data.
This isn't a bug, but could this changelog mention the preemption fix it
is preparing for, the way the next two patches in the series do? The two
immediately following commits (7199cef3355ad and d872b6f929f3e) both
explicitly state they are making things easier for the preemption fix in
a following change, and the series does land that fix in 2c464d58d2665.
Also, 'user supplied buffer' reads as a userspace buffer, whereas buf
here is a kernel buffer supplied by the calling BPF helper. Could it say
'caller supplied buffer' to avoid that reading?
> diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
> index 5b18d728f4b88..ee9905d67b8ec 100644
> --- a/kernel/bpf/stackmap.c
> +++ b/kernel/bpf/stackmap.c
> @@ -741,6 +741,29 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
> .arg3_type = ARG_ANYTHING,
> };
>
> +static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
> + u32 elem_size, u64 flags)
This isn't a bug, but since perf_callchain_store() appends an IP into a
callchain and this one copies IPs out of one, would a name like
callchain_copy_out() (or a stack_map_ prefix, matching the other helpers
in this file) be less easy to mix up? The new static helper is named
callchain_store(), which is one word away from perf_callchain_store() in
include/linux/perf_event.h (already reachable in this file via the
include at the top). The two have opposite data direction:
perf_callchain_store(ctx, ip) appends a single IP into a callchain
entry, whereas the new callchain_store(trace, buf, ...) copies the whole
IP array out of a callchain entry. Every other file-local helper in
stackmap.c carries a subsystem prefix that signals its domain (stack_map_*
for the build-id/map helpers, stackid_* for the stackid helpers added
earlier in this series).
> +{
> + bool user_build_id = flags & BPF_F_USER_BUILD_ID;
> + u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
> + u32 trace_nr, copy_len;
> + u64 *ips;
> +
> + trace_nr = trace->nr - skip;
> + copy_len = trace_nr * elem_size;
> +
> + ips = trace->ip + skip;
> + if (user_build_id) {
> + struct bpf_stack_build_id *id_offs = buf;
> +
> + for (u32 i = 0; i < trace_nr; i++)
> + id_offs[i].ip = ips[i];
> + } else {
> + memcpy(buf, ips, copy_len);
> + }
> + return trace_nr;
> +}
[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/30854471987
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCHv3 bpf-next 07/12] bpf: Factor callchain_finalize function from __bpf_get_stack
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (5 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 06/12] bpf: Factor callchain_store function from __bpf_get_stack Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 08/12] bpf: Remove trace_in argument " Jiri Olsa
` (4 subsequent siblings)
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
The new callchain_finalize function calls the build-id retrieval
(if needed) and zeroes the buffer. This makes things easier for
preemption fix in following change.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index ee9905d67b8e..cdeb6c2e50da 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -764,16 +764,31 @@ static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
return trace_nr;
}
+static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size,
+ u64 flags, bool may_fault)
+{
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ bool user = flags & BPF_F_USER_STACK;
+ u32 copy_len = trace_nr * elem_size;
+
+ if (user_build_id)
+ stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
+
+ if (size > copy_len)
+ memset(buf + copy_len, 0, size - copy_len);
+ return copy_len;
+}
+
static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
struct perf_callchain_entry *trace_in,
void *buf, u32 size, u64 flags, bool may_fault)
{
- u32 trace_nr, copy_len, elem_size, max_depth;
bool user_build_id = flags & BPF_F_USER_BUILD_ID;
bool crosstask = task && task != current;
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
bool user = flags & BPF_F_USER_STACK;
struct perf_callchain_entry *trace;
+ u32 trace_nr, elem_size, max_depth;
bool kernel = !user;
int err = -EINVAL;
@@ -821,18 +836,12 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
}
trace_nr = callchain_store(trace, buf, elem_size, flags);
- copy_len = trace_nr * elem_size;
/* trace should not be dereferenced after this point */
if (may_fault)
rcu_read_unlock();
- if (user_build_id)
- stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
-
- if (size > copy_len)
- memset(buf + copy_len, 0, size - copy_len);
- return copy_len;
+ return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
err_fault:
err = -EFAULT;
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 08/12] bpf: Remove trace_in argument from __bpf_get_stack
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (6 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 07/12] bpf: Factor callchain_finalize " Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 22:21 ` bot+bpf-ci
2026-08-03 21:01 ` [PATCHv3 bpf-next 09/12] bpf: Clear buf on error in __bpf_get_task_stack Jiri Olsa
` (3 subsequent siblings)
11 siblings, 1 reply; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
Now with the new callchain_* helper functions we can process trace_in
case directly in bpf_get_stack_pe function and remove it from
__bpf_get_stack which makes things easier for preemption fix in
following change.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 49 ++++++++++++++++++++++++++++++-------------
1 file changed, 35 insertions(+), 14 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index cdeb6c2e50da..976c4e4c1af6 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -780,7 +780,6 @@ static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size,
}
static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
- struct perf_callchain_entry *trace_in,
void *buf, u32 size, u64 flags, bool may_fault)
{
bool user_build_id = flags & BPF_F_USER_BUILD_ID;
@@ -819,10 +818,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
if (may_fault)
rcu_read_lock(); /* need RCU for perf's callchain below */
- if (trace_in) {
- trace = trace_in;
- trace->nr = min_t(u32, trace->nr, max_depth);
- } else if (kernel && task) {
+ if (kernel && task) {
trace = get_callchain_entry_for_task(task, max_depth);
} else {
trace = get_perf_callchain(regs, kernel, user, max_depth,
@@ -853,7 +849,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
u64, flags)
{
- return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
+ return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */);
}
const struct bpf_func_proto bpf_get_stack_proto = {
@@ -869,7 +865,7 @@ const struct bpf_func_proto bpf_get_stack_proto = {
BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size,
u64, flags)
{
- return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */);
+ return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */);
}
const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
@@ -893,7 +889,7 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
regs = task_pt_regs(task);
if (regs)
- res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault);
+ res = __bpf_get_stack(regs, task, buf, size, flags, may_fault);
put_task_stack(task);
return res;
@@ -933,6 +929,32 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
.arg4_type = ARG_ANYTHING,
};
+static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size,
+ u64 flags)
+{
+ bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+ u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ bool user = flags & BPF_F_USER_STACK;
+ u32 elem_size, max_depth, nr_trace;
+ bool kernel = !user;
+
+ if (kernel && user_build_id)
+ return -EINVAL;
+
+ elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
+ if (unlikely(size % elem_size))
+ return -EINVAL;
+
+ max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
+ trace->nr = min_t(u32, trace->nr, max_depth);
+
+ if (trace->nr < skip)
+ return -EFAULT;
+
+ nr_trace = callchain_store(trace, buf, elem_size, flags);
+ return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */);
+}
+
BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
void *, buf, u32, size, u64, flags)
{
@@ -944,7 +966,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
__u64 nr_kernel;
if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
- return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
+ return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */);
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_USER_BUILD_ID)))
@@ -964,7 +986,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
__u64 nr = trace->nr;
trace->nr = nr_kernel;
- err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
+ err = __bpf_get_stack_pe(trace, buf, size, flags);
/* restore nr */
trace->nr = nr;
@@ -974,14 +996,13 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
skip += nr_kernel;
if (skip > BPF_F_SKIP_FIELD_MASK)
goto clear;
-
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
- err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
+ err = __bpf_get_stack_pe(trace, buf, size, flags);
}
- return err;
clear:
- memset(buf, 0, size);
+ if (err < 0)
+ memset(buf, 0, size);
return err;
}
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCHv3 bpf-next 08/12] bpf: Remove trace_in argument from __bpf_get_stack
2026-08-03 21:01 ` [PATCHv3 bpf-next 08/12] bpf: Remove trace_in argument " Jiri Olsa
@ 2026-08-03 22:21 ` bot+bpf-ci
2026-08-04 20:05 ` Jiri Olsa
0 siblings, 1 reply; 21+ messages in thread
From: bot+bpf-ci @ 2026-08-03 22:21 UTC (permalink / raw)
To: jolsa, ast, daniel, andrii
Cc: bpf, martin.lau, eddyz87, songliubraving, yhs, qmo, chen.dylane,
info, contact, ast, andrii, daniel, martin.lau, eddyz87,
yonghong.song, clm, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 2233 bytes --]
> diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
> index cdeb6c2e50da0..976c4e4c1af66 100644
> --- a/kernel/bpf/stackmap.c
> +++ b/kernel/bpf/stackmap.c
[ ... ]
> @@ -933,6 +929,32 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
> .arg4_type = ARG_ANYTHING,
> };
>
> +static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size,
> + u64 flags)
> +{
> + bool user_build_id = flags & BPF_F_USER_BUILD_ID;
> + u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
> + bool user = flags & BPF_F_USER_STACK;
> + u32 elem_size, max_depth, nr_trace;
> + bool kernel = !user;
> +
> + if (kernel && user_build_id)
> + return -EINVAL;
> +
> + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
> + if (unlikely(size % elem_size))
> + return -EINVAL;
> +
> + max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
> + trace->nr = min_t(u32, trace->nr, max_depth);
> +
> + if (trace->nr < skip)
> + return -EFAULT;
> +
> + nr_trace = callchain_store(trace, buf, elem_size, flags);
> + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */);
> +}
This isn't a bug, but would naming this variable trace_nr match
__bpf_get_stack() and the callchain_*() parameters it feeds?
Looking at __bpf_get_stack():
kernel/bpf/stackmap.c:__bpf_get_stack() {
u32 trace_nr, elem_size, max_depth;
...
trace_nr = callchain_store(trace, buf, elem_size, flags);
return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
}
And the shared helpers both expect trace_nr:
kernel/bpf/stackmap.c:callchain_store(struct perf_callchain_entry *trace, ...) {
for (i = skip; i < trace_nr; i++) {
...
}
return trace_nr - skip;
}
kernel/bpf/stackmap.c:callchain_finalize(void *buf, u32 size, u32 trace_nr, ...) {
u32 copy_len = trace_nr * elem_size;
...
}
The new helper uses nr_trace for the same role, making the two otherwise
parallel functions read differently.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/30854471987
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCHv3 bpf-next 08/12] bpf: Remove trace_in argument from __bpf_get_stack
2026-08-03 22:21 ` bot+bpf-ci
@ 2026-08-04 20:05 ` Jiri Olsa
0 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-04 20:05 UTC (permalink / raw)
To: bot+bpf-ci
Cc: ast, daniel, andrii, bpf, martin.lau, eddyz87, songliubraving,
yhs, qmo, chen.dylane, info, contact, martin.lau, yonghong.song,
clm, ihor.solodrai
On Mon, Aug 03, 2026 at 10:21:34PM +0000, bot+bpf-ci@kernel.org wrote:
SNIP
> > + if (kernel && user_build_id)
> > + return -EINVAL;
> > +
> > + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
> > + if (unlikely(size % elem_size))
> > + return -EINVAL;
> > +
> > + max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
> > + trace->nr = min_t(u32, trace->nr, max_depth);
> > +
> > + if (trace->nr < skip)
> > + return -EFAULT;
> > +
> > + nr_trace = callchain_store(trace, buf, elem_size, flags);
> > + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */);
> > +}
>
> This isn't a bug, but would naming this variable trace_nr match
> __bpf_get_stack() and the callchain_*() parameters it feeds?
>
> Looking at __bpf_get_stack():
>
> kernel/bpf/stackmap.c:__bpf_get_stack() {
> u32 trace_nr, elem_size, max_depth;
> ...
> trace_nr = callchain_store(trace, buf, elem_size, flags);
> return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
> }
>
> And the shared helpers both expect trace_nr:
>
> kernel/bpf/stackmap.c:callchain_store(struct perf_callchain_entry *trace, ...) {
> for (i = skip; i < trace_nr; i++) {
> ...
> }
> return trace_nr - skip;
> }
>
> kernel/bpf/stackmap.c:callchain_finalize(void *buf, u32 size, u32 trace_nr, ...) {
> u32 copy_len = trace_nr * elem_size;
> ...
> }
>
> The new helper uses nr_trace for the same role, making the two otherwise
> parallel functions read differently.
that rename actualy uncovered that later __bpf_get_stack_pe used both
trace_nr and nr_trace.. we're fine with just one trace_nr, will change
jirka
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCHv3 bpf-next 09/12] bpf: Clear buf on error in __bpf_get_task_stack
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (7 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 08/12] bpf: Remove trace_in argument " Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 10/12] bpf: Disable preemption in __bpf_get_stack Jiri Olsa
` (2 subsequent siblings)
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: Sashiko, bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu,
Yonghong Song, Quentin Monnet, Tao Chen, STAR Labs SG,
Arnaud Lecomte
Both bpf_get_task_stack and bpf_get_task_stack_sleepable helpers that
use __bpf_get_task_stack have buf defined as ARG_PTR_TO_UNINIT_MEM
argument and we should initialize the buf on every return path.
Adding missing buf memset for __bpf_get_task_stack fail paths. This
provides deterministic buffer contents, which is useful when the buffer
is used directly as a map key.
Fixes: 06ab134ce8ec ("bpf: Refcount task stack in bpf_get_task_stack")
Fixes: b992f01e6615 ("bpf: Guard against accessing NULL pt_regs in bpf_get_task_stack()")
Reported-by: Sashiko <sashiko-bot@kernel.org>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 976c4e4c1af6..7f728d319a65 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -884,14 +884,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
struct pt_regs *regs;
long res = -EINVAL;
- if (!try_get_task_stack(task))
+ if (!try_get_task_stack(task)) {
+ memset(buf, 0, size);
return -EFAULT;
+ }
regs = task_pt_regs(task);
if (regs)
res = __bpf_get_stack(regs, task, buf, size, flags, may_fault);
+ else
+ memset(buf, 0, size);
put_task_stack(task);
-
return res;
}
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 10/12] bpf: Disable preemption in __bpf_get_stack
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (8 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 09/12] bpf: Clear buf on error in __bpf_get_task_stack Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 11/12] bpf: Avoid changing callchain in bpf_get_stack_pe Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe Jiri Olsa
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: stable, Tao Chen, STAR Labs SG, bpf, Martin KaFai Lau,
Eduard Zingerman, Song Liu, Yonghong Song, Quentin Monnet,
Arnaud Lecomte
From: Daniel Borkmann <borkmann@iogearbox.net>
get_perf_callchain() returns a per-CPU perf_callchain_entry buffer and
releases its recursion slot via put_callchain_entry() before returning,
so nothing keeps the entry reserved while __bpf_get_stack() consumes
it below.
A preemptible BPF program (e.g. a non-sleepable raw tracepoint program
on a PREEMPT kernel, which runs under migrate_disable() but not
preempt_disable()) can be scheduled out between obtaining the entry
and the copy. Another task scheduled on the same CPU then reuses the
same per-CPU buffer and overwrites trace->nr with a larger value.
copy_len is then computed from the inflated trace->nr and can exceed
the caller's buffer, causing an out-of-bounds write in the memcpy()
and in the build_id path.
The rcu_read_lock() taken here alone does not prevent this. It is
only taken on the may_fault path, and under CONFIG_PREEMPT_RCU it does
not disable preemption; it merely keeps perf's callchain buffer array
alive (freed via call_rcu()) and does nothing to stop another task
from reusing the entry.
Disable preemption around obtaining the callchain entry and copying
it into the caller's buffer, so the entry cannot be reused underneath
us and trace->nr stays bounded by max_depth. Build ID resolution may
fault and is therefore deferred until after preemption is re-enabled;
by then the instruction pointers have already been copied into buf,
so it operates only on that private copy. Note, preempt_disable() also
subsumes the buffer-lifetime guarantee the rcu_read_lock() provided,
since a preempt-disabled section is an RCU read-side critical section
for the callchain buffers' call_rcu() reclaim.
Cc: stable@vger.kernel.org
Fixes: c195651e565a ("bpf: add bpf_get_stack helper")
Reported-by: Tao Chen <chen.dylane@linux.dev>
Closes: https://lore.kernel.org/bpf/20260206090653.1336687-1-chen.dylane@linux.dev/
Reported-by: STAR Labs SG <info@starlabs.sg>
Signed-off-by: Daniel Borkmann <borkmann@iogearbox.net>
[ changed Fixes: commit ]
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 7f728d319a65..ab5c9bf5fec5 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -817,6 +817,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
if (may_fault)
rcu_read_lock(); /* need RCU for perf's callchain below */
+ preempt_disable();
if (kernel && task) {
trace = get_callchain_entry_for_task(task, max_depth);
@@ -828,6 +829,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
if (unlikely(!trace) || trace->nr < skip) {
if (may_fault)
rcu_read_unlock();
+ preempt_enable();
goto err_fault;
}
@@ -836,6 +838,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
/* trace should not be dereferenced after this point */
if (may_fault)
rcu_read_unlock();
+ preempt_enable();
return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 11/12] bpf: Avoid changing callchain in bpf_get_stack_pe
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (9 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 10/12] bpf: Disable preemption in __bpf_get_stack Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 21:01 ` [PATCHv3 bpf-next 12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe Jiri Olsa
11 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
There's no need to modify the trace object bpf_get_stack_pe, we just
need to pass the needed callchain length in separate argument.
This way we can have callchain pointers const and remove the trace->nr
modification and restoration.
Assisted-by: Codex:GPT-5.5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 36 +++++++++++++++---------------------
1 file changed, 15 insertions(+), 21 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index ab5c9bf5fec5..19858e1562ce 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -661,7 +661,7 @@ const struct bpf_func_proto bpf_get_stackid_proto = {
.arg3_type = ARG_ANYTHING,
};
-static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
+static __u64 count_kernel_ip(const struct perf_callchain_entry *trace)
{
__u64 nr_kernel = 0;
@@ -741,15 +741,15 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = {
.arg3_type = ARG_ANYTHING,
};
-static u32 callchain_store(struct perf_callchain_entry *trace, void *buf,
- u32 elem_size, u64 flags)
+static u32 callchain_store(const struct perf_callchain_entry *trace, u32 trace_nr,
+ void *buf, u32 elem_size, u64 flags)
{
bool user_build_id = flags & BPF_F_USER_BUILD_ID;
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
- u32 trace_nr, copy_len;
- u64 *ips;
+ const u64 *ips;
+ u32 copy_len;
- trace_nr = trace->nr - skip;
+ trace_nr = trace_nr - skip;
copy_len = trace_nr * elem_size;
ips = trace->ip + skip;
@@ -833,7 +833,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
goto err_fault;
}
- trace_nr = callchain_store(trace, buf, elem_size, flags);
+ trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags);
/* trace should not be dereferenced after this point */
if (may_fault)
@@ -935,8 +935,8 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
.arg4_type = ARG_ANYTHING,
};
-static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size,
- u64 flags)
+static int __bpf_get_stack_pe(const struct perf_callchain_entry *trace, u32 trace_nr,
+ void *buf, u32 size, u64 flags)
{
bool user_build_id = flags & BPF_F_USER_BUILD_ID;
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
@@ -952,12 +952,12 @@ static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32
return -EINVAL;
max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
- trace->nr = min_t(u32, trace->nr, max_depth);
+ trace_nr = min_t(u32, trace_nr, max_depth);
- if (trace->nr < skip)
+ if (trace_nr < skip)
return -EFAULT;
- nr_trace = callchain_store(trace, buf, elem_size, flags);
+ nr_trace = callchain_store(trace, trace_nr, buf, elem_size, flags);
return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */);
}
@@ -965,8 +965,8 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
void *, buf, u32, size, u64, flags)
{
struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
+ const struct perf_callchain_entry *trace;
struct perf_event *event = ctx->event;
- struct perf_callchain_entry *trace;
bool kernel, user;
int err = -EINVAL;
__u64 nr_kernel;
@@ -989,13 +989,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
nr_kernel = count_kernel_ip(trace);
if (kernel) {
- __u64 nr = trace->nr;
-
- trace->nr = nr_kernel;
- err = __bpf_get_stack_pe(trace, buf, size, flags);
-
- /* restore nr */
- trace->nr = nr;
+ err = __bpf_get_stack_pe(trace, nr_kernel, buf, size, flags);
} else { /* user */
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
@@ -1003,7 +997,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
if (skip > BPF_F_SKIP_FIELD_MASK)
goto clear;
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
- err = __bpf_get_stack_pe(trace, buf, size, flags);
+ err = __bpf_get_stack_pe(trace, trace->nr, buf, size, flags);
}
clear:
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCHv3 bpf-next 12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe
2026-08-03 21:01 [PATCHv3 bpf-next 00/12] bpf: Disable preemption in stack map code Jiri Olsa
` (10 preceding siblings ...)
2026-08-03 21:01 ` [PATCHv3 bpf-next 11/12] bpf: Avoid changing callchain in bpf_get_stack_pe Jiri Olsa
@ 2026-08-03 21:01 ` Jiri Olsa
2026-08-03 22:06 ` bot+bpf-ci
11 siblings, 1 reply; 21+ messages in thread
From: Jiri Olsa @ 2026-08-03 21:01 UTC (permalink / raw)
To: Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko
Cc: bpf, Martin KaFai Lau, Eduard Zingerman, Song Liu, Yonghong Song,
Quentin Monnet, Tao Chen, STAR Labs SG, Arnaud Lecomte
There's no need to modify the trace object bpf_get_stackid_pe, we just
need to pass the needed callchain length in separate argument.
This way we can have callchain pointers const and remove the trace->nr
modification and restoration.
Assisted-by: Codex:GPT-5.5
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
kernel/bpf/stackmap.c | 39 +++++++++++++++++----------------------
1 file changed, 17 insertions(+), 22 deletions(-)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 19858e1562ce..da72efc8b774 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -506,7 +506,7 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
struct stackid {
struct stack_map_bucket *bucket;
- u64 *ips;
+ const u64 *ips;
u32 nr;
u32 len;
u32 hash;
@@ -515,21 +515,21 @@ struct stackid {
};
static int stackid_init(struct stackid *stackid, struct bpf_map *map,
- struct perf_callchain_entry *trace, u64 flags)
+ const struct perf_callchain_entry *trace, u32 trace_nr, u64 flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
u32 max_depth;
- if (trace->nr <= skip)
+ if (trace_nr <= skip)
/* skipping more than usable stack trace */
return -EFAULT;
max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
- stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip);
+ stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip);
stackid->len = stackid->nr * sizeof(u64);
stackid->ips = trace->ip + skip;
- stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
+ stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
stackid->id = stackid->hash & (smap->n_buckets - 1);
stackid->bucket = READ_ONCE(smap->buckets[stackid->id]);
stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash;
@@ -537,11 +537,12 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
}
static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map,
- struct perf_callchain_entry *trace, u64 flags)
+ const struct perf_callchain_entry *trace, u32 trace_nr,
+ u64 flags)
{
int err;
- err = stackid_init(stackid, map, trace, flags);
+ err = stackid_init(stackid, map, trace, trace_nr, flags);
if (err)
return err;
@@ -640,7 +641,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
/* couldn't fetch the stack trace */
return -EFAULT;
- err = stackid_fastpath(&stackid, map, trace, flags);
+ err = stackid_fastpath(&stackid, map, trace, trace->nr, flags);
if (err != -ENOENT)
return err;
@@ -676,12 +677,13 @@ static __u64 count_kernel_ip(const struct perf_callchain_entry *trace)
BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
struct bpf_map *, map, u64, flags)
{
+ const struct perf_callchain_entry *trace;
struct perf_event *event = ctx->event;
struct stack_map_bucket *new_bucket;
- struct perf_callchain_entry *trace;
struct stackid stackid;
bool kernel, user;
__u64 nr_kernel;
+ u32 trace_nr;
int ret;
/* perf_sample_data doesn't have callchain, use bpf_get_stackid */
@@ -701,13 +703,13 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
return -EFAULT;
nr_kernel = count_kernel_ip(trace);
- __u64 nr = trace->nr; /* save original */
if (kernel) {
- trace->nr = nr_kernel;
+ trace_nr = nr_kernel;
} else { /* user */
u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
+ trace_nr = trace->nr;
skip += nr_kernel;
if (skip > BPF_F_SKIP_FIELD_MASK)
return -EFAULT;
@@ -715,21 +717,14 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
}
- ret = stackid_fastpath(&stackid, map, trace, flags);
+ ret = stackid_fastpath(&stackid, map, trace, trace_nr, flags);
if (ret != -ENOENT)
- goto out;
+ return ret;
new_bucket = stackid_new_bucket(&stackid, map);
- if (new_bucket) {
- trace->nr = nr;
+ if (new_bucket)
return stackid_install(&stackid, map, new_bucket, flags);
- }
- ret = -ENOMEM;
-
-out:
- /* restore nr */
- trace->nr = nr;
- return ret;
+ return -ENOMEM;
}
const struct bpf_func_proto bpf_get_stackid_proto_pe = {
--
2.54.0
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCHv3 bpf-next 12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe
2026-08-03 21:01 ` [PATCHv3 bpf-next 12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe Jiri Olsa
@ 2026-08-03 22:06 ` bot+bpf-ci
2026-08-04 20:05 ` Jiri Olsa
0 siblings, 1 reply; 21+ messages in thread
From: bot+bpf-ci @ 2026-08-03 22:06 UTC (permalink / raw)
To: jolsa, ast, daniel, andrii
Cc: bpf, martin.lau, eddyz87, songliubraving, yhs, qmo, chen.dylane,
info, contact, ast, andrii, daniel, martin.lau, eddyz87,
yonghong.song, clm, ihor.solodrai
[-- Attachment #1: Type: text/plain, Size: 671 bytes --]
> bpf: Avoid changing callchain in bpf_get_stackid_pe
>
> There's no need to modify the trace object bpf_get_stackid_pe, we just
> need to pass the needed callchain length in separate argument.
This isn't a bug, but should the first sentence read "modify the trace
object in bpf_get_stackid_pe"? As written, "the trace object
bpf_get_stackid_pe" parses as if the function name is the name of the
trace object itself.
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/30854471987
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCHv3 bpf-next 12/12] bpf: Avoid changing callchain in bpf_get_stackid_pe
2026-08-03 22:06 ` bot+bpf-ci
@ 2026-08-04 20:05 ` Jiri Olsa
0 siblings, 0 replies; 21+ messages in thread
From: Jiri Olsa @ 2026-08-04 20:05 UTC (permalink / raw)
To: bot+bpf-ci
Cc: ast, daniel, andrii, bpf, martin.lau, eddyz87, songliubraving,
yhs, qmo, chen.dylane, info, contact, martin.lau, yonghong.song,
clm, ihor.solodrai
On Mon, Aug 03, 2026 at 10:06:21PM +0000, bot+bpf-ci@kernel.org wrote:
> > bpf: Avoid changing callchain in bpf_get_stackid_pe
> >
> > There's no need to modify the trace object bpf_get_stackid_pe, we just
> > need to pass the needed callchain length in separate argument.
>
> This isn't a bug, but should the first sentence read "modify the trace
> object in bpf_get_stackid_pe"? As written, "the trace object
> bpf_get_stackid_pe" parses as if the function name is the name of the
> trace object itself.
yep, will fix
jirka
>
>
> ---
> AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
> See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
>
> CI run summary: https://github.com/kernel-patches/bpf/actions/runs/30854471987
^ permalink raw reply [flat|nested] 21+ messages in thread