* [PATCH perf-tools-next v8 1/6] perf ftrace: Optimise __cmd_ftrace() stack frame with strbuf streaming
2026-10-18 2:37 [PATCH perf-tools-next v8 0/6] perf ftrace: Support inlined functions and display enhancements in function graph tracer Aaron Tomlin
@ 2026-10-18 2:37 ` Aaron Tomlin
2026-10-18 2:49 ` sashiko-bot
2026-10-18 2:37 ` [PATCH perf-tools-next v8 2/6] perf ftrace: Support filtering return address comments Aaron Tomlin
` (4 subsequent siblings)
5 siblings, 1 reply; 13+ messages in thread
From: Aaron Tomlin @ 2026-10-18 2:37 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung
Cc: mark.rutland, alexander.shishkin, jolsa, irogers, adrian.hunter,
atomlin, linux-perf-users, linux-kernel
In __cmd_ftrace(), incoming trace data from trace_pipe was historically
buffered in a fixed 4096-byte array allocated directly on the stack
frame. Furthermore, reads were written directly to stdout via fwrite()
in arbitrary chunk sizes, rather than as discrete, well-delimited
trace lines.
Refactor __cmd_ftrace() to employ dynamic heap allocation for the read
buffer alongside a dynamic struct strbuf to accumulate streamed
characters into whole lines. Pre-allocating the strbuf ensures that
line processing incurs zero heap reallocations during steady-state
polling, while the stack frame of __cmd_ftrace() is reduced by over 96%
(down to ~160 bytes). Robust error and EOF detection on stdout and the
trace stream ensures polling terminates cleanly upon pipeline or pager
exit, while retrying on non-fatal EAGAIN and EINTR conditions prevents
premature termination on non-blocking reads or signal interruptions.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
tools/perf/builtin-ftrace.c | 79 +++++++++++++++++++++++++++++++------
1 file changed, 66 insertions(+), 13 deletions(-)
diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 6017493ff179..65983259b07e 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -14,6 +14,7 @@
#include <poll.h>
#include <semaphore.h>
#include <signal.h>
+#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <linux/capability.h>
@@ -41,7 +42,10 @@
#include "util/units.h"
#include "util/parse-sublevel-options.h"
+#include "util/strbuf.h"
+
#define DEFAULT_TRACER "function_graph"
+#define TRACE_BUF_SIZE 4096
static volatile sig_atomic_t workload_exec_errno;
static volatile sig_atomic_t done;
@@ -738,7 +742,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
{
char *trace_file;
int trace_fd;
- char buf[4096];
+ char *buf = NULL;
+ struct strbuf linebuf = STRBUF_INIT;
struct pollfd pollfd = {
.events = POLLIN,
};
@@ -767,10 +772,18 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
setup_pager();
+ buf = malloc(TRACE_BUF_SIZE);
+ if (!buf) {
+ pr_err("failed to allocate trace buffer\n");
+ goto out_reset;
+ }
+ strbuf_init(&linebuf, 512);
+ strbuf_setlen(&linebuf, 0);
+
trace_file = get_tracing_instance_file("trace_pipe");
if (!trace_file) {
pr_err("failed to open trace_pipe\n");
- goto out_reset;
+ goto out_free;
}
trace_fd = open(trace_file, O_RDONLY);
@@ -779,7 +792,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
if (trace_fd < 0) {
pr_err("failed to open trace_pipe\n");
- goto out_reset;
+ goto out_free;
}
fcntl(trace_fd, F_SETFL, O_NONBLOCK);
@@ -806,24 +819,42 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
}
while (!done) {
- if (poll(&pollfd, 1, -1) < 0)
+ if (poll(&pollfd, 1, -1) < 0) {
+ if (errno == EINTR)
+ continue;
break;
+ }
if (pollfd.revents & POLLIN) {
- int n = read(trace_fd, buf, sizeof(buf));
- if (n < 0)
+ int n = read(trace_fd, buf, TRACE_BUF_SIZE);
+
+ if (n < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
+ continue;
break;
- if (fwrite(buf, n, 1, stdout) != 1)
+ }
+ if (n == 0)
break;
+ for (int i = 0; i < n; i++) {
+ if (buf[i] == '\n') {
+ if (fprintf(stdout, "%s\n", linebuf.buf) < 0)
+ break;
+ strbuf_setlen(&linebuf, 0);
+ } else {
+ if (strbuf_addch(&linebuf, buf[i]) < 0)
+ goto out_close_fd;
+ }
+ }
/* flush output since stdout is in full buffering mode due to pager */
- fflush(stdout);
+ if (fflush(stdout) < 0 || ferror(stdout))
+ break;
}
}
write_tracing_file("tracing_on", "0");
if (workload_exec_errno) {
- const char *emsg = str_error_r(workload_exec_errno, buf, sizeof(buf));
+ const char *emsg = str_error_r(workload_exec_errno, buf, TRACE_BUF_SIZE);
/* flush stdout first so below error msg appears at the end. */
fflush(stdout);
pr_err("workload failed: %s\n", emsg);
@@ -831,16 +862,38 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
}
/* read remaining buffer contents */
- while (true) {
- int n = read(trace_fd, buf, sizeof(buf));
- if (n <= 0)
+ while (!ferror(stdout)) {
+ int n = read(trace_fd, buf, TRACE_BUF_SIZE);
+
+ if (n < 0) {
+ if (errno == EINTR)
+ continue;
break;
- if (fwrite(buf, n, 1, stdout) != 1)
+ }
+ if (n == 0)
break;
+ for (int i = 0; i < n; i++) {
+ if (buf[i] == '\n') {
+ if (fprintf(stdout, "%s\n", linebuf.buf) < 0)
+ break;
+ strbuf_setlen(&linebuf, 0);
+ } else {
+ if (strbuf_addch(&linebuf, buf[i]) < 0)
+ goto out_close_fd;
+ }
+ }
+ }
+
+ if (!ferror(stdout) && linebuf.len > 0) {
+ fprintf(stdout, "%s\n", linebuf.buf);
+ strbuf_setlen(&linebuf, 0);
}
out_close_fd:
close(trace_fd);
+out_free:
+ free(buf);
+ strbuf_release(&linebuf);
out_reset:
exit_tracing_instance();
out:
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH perf-tools-next v8 1/6] perf ftrace: Optimise __cmd_ftrace() stack frame with strbuf streaming
2026-10-18 2:37 ` [PATCH perf-tools-next v8 1/6] perf ftrace: Optimise __cmd_ftrace() stack frame with strbuf streaming Aaron Tomlin
@ 2026-10-18 2:49 ` sashiko-bot
0 siblings, 0 replies; 13+ messages in thread
From: sashiko-bot @ 2026-10-18 2:49 UTC (permalink / raw)
To: Aaron Tomlin; +Cc: linux-perf-users
> In __cmd_ftrace(), incoming trace data from trace_pipe was historically
> buffered in a fixed 4096-byte array allocated directly on the stack
> frame. Furthermore, reads were written directly to stdout via fwrite()
> in arbitrary chunk sizes, rather than as discrete, well-delimited
> trace lines.
>
> Refactor __cmd_ftrace() to employ dynamic heap allocation for the read
> buffer alongside a dynamic struct strbuf to accumulate streamed
> characters into whole lines. Pre-allocating the strbuf ensures that
> line processing incurs zero heap reallocations during steady-state
> polling, while the stack frame of __cmd_ftrace() is reduced by over 96%
> (down to ~160 bytes). Robust error and EOF detection on stdout and the
> trace stream ensures polling terminates cleanly upon pipeline or pager
> exit, while retrying on non-fatal EAGAIN and EINTR conditions prevents
> premature termination on non-blocking reads or signal interruptions.
>
> Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
Sashiko has reviewed this patch and found no issues.
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261018023753.584873-1-atomlin@atomlin.com?part=1
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH perf-tools-next v8 2/6] perf ftrace: Support filtering return address comments
2026-10-18 2:37 [PATCH perf-tools-next v8 0/6] perf ftrace: Support inlined functions and display enhancements in function graph tracer Aaron Tomlin
2026-10-18 2:37 ` [PATCH perf-tools-next v8 1/6] perf ftrace: Optimise __cmd_ftrace() stack frame with strbuf streaming Aaron Tomlin
@ 2026-10-18 2:37 ` Aaron Tomlin
2026-10-18 2:46 ` sashiko-bot
2026-10-18 2:37 ` [PATCH perf-tools-next v8 3/6] perf symbol: Introduce vmlinux_path__find() helper Aaron Tomlin
` (3 subsequent siblings)
5 siblings, 1 reply; 13+ messages in thread
From: Aaron Tomlin @ 2026-10-18 2:37 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung
Cc: mark.rutland, alexander.shishkin, jolsa, irogers, adrian.hunter,
atomlin, linux-perf-users, linux-kernel
When tracing with the Linux kernel's function graph tracer, enabling
funcgraph-retaddr records the return address of each function entry as
a comment in the trace stream (e.g. /* <-load_elf_phdrs+0x6c/0xb0 */).
While valuable for tracing execution origins, these trailing comments
can add considerable visual clutter to the call graph, particularly
when examining deeply nested call trees or when synthesising higher-level
abstractions.
Introduce support for filtering out return address comments from
function graph trace output. Provide a standalone --filter-retaddr
flag.
Key aspects of this implementation:
1. Return address parsing and filtering
Implement ftrace_filter_retaddr() in tools/perf/util/ftrace.c
to locate return address comments and strip them in-place with
memmove(3).
Additionally, introduce ftrace_parse_retaddr() to parse caller
symbol names and offsets across diverse output styles (standard
format, return values, and bracketed addresses), establishing
the parsing foundation for subsequent inlined callchain
resolution.
2. Command-line interface
Expose --filter-retaddr in
tools/perf/builtin-ftrace.c, updating the documentation in
tools/perf/Documentation/perf-ftrace.txt accordingly.
3. Automated verification
Add a dedicated unit test suite ("Ftrace return address
processing") in tools/perf/tests/ftrace.c to validate comment
parsing and string filtering across all supported formats.
Assisted-by: Antigravity:gemini-3.1-pro
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
tools/perf/Documentation/perf-ftrace.txt | 4 ++
tools/perf/builtin-ftrace.c | 8 +++
tools/perf/tests/Build | 1 +
tools/perf/tests/builtin-test.c | 1 +
tools/perf/tests/ftrace.c | 80 ++++++++++++++++++++++
tools/perf/tests/tests.h | 1 +
tools/perf/util/Build | 1 +
tools/perf/util/ftrace.c | 86 ++++++++++++++++++++++++
tools/perf/util/ftrace.h | 5 ++
9 files changed, 187 insertions(+)
create mode 100644 tools/perf/tests/ftrace.c
create mode 100644 tools/perf/util/ftrace.c
diff --git a/tools/perf/Documentation/perf-ftrace.txt b/tools/perf/Documentation/perf-ftrace.txt
index 3f3808e513fe..e8105083e67c 100644
--- a/tools/perf/Documentation/perf-ftrace.txt
+++ b/tools/perf/Documentation/perf-ftrace.txt
@@ -134,6 +134,10 @@ OPTIONS for 'perf ftrace trace'
- depth=<n> - Set max depth for function graph tracer to follow.
- tail - Print function name at the end.
+--filter-retaddr::
+ Filter out function return address comments (`/* <-caller+offset */`)
+ from function_graph output.
+
OPTIONS for 'perf ftrace latency'
---------------------------------
diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 65983259b07e..c715cbebaabe 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -837,6 +837,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
break;
for (int i = 0; i < n; i++) {
if (buf[i] == '\n') {
+ if (ftrace->filter_retaddr)
+ ftrace_filter_retaddr(linebuf.buf);
if (fprintf(stdout, "%s\n", linebuf.buf) < 0)
break;
strbuf_setlen(&linebuf, 0);
@@ -874,6 +876,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
break;
for (int i = 0; i < n; i++) {
if (buf[i] == '\n') {
+ if (ftrace->filter_retaddr)
+ ftrace_filter_retaddr(linebuf.buf);
if (fprintf(stdout, "%s\n", linebuf.buf) < 0)
break;
strbuf_setlen(&linebuf, 0);
@@ -885,6 +889,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
}
if (!ferror(stdout) && linebuf.len > 0) {
+ if (ftrace->filter_retaddr)
+ ftrace_filter_retaddr(linebuf.buf);
fprintf(stdout, "%s\n", linebuf.buf);
strbuf_setlen(&linebuf, 0);
}
@@ -1850,6 +1856,8 @@ int cmd_ftrace(int argc, const char **argv)
"Size of per cpu buffer, needs to use a B, K, M or G suffix.", parse_buffer_size),
OPT_BOOLEAN(0, "inherit", &ftrace.inherit,
"Trace children processes"),
+ OPT_BOOLEAN_SET(0, "filter-retaddr", &ftrace.filter_retaddr, &ftrace.filter_retaddr_set,
+ "Filter out funcgraph return address comments"),
OPT_INTEGER('D', "delay", &ftrace.target.initial_delay,
"Number of milliseconds to wait before starting tracing after program start"),
OPT_PARENT(common_options),
diff --git a/tools/perf/tests/Build b/tools/perf/tests/Build
index 395f2866094a..136138932fcb 100644
--- a/tools/perf/tests/Build
+++ b/tools/perf/tests/Build
@@ -72,6 +72,7 @@ perf-test-y += hwmon_pmu.o
perf-test-y += tool_pmu.o
perf-test-y += subcmd-help.o
perf-test-y += kallsyms-split.o
+perf-test-y += ftrace.o
ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc riscv))
perf-test-$(CONFIG_DWARF_UNWIND) += dwarf-unwind.o
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 880eece0bc12..640d6d327215 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -156,6 +156,7 @@ static struct test_suite *generic_tests[] = {
&suite__perf_hooks,
&suite__unit_number__scnprint,
&suite__mem2node,
+ &suite__ftrace,
&suite__time_utils,
&suite__jit_write_elf,
&suite__pfm,
diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
new file mode 100644
index 000000000000..596cecee844e
--- /dev/null
+++ b/tools/perf/tests/ftrace.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "tests.h"
+#include "debug.h"
+#include "util/ftrace.h"
+
+static int test_parse_retaddr(void)
+{
+ char sym[128];
+ u64 offset = 0;
+
+ /* Standard format */
+ TEST_ASSERT_VAL("parse standard retaddr",
+ ftrace_parse_retaddr("kernel_read() { /* <-load_elf_phdrs+0x6c/0xb0 */",
+ sym, sizeof(sym), &offset));
+ TEST_ASSERT_EQUAL("sym name", strcmp(sym, "load_elf_phdrs"), 0);
+ TEST_ASSERT_VAL("offset", offset == 0x6c);
+
+ /* With return value */
+ TEST_ASSERT_VAL("parse retaddr with ret",
+ ftrace_parse_retaddr(
+ "__cond_resched(); /* <-load_elf_phdrs+0x6c/0xb0 ret=0x0 */",
+ sym, sizeof(sym), &offset));
+ TEST_ASSERT_EQUAL("sym name", strcmp(sym, "load_elf_phdrs"), 0);
+ TEST_ASSERT_VAL("offset", offset == 0x6c);
+
+ /* Bracketed address */
+ TEST_ASSERT_VAL("parse bracketed retaddr",
+ ftrace_parse_retaddr("foo() { /* <-[0xffffffff818534c8] bar+0x20/0x40 */",
+ sym, sizeof(sym), &offset));
+ TEST_ASSERT_EQUAL("sym name", strcmp(sym, "bar"), 0);
+ TEST_ASSERT_VAL("offset", offset == 0x20);
+
+ /* No return address */
+ TEST_ASSERT_VAL("no retaddr",
+ !ftrace_parse_retaddr("do_filp_open();", sym, sizeof(sym), &offset));
+
+ return TEST_OK;
+}
+
+static int test_filter_retaddr(void)
+{
+ char line1[] = "kernel_read() { /* <-load_elf_phdrs+0x6c/0xb0 */";
+ char line2[] = "__cond_resched(); /* <-load_elf_phdrs+0x6c/0xb0 ret=0x0 */";
+ char line3[] = "do_filp_open();";
+
+ ftrace_filter_retaddr(line1);
+ TEST_ASSERT_EQUAL("filter standard retaddr", strcmp(line1, "kernel_read() {"), 0);
+
+ ftrace_filter_retaddr(line2);
+ TEST_ASSERT_EQUAL("filter retaddr with ret", strcmp(line2, "__cond_resched();"), 0);
+
+ ftrace_filter_retaddr(line3);
+ TEST_ASSERT_EQUAL("filter line without retaddr unchanged",
+ strcmp(line3, "do_filp_open();"), 0);
+
+ return TEST_OK;
+}
+
+static int test__ftrace(struct test_suite *test __maybe_unused, int subtest __maybe_unused)
+{
+ int ret;
+
+ ret = test_parse_retaddr();
+ if (ret != TEST_OK)
+ return ret;
+
+ ret = test_filter_retaddr();
+ if (ret != TEST_OK)
+ return ret;
+
+ return TEST_OK;
+}
+
+DEFINE_SUITE("Ftrace return address processing", ftrace);
diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index 5982c4e62cd3..cd12ae85ccfa 100644
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -180,6 +180,7 @@ DECLARE_SUITE(perf_hooks);
DECLARE_SUITE(unit_number__scnprint);
DECLARE_SUITE(mem2node);
DECLARE_SUITE(maps);
+DECLARE_SUITE(ftrace);
DECLARE_SUITE(time_utils);
DECLARE_SUITE(jit_write_elf);
DECLARE_SUITE(api_io);
diff --git a/tools/perf/util/Build b/tools/perf/util/Build
index 2c1f880c4c47..5b082d09006d 100644
--- a/tools/perf/util/Build
+++ b/tools/perf/util/Build
@@ -169,6 +169,7 @@ perf-util-y += list_sort.o
perf-util-y += mutex.o
perf-util-y += sharded_mutex.o
perf-util-y += intel-tpebs.o
+perf-util-y += ftrace.o
perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_counter.o
perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_counter_cgroup.o
diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
new file mode 100644
index 000000000000..aaf108369c36
--- /dev/null
+++ b/tools/perf/util/ftrace.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include "util/ftrace.h"
+
+/**
+ * ftrace_parse_retaddr - Parse return address comment from trace line
+ * @str: The trace line containing the comment
+ * @sym_name: Buffer to store the parsed symbol name
+ * @sym_len: Length of the @sym_name buffer
+ * @offset: Pointer to store the parsed offset
+ *
+ * Extract the function name and offset from a funcgraph-retaddr comment
+ * (e.g. "<-load_elf_phdrs+0x6c/0xb0").
+ *
+ * Return: true if successfully parsed, false otherwise.
+ */
+bool ftrace_parse_retaddr(const char *str, char *sym_name, size_t sym_len, u64 *offset)
+{
+ const char *p = strstr(str, "<-");
+ const char *plus;
+ const char *bracket;
+ size_t name_len;
+
+ if (!p)
+ return false;
+
+ p += 2;
+ while (*p == ' ')
+ p++;
+
+ if (*p == '[') {
+ bracket = strchr(p, ']');
+ if (bracket)
+ p = bracket + 1;
+ while (*p == ' ')
+ p++;
+ }
+
+ plus = strchr(p, '+');
+ if (!plus)
+ return false;
+
+ name_len = plus - p;
+ if (name_len == 0 || name_len >= sym_len)
+ return false;
+
+ memcpy(sym_name, p, name_len);
+ sym_name[name_len] = '\0';
+
+ *offset = strtoull(plus + 1, NULL, 16);
+ return true;
+}
+
+/**
+ * ftrace_filter_retaddr - Remove return address comment from trace line
+ * @str: The trace line to modify
+ *
+ * Find and strip the funcgraph-retaddr comment from the trace line
+ * in-place, removing trailing spaces where applicable.
+ */
+void ftrace_filter_retaddr(char *str)
+{
+ char *start = strstr(str, "/* <-");
+ char *end;
+
+ if (!start)
+ return;
+
+ end = strstr(start, "*/");
+ if (!end)
+ return;
+
+ end += 2; /* skip end-of-comment delimiter */
+
+ /* Also backtrack any spaces before comment */
+ while (start > str && *(start - 1) == ' ')
+ start--;
+
+ memmove(start, end, strlen(end) + 1);
+}
diff --git a/tools/perf/util/ftrace.h b/tools/perf/util/ftrace.h
index 950f2efafad2..50216d3fef64 100644
--- a/tools/perf/util/ftrace.h
+++ b/tools/perf/util/ftrace.h
@@ -39,6 +39,8 @@ struct perf_ftrace {
int graph_verbose;
int graph_thresh;
int graph_tail;
+ bool filter_retaddr;
+ bool filter_retaddr_set;
};
struct filter_entry {
@@ -93,4 +95,7 @@ perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace __maybe_unused)
#endif /* HAVE_BPF_SKEL */
+bool ftrace_parse_retaddr(const char *str, char *sym_name, size_t sym_len, u64 *offset);
+void ftrace_filter_retaddr(char *str);
+
#endif /* __PERF_FTRACE_H__ */
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH perf-tools-next v8 2/6] perf ftrace: Support filtering return address comments
2026-10-18 2:37 ` [PATCH perf-tools-next v8 2/6] perf ftrace: Support filtering return address comments Aaron Tomlin
@ 2026-10-18 2:46 ` sashiko-bot
0 siblings, 0 replies; 13+ messages in thread
From: sashiko-bot @ 2026-10-18 2:46 UTC (permalink / raw)
To: Aaron Tomlin; +Cc: linux-perf-users
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] perf ftrace: ftrace_filter_retaddr() strips leaf return values
--
commit bb5a10abc2299a6ae2ba0cbda70fab044e68ad36
Author: Aaron Tomlin <atomlin@atomlin.com>
perf ftrace: Support filtering return address comments
This commit introduces ftrace_parse_retaddr() and ftrace_filter_retaddr()
along with a --filter-retaddr command-line option and unit tests to strip
funcgraph-retaddr return address comments from function_graph trace output.
> diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
> new file mode 100644
> index 0000000000000..596cecee844e3
> --- /dev/null
> +++ b/tools/perf/tests/ftrace.c
> @@ -0,0 +1,80 @@
[ ... ]
> +static int test_filter_retaddr(void)
> +{
> + char line1[] = "kernel_read() { /* <-load_elf_phdrs+0x6c/0xb0 */";
> + char line2[] = "__cond_resched(); /* <-load_elf_phdrs+0x6c/0xb0 ret=0x0 */";
> + char line3[] = "do_filp_open();";
> +
> + ftrace_filter_retaddr(line1);
> + TEST_ASSERT_EQUAL("filter standard retaddr", strcmp(line1, "kernel_read() {"), 0);
> +
> + ftrace_filter_retaddr(line2);
> + TEST_ASSERT_EQUAL("filter retaddr with ret", strcmp(line2, "__cond_resched();"), 0);
> +
> + ftrace_filter_retaddr(line3);
> + TEST_ASSERT_EQUAL("filter line without retaddr unchanged",
> + strcmp(line3, "do_filp_open();"), 0);
> +
> + return TEST_OK;
> +}
[ ... ]
> diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
> new file mode 100644
> index 0000000000000..aaf108369c365
> --- /dev/null
> +++ b/tools/perf/util/ftrace.c
> @@ -0,0 +1,86 @@
[ ... ]
> +/**
> + * ftrace_filter_retaddr - Remove return address comment from trace line
> + * @str: The trace line to modify
> + *
> + * Find and strip the funcgraph-retaddr comment from the trace line
> + * in-place, removing trailing spaces where applicable.
> + */
> +void ftrace_filter_retaddr(char *str)
> +{
> + char *start = strstr(str, "/* <-");
> + char *end;
> +
> + if (!start)
> + return;
> +
> + end = strstr(start, "*/");
> + if (!end)
> + return;
> +
> + end += 2; /* skip end-of-comment delimiter */
> +
> + /* Also backtrack any spaces before comment */
> + while (start > str && *(start - 1) == ' ')
> + start--;
> +
> + memmove(start, end, strlen(end) + 1);
> +}
[Severity: Medium]
Does ftrace_filter_retaddr() unintentionally strip return values from leaf
functions when both funcgraph-retaddr and funcgraph-retval are enabled?
In kernel/trace/trace_functions_graph.c:print_graph_retval(), when both
TRACE_GRAPH_PRINT_RETADDR and TRACE_GRAPH_PRINT_RETVAL are active, the
kernel combines the return address and return value into a single comment on
leaf functions:
__cond_resched(); /* <-load_elf_phdrs+0x6c/0xb0 ret=0x0 */
Non-leaf functions place the return address on the entry line and the return
value on the closing brace line, so their return values are unaffected.
However, on leaf functions, ftrace_filter_retaddr() removes the entire
comment from "/* <-" through "*/", discarding "ret=0x0" as well (which
test_filter_retaddr() also asserts).
When running with "--graph-opts retaddr,retval --filter-retaddr" (or when a
subsequent change such as "perf ftrace: Support display of inlined functions
in function graph tracer" automatically enables funcgraph-retaddr and
filter_retaddr alongside "--graph-opts retval"), shouldn't
ftrace_filter_retaddr() preserve the "/* ret=... */" portion of the comment
on leaf functions instead of removing the whole comment block?
([ ... ])
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261018023753.584873-1-atomlin@atomlin.com?part=2
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH perf-tools-next v8 3/6] perf symbol: Introduce vmlinux_path__find() helper
2026-10-18 2:37 [PATCH perf-tools-next v8 0/6] perf ftrace: Support inlined functions and display enhancements in function graph tracer Aaron Tomlin
2026-10-18 2:37 ` [PATCH perf-tools-next v8 1/6] perf ftrace: Optimise __cmd_ftrace() stack frame with strbuf streaming Aaron Tomlin
2026-10-18 2:37 ` [PATCH perf-tools-next v8 2/6] perf ftrace: Support filtering return address comments Aaron Tomlin
@ 2026-10-18 2:37 ` Aaron Tomlin
2026-10-18 2:45 ` sashiko-bot
2026-10-18 2:37 ` [PATCH perf-tools-next v8 4/6] perf ftrace: Support display of inlined functions in function graph tracer Aaron Tomlin
` (2 subsequent siblings)
5 siblings, 1 reply; 13+ messages in thread
From: Aaron Tomlin @ 2026-10-18 2:37 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung
Cc: mark.rutland, alexander.shishkin, jolsa, irogers, adrian.hunter,
atomlin, linux-perf-users, linux-kernel
The vmlinux_path array maintains standard paths used by perf to locate
the kernel image (e.g. "vmlinux", "/boot/vmlinux",
"/lib/modules/%s/build/vmlinux", "/usr/lib/debug/..."). While
vmlinux_path__init() populates these candidates and dso__load_vmlinux_path()
loads symbols into a DSO/map, callers (such as test suites or probing tools)
often need to verify the existence of an accessible vmlinux on disk before
attempting machine setup or test execution.
Introduce vmlinux_path__find() to search through the configured vmlinux_path
entries (or symbol_conf.vmlinux_name if explicitly set) and return the
path to the first readable vmlinux file, or NULL if none is accessible.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
tools/perf/util/symbol.c | 30 ++++++++++++++++++++++++++++++
tools/perf/util/symbol.h | 2 ++
2 files changed, 32 insertions(+)
diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
index b61a574ce9a3..a0fc430be13a 100644
--- a/tools/perf/util/symbol.c
+++ b/tools/perf/util/symbol.c
@@ -2372,6 +2372,36 @@ static int vmlinux_path__init(struct perf_env *env)
return -1;
}
+/**
+ * vmlinux_path__find - Locate the first accessible vmlinux file
+ *
+ * Check symbol_conf.vmlinux_name first if set, then search through the
+ * initialized paths in vmlinux_path.
+ *
+ * Return: A pointer to the path of the first readable vmlinux file, or NULL if
+ * none is accessible.
+ */
+const char *vmlinux_path__find(void)
+{
+ int i;
+
+ if (symbol_conf.vmlinux_name && access(symbol_conf.vmlinux_name, R_OK) == 0)
+ return symbol_conf.vmlinux_name;
+
+ if (!vmlinux_path) {
+ symbol_conf.try_vmlinux_path = true;
+ if (vmlinux_path__init(NULL) < 0)
+ return NULL;
+ }
+
+ for (i = 0; i < vmlinux_path__nr_entries; ++i) {
+ if (access(vmlinux_path[i], R_OK) == 0)
+ return vmlinux_path[i];
+ }
+
+ return NULL;
+}
+
int setup_list(struct strlist **list, const char *list_str,
const char *list_name)
{
diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h
index d0bac824c79c..6d1eb08076ab 100644
--- a/tools/perf/util/symbol.h
+++ b/tools/perf/util/symbol.h
@@ -184,6 +184,8 @@ static inline int __symbol__join_symfs(char *bf, size_t size, const char *path)
extern int vmlinux_path__nr_entries;
extern char **vmlinux_path;
+const char *vmlinux_path__find(void);
+
static inline void *symbol__priv(struct symbol *sym)
{
return ((void *)sym) - symbol_conf.priv_size;
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH perf-tools-next v8 3/6] perf symbol: Introduce vmlinux_path__find() helper
2026-10-18 2:37 ` [PATCH perf-tools-next v8 3/6] perf symbol: Introduce vmlinux_path__find() helper Aaron Tomlin
@ 2026-10-18 2:45 ` sashiko-bot
0 siblings, 0 replies; 13+ messages in thread
From: sashiko-bot @ 2026-10-18 2:45 UTC (permalink / raw)
To: Aaron Tomlin; +Cc: linux-perf-users
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Medium] perf symbol: heap overflow in vmlinux_path__init() after vmlinux_path__find()
--
commit 075bbaee1dd0bb499313531ce795ad3e9b4f107b
Author: Aaron Tomlin <atomlin@atomlin.com>
perf symbol: Introduce vmlinux_path__find() helper
This commit introduces vmlinux_path__find() to search through the configured
vmlinux_path entries (or symbol_conf.vmlinux_name if set) and return the path
to the first readable vmlinux file.
> diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
> index b61a574ce9a30..a0fc430be13ae 100644
> --- a/tools/perf/util/symbol.c
> +++ b/tools/perf/util/symbol.c
> @@ -2372,6 +2372,36 @@ static int vmlinux_path__init(struct perf_env *env)
> return -1;
> }
>
> +/**
> + * vmlinux_path__find - Locate the first accessible vmlinux file
> + *
> + * Check symbol_conf.vmlinux_name first if set, then search through the
> + * initialized paths in vmlinux_path.
> + *
> + * Return: A pointer to the path of the first readable vmlinux file, or NULL if
> + * none is accessible.
> + */
> +const char *vmlinux_path__find(void)
> +{
> + int i;
> +
> + if (symbol_conf.vmlinux_name && access(symbol_conf.vmlinux_name, R_OK) == 0)
> + return symbol_conf.vmlinux_name;
> +
> + if (!vmlinux_path) {
> + symbol_conf.try_vmlinux_path = true;
> + if (vmlinux_path__init(NULL) < 0)
> + return NULL;
> + }
[Severity: Medium]
Can calling vmlinux_path__find() before symbol__init() cause
vmlinux_path__init() to overflow vmlinux_path[] and leak memory?
When vmlinux_path__find() is called prior to symbol__init(), vmlinux_path is
NULL and symbol_conf.initialized is false. vmlinux_path__find() sets
symbol_conf.try_vmlinux_path = true and calls vmlinux_path__init(NULL), which
allocates the 7-entry vmlinux_path array and populates it via
vmlinux_path__add(), advancing vmlinux_path__nr_entries to 7 (or 2 if
symbol_conf.symfs[0] != 0):
tools/perf/util/symbol.c:vmlinux_path__add() {
vmlinux_path[vmlinux_path__nr_entries] = strdup(new_entry);
if (vmlinux_path[vmlinux_path__nr_entries] == NULL)
return -1;
++vmlinux_path__nr_entries;
return 0;
}
When symbol__init() is subsequently called, symbol_conf.initialized is still
false and symbol_conf.try_vmlinux_path is true, so symbol__init() invokes
vmlinux_path__init(env) a second time:
tools/perf/util/symbol.c:symbol__init() {
...
if (symbol_conf.try_vmlinux_path && vmlinux_path__init(env) < 0)
return -1;
...
}
Because vmlinux_path__init() neither frees an existing vmlinux_path nor
resets vmlinux_path__nr_entries to 0:
tools/perf/util/symbol.c:vmlinux_path__init() {
...
vmlinux_path = malloc(sizeof(char *) * (ARRAY_SIZE(vmlinux_paths) +
ARRAY_SIZE(vmlinux_paths_upd)));
if (vmlinux_path == NULL)
return -1;
for (i = 0; i < ARRAY_SIZE(vmlinux_paths); i++)
if (vmlinux_path__add(vmlinux_paths[i]) < 0)
goto out_fail;
...
}
Doesn't the second call overwrite vmlinux_path with a new 7-entry buffer
(leaking the initial array and its strdup() strings) and then write the new
strdup() pointers starting at vmlinux_path[7] (or vmlinux_path[2]),
overflowing the heap buffer while leaving vmlinux_path[0..6] uninitialized
for dso__load_vmlinux_path() and vmlinux_path__exit()?
Additionally, if a caller invokes vmlinux_path__find() and skips
symbol__init(), symbol_conf.initialized remains false, so symbol__exit() will
return early without calling vmlinux_path__exit() to free vmlinux_path:
tools/perf/util/symbol.c:symbol__exit() {
if (!symbol_conf.initialized)
return;
...
vmlinux_path__exit();
...
}
> +
> + for (i = 0; i < vmlinux_path__nr_entries; ++i) {
> + if (access(vmlinux_path[i], R_OK) == 0)
> + return vmlinux_path[i];
> + }
> +
> + return NULL;
> +}
[ ... ]
>
> --
> 2.51.2
>
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261018023753.584873-1-atomlin@atomlin.com?part=3
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH perf-tools-next v8 4/6] perf ftrace: Support display of inlined functions in function graph tracer
2026-10-18 2:37 [PATCH perf-tools-next v8 0/6] perf ftrace: Support inlined functions and display enhancements in function graph tracer Aaron Tomlin
` (2 preceding siblings ...)
2026-10-18 2:37 ` [PATCH perf-tools-next v8 3/6] perf symbol: Introduce vmlinux_path__find() helper Aaron Tomlin
@ 2026-10-18 2:37 ` Aaron Tomlin
2026-10-18 3:00 ` sashiko-bot
2026-10-18 2:37 ` [PATCH perf-tools-next v8 5/6] perf ftrace: Support omitting execution duration " Aaron Tomlin
2026-10-18 2:37 ` [PATCH perf-tools-next v8 6/6] perf ftrace: Filter extraneous functions " Aaron Tomlin
5 siblings, 1 reply; 13+ messages in thread
From: Aaron Tomlin @ 2026-10-18 2:37 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung
Cc: mark.rutland, alexander.shishkin, jolsa, irogers, adrian.hunter,
atomlin, linux-perf-users, linux-kernel
The Linux kernel's function graph tracer operates at the machine
instruction level via compiler instrumentation
(-fpatchable-function-entry), recording entry and return events for
physical function calls. Consequently, functions inlined by the compiler
are invisible in the resulting call graph, as no discrete call or return
instructions are emitted for them. While technically expected, this
omission frequently obscures the logical execution flow when analysing
kernel subsystems heavily reliant upon inlining.
Address this by introducing the --inline option to perf ftrace for the
function_graph tracer.
When the kernel is configured with CONFIG_FUNCTION_GRAPH_RETADDR=y, the
funcgraph-retaddr trace option records the caller's return address on
each function entry, manifested in the trace stream as a comment
(e.g. /* <-irq_work_queue_on+0x81/0xa0 */). By interrogating DWARF debug
information from the kernel image (vmlinux) utilising libdw, perf ftrace
resolves this return address to its inlined callchain and synthesises the
intermediate inlined frames directly into the streamed call graph.
Key aspects of this implementation:
1. Inlined frame presentation
Synthesised inlined functions are rendered with an explicit
/* (inline) */ annotation at both entry and exit:
# CPU DURATION FUNCTION CALLS
# | | | | | | |
0) | wake_up_new_task() {
0) | activate_task() { /* (inline) */
0) 1.234 us | enqueue_task();
0) | } /* activate_task (inline) */
0) 2.567 us | }
This affords immediate visual clarity to the reader upon
entering an inlined scope, without necessitating a search for
the closing delimiter.
2. Call graph reconstruction and per-task stack tracking
Maintains a per-task synthetic call stack tracking real and inlined
frames across context switches, synthesising entry and exit
delimiters to accurately reconstruct the execution hierarchy. Tasks
are tracked in a hash table keyed by PID, with per-CPU active bindings
updated upon context-switch banners or verbose line records. Inlined
frames are structured strictly root-to-leaf and validated against the
active stack to prevent un-traced callers from attaching orphan inlines.
Bare closing braces without trailing function names (the default when
funcgraph-tail is disabled) are correctly matched against preceding
frames on the stack. Context column alignment, including task comm-pid
delimiters in verbose mode, is preserved when generating prefixes for
synthesised inline frames. Furthermore, stack depth saturation guards
ensure that inlined frame depth tracking remains clamped, indentation
cannot drift even under deep call trees exceeding FTRACE_MAX_STACK, and
overflow depth is automatically recovered upon caller detection.
3. Option handling and diagnostics
Adds -k or --vmlinux to designate an explicit debug image, and
provides clear diagnostic guidance should --inline be invoked on
a kernel lacking CONFIG_FUNCTION_GRAPH_RETADDR. Automatically
enables return address comment filtering unless explicitly
requested.
4. Automated verification
Expands the unit test suite in tools/perf/tests/ftrace.c to
validate DWARF inline symbol resolution, comment filtering
toggles, bare closing braces handling, task context-switch
isolation and migration, and stack saturation/overflow recovery.
Assisted-by: Antigravity:gemini-3.1-pro
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
tools/perf/Documentation/perf-ftrace.txt | 17 +-
tools/perf/builtin-ftrace.c | 58 +-
tools/perf/tests/ftrace.c | 481 +++++++++++++-
tools/perf/util/ftrace.c | 762 ++++++++++++++++++++++-
tools/perf/util/ftrace.h | 12 +
5 files changed, 1312 insertions(+), 18 deletions(-)
diff --git a/tools/perf/Documentation/perf-ftrace.txt b/tools/perf/Documentation/perf-ftrace.txt
index e8105083e67c..445c8ccc22d2 100644
--- a/tools/perf/Documentation/perf-ftrace.txt
+++ b/tools/perf/Documentation/perf-ftrace.txt
@@ -134,9 +134,24 @@ OPTIONS for 'perf ftrace trace'
- depth=<n> - Set max depth for function graph tracer to follow.
- tail - Print function name at the end.
+--inline::
+ Show inlined functions in function_graph tracer. This requires
+ kernel support for the funcgraph-retaddr option and a vmlinux with
+ debug symbols. Inlined functions are displayed in the call graph
+ hierarchy, and their inlined status is indicated with `/* (inline) */`
+ on entry and exit, e.g. `func() { /* (inline) */` and `} /* func (inline) */`.
+ By default, raw return address comments (`/* <-caller+offset */`)
+ are hidden in the output; specify `--graph-opts retaddr` to show them.
+
--filter-retaddr::
Filter out function return address comments (`/* <-caller+offset */`)
- from function_graph output.
+ from function_graph output. This is enabled by default when `--inline`
+ is used; use `--no-filter-retaddr` to disable it.
+
+-k::
+--vmlinux=::
+ Path to the vmlinux file containing debug symbols for resolving
+ inlined functions.
OPTIONS for 'perf ftrace latency'
diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index c715cbebaabe..cdacf5af14a6 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -575,9 +575,14 @@ static int set_tracing_funcgraph_retval(struct perf_ftrace *ftrace)
static int set_tracing_funcgraph_retaddr(struct perf_ftrace *ftrace)
{
- if (ftrace->graph_retaddr) {
- if (write_tracing_option_file("funcgraph-retaddr", "1") < 0)
+ if (ftrace->graph_retaddr || ftrace->use_inline) {
+ if (write_tracing_option_file("funcgraph-retaddr", "1") < 0) {
+ if (ftrace->use_inline) {
+ pr_err("failed to set funcgraph-retaddr option in tracing\n");
+ pr_err("Hint: ensure kernel is compiled with CONFIG_FUNCTION_GRAPH_RETADDR=y\n");
+ }
return -1;
+ }
}
return 0;
@@ -770,6 +775,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
goto out_reset;
}
+ if (perf_ftrace__setup_inlines(ftrace) < 0)
+ goto out_reset;
+
setup_pager();
buf = malloc(TRACE_BUF_SIZE);
@@ -837,9 +845,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
break;
for (int i = 0; i < n; i++) {
if (buf[i] == '\n') {
- if (ftrace->filter_retaddr)
- ftrace_filter_retaddr(linebuf.buf);
- if (fprintf(stdout, "%s\n", linebuf.buf) < 0)
+ if (ftrace_process_fgraph_line(ftrace,
+ linebuf.buf,
+ stdout) < 0)
+ goto out_close_fd;
+ if (ferror(stdout))
break;
strbuf_setlen(&linebuf, 0);
} else {
@@ -876,9 +886,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
break;
for (int i = 0; i < n; i++) {
if (buf[i] == '\n') {
- if (ftrace->filter_retaddr)
- ftrace_filter_retaddr(linebuf.buf);
- if (fprintf(stdout, "%s\n", linebuf.buf) < 0)
+ if (ftrace_process_fgraph_line(ftrace,
+ linebuf.buf,
+ stdout) < 0)
+ goto out_close_fd;
+ if (ferror(stdout))
break;
strbuf_setlen(&linebuf, 0);
} else {
@@ -889,9 +901,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
}
if (!ferror(stdout) && linebuf.len > 0) {
- if (ftrace->filter_retaddr)
- ftrace_filter_retaddr(linebuf.buf);
- fprintf(stdout, "%s\n", linebuf.buf);
+ if (ftrace_process_fgraph_line(ftrace, linebuf.buf,
+ stdout) < 0)
+ goto out_close_fd;
strbuf_setlen(&linebuf, 0);
}
@@ -901,6 +913,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace)
free(buf);
strbuf_release(&linebuf);
out_reset:
+ perf_ftrace__cleanup_inlines(ftrace);
exit_tracing_instance();
out:
return (done && !workload_exec_errno) ? 0 : -1;
@@ -1856,8 +1869,13 @@ int cmd_ftrace(int argc, const char **argv)
"Size of per cpu buffer, needs to use a B, K, M or G suffix.", parse_buffer_size),
OPT_BOOLEAN(0, "inherit", &ftrace.inherit,
"Trace children processes"),
- OPT_BOOLEAN_SET(0, "filter-retaddr", &ftrace.filter_retaddr, &ftrace.filter_retaddr_set,
- "Filter out funcgraph return address comments"),
+ OPT_BOOLEAN(0, "inline", &ftrace.use_inline,
+ "Show inlined functions in function_graph tracer"),
+ OPT_BOOLEAN_SET(0, "filter-retaddr", &ftrace.filter_retaddr,
+ &ftrace.filter_retaddr_set,
+ "Filter out funcgraph return address comments (default with --inline)"),
+ OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name, "file",
+ "vmlinux pathname"),
OPT_INTEGER('D', "delay", &ftrace.target.initial_delay,
"Number of milliseconds to wait before starting tracing after program start"),
OPT_PARENT(common_options),
@@ -1973,6 +1991,20 @@ int cmd_ftrace(int argc, const char **argv)
switch (subcmd) {
case PERF_FTRACE_TRACE:
+ if (ftrace.use_inline) {
+ if (ftrace.tracer && strcmp(ftrace.tracer, "function_graph") != 0) {
+ pr_err("Error: --inline option is only supported with function_graph tracer\n");
+ ret = -EINVAL;
+ goto out_delete_filters;
+ }
+ ftrace.tracer = "function_graph";
+ }
+ if (!ftrace.filter_retaddr_set) {
+ if (ftrace.use_inline && !ftrace.graph_retaddr)
+ ftrace.filter_retaddr = true;
+ else
+ ftrace.filter_retaddr = false;
+ }
cmd_func = __cmd_ftrace;
break;
case PERF_FTRACE_LATENCY:
diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
index 596cecee844e..6e20a61860a9 100644
--- a/tools/perf/tests/ftrace.c
+++ b/tools/perf/tests/ftrace.c
@@ -7,6 +7,10 @@
#include <unistd.h>
#include "tests.h"
#include "debug.h"
+#include "dso.h"
+#include "machine.h"
+#include "map.h"
+#include "symbol.h"
#include "util/ftrace.h"
static int test_parse_retaddr(void)
@@ -62,6 +66,465 @@ static int test_filter_retaddr(void)
return TEST_OK;
}
+#define TEST_CHECK(text, cond) \
+do { \
+ if (!(cond)) { \
+ pr_debug("FAILED %s:%d %s\n", __FILE__, __LINE__, text); \
+ ret = TEST_FAIL; \
+ goto out_cleanup; \
+ } \
+} while (0)
+
+static int test_inline_processing(void)
+{
+ static const char line_entry[] = " 0) | wake_up_new_task() {";
+ static const char line_enqueue[] =
+ " 0) | enqueue_task() { /* <-wake_up_new_task+0x1d1/0x3e0 */";
+ static const char line_exit_enqueue[] = " 0) + 10.000 us | } /* enqueue_task */";
+ static const char line_exit_task[] = " 0) + 20.000 us | } /* wake_up_new_task */";
+ static const char line_bare_enqueue[] = " 0) + 10.000 us | }";
+ static const char line_bare_task[] = " 0) + 20.000 us | }";
+ static const char line_unmatched[] = " 0) + 5.000 us | } /* unrelated_func */";
+ static const char line_verbose_entry[] =
+ " 1234.567890 | 0) bash-1234 | | wake_up_new_task() {";
+ static const char line_verbose_enqueue[] =
+ " 1234.567890 | 0) bash-1234 | | enqueue_task() { /* <-wake_up_new_task+0x1d1/0x3e0 */";
+ static const char line_verbose_exit_enqueue[] =
+ " 1234.567890 | 0) bash-1234 | 1.234 us | } /* enqueue_task */";
+ static const char line_verbose_exit_task[] =
+ " 1234.567890 | 0) bash-1234 | 2.567 us | } /* wake_up_new_task */";
+ static const char line_verbose_inline_entry[] =
+ " 1234.567890 | 0) bash-1234 | | activate_task() { /* (inline) */";
+ static const char line_verbose_inline_exit[] =
+ " 1234.567890 | 0) bash-1234 | | } /* activate_task (inline) */";
+ struct perf_ftrace ftrace;
+ char *out_buf = NULL;
+ size_t out_len = 0;
+ FILE *out;
+ const char *unmatched_pos;
+ const char *inline_exit_pos;
+ const char *vmlinux;
+ int ret = TEST_OK;
+
+ memset(&ftrace, 0, sizeof(ftrace));
+ ftrace.use_inline = true;
+ ftrace.filter_retaddr = 1; /* Default when --graph-opts retaddr is not specified */
+ ftrace.tracer = "function_graph";
+
+ vmlinux = vmlinux_path__find();
+
+ /* If vmlinux is present, test resolving symbols */
+ if (vmlinux) {
+ symbol_conf.vmlinux_name = vmlinux;
+ symbol_conf.ignore_vmlinux_buildid = true;
+ ret = perf_ftrace__setup_inlines(&ftrace);
+ if (ret < 0 || !ftrace.machine) {
+ ret = TEST_SKIP;
+ goto out_cleanup;
+ }
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ /* Feed simulated function graph trace lines */
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ pr_debug("Processed output:\n%s\n", out_buf);
+
+ /* Verify activate_task entry with inline hint */
+ TEST_CHECK("contains activate_task entry with inline hint",
+ strstr(out_buf, "activate_task() { /* (inline) */") != NULL);
+
+ /* Verify activate_task exit with inline hint */
+ TEST_CHECK("contains activate_task exit with inline hint",
+ strstr(out_buf, "} /* activate_task (inline) */") != NULL);
+
+ /* Verify return address comment was filtered out by default */
+ TEST_CHECK("retaddr comment filtered out",
+ strstr(out_buf, "/* <-wake_up_new_task") == NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Now test with filter_retaddr = 0 (user specified --graph-opts retaddr) */
+ ftrace.filter_retaddr = 0;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ /* Verify return address comment IS preserved when filter_retaddr = 0 */
+ TEST_CHECK("retaddr comment preserved when requested",
+ strstr(out_buf, "/* <-wake_up_new_task") != NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test with bare closing braces (funcgraph-tail disabled by default) */
+ ftrace.filter_retaddr = 1;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_bare_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_bare_task, out);
+
+ fclose(out);
+
+ TEST_CHECK("bare braces inline entry",
+ strstr(out_buf, "activate_task() { /* (inline) */") != NULL);
+ TEST_CHECK("bare braces inline exit",
+ strstr(out_buf, "} /* activate_task (inline) */") != NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test that unmatched exit events do not pop active inlined frames */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_enqueue, out);
+ /* Unmatched exit event while activate_task is still active */
+ ftrace_process_fgraph_line(&ftrace, line_unmatched, out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
+
+ fclose(out);
+
+ unmatched_pos = strstr(out_buf, "} /* unrelated_func */");
+ inline_exit_pos = strstr(out_buf, "} /* activate_task (inline) */");
+
+ TEST_CHECK("unmatched exit present", unmatched_pos != NULL);
+ TEST_CHECK("inline exit present", inline_exit_pos != NULL);
+ TEST_CHECK("inline exit not prematurely popped before unmatched exit",
+ inline_exit_pos > unmatched_pos);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test recovery when an intermediate real function exit event is dropped */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
+ /* line_exit_enqueue is dropped; outer wake_up_new_task exits directly */
+ ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
+
+ fclose(out);
+
+ TEST_CHECK("inline exit synthesized despite dropped callee exit",
+ strstr(out_buf, "} /* activate_task (inline) */") != NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test recovery when intermediate callee exit is dropped before sibling call */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
+ /* line_exit_enqueue dropped; caller makes sibling call without inlines */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | check_preempt_curr(); /* <-wake_up_new_task+0x0/0x3e0 */",
+ out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
+
+ fclose(out);
+
+ TEST_CHECK("inline exit synthesized when callee exit dropped before sibling call",
+ strstr(out_buf, "} /* activate_task (inline) */") != NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test that callers not on active stack do not synthesise inlines */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_entry, out);
+ /* housekeeping_any_cpu called by untraced ring_buffer_unlock_commit */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | housekeeping_any_cpu(); /* <-ring_buffer_unlock_commit+0x1a0/0x1f0 */",
+ out);
+ ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
+
+ fclose(out);
+
+ TEST_CHECK("untraced caller inlines not synthesised",
+ strstr(out_buf, "rb_wakeups") == NULL &&
+ strstr(out_buf, "rb_irq_work_queue") == NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test verbose mode prefix formatting for synthesised inlines */
+ ftrace.graph_verbose = true;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_verbose_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_verbose_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_verbose_exit_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_verbose_exit_task, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ TEST_CHECK("verbose inline entry preserves comm-pid",
+ strstr(out_buf, line_verbose_inline_entry) != NULL);
+ TEST_CHECK("verbose inline exit preserves comm-pid",
+ strstr(out_buf, line_verbose_inline_exit) != NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+ ftrace.graph_verbose = false;
+ }
+
+out_cleanup:
+ free(out_buf);
+ perf_ftrace__cleanup_inlines(&ftrace);
+ return ret;
+}
+#undef TEST_CHECK
+
+static int test_stack_saturation(void)
+{
+ struct perf_ftrace ftrace;
+ char *out_buf = NULL;
+ size_t out_len = 0;
+ FILE *out;
+ int i;
+
+ memset(&ftrace, 0, sizeof(ftrace));
+ ftrace.use_inline = true;
+ ftrace_cpu_states__clear();
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out)
+ return TEST_FAIL;
+
+ /* Push root frame at depth 0 */
+ ftrace_process_fgraph_line(&ftrace, " 0) | root_func() {", out);
+
+ /* Push 255 more frames to fill FTRACE_MAX_STACK (256) */
+ for (i = 1; i < 256; i++)
+ ftrace_process_fgraph_line(&ftrace, " 0) | sub_func() {", out);
+
+ /* Push 44 overflow frames beyond FTRACE_MAX_STACK */
+ for (i = 0; i < 44; i++)
+ ftrace_process_fgraph_line(&ftrace, " 0) | overflow_func() {", out);
+
+ /* Pop the 44 overflow frames with bare closing braces */
+ for (i = 0; i < 44; i++)
+ ftrace_process_fgraph_line(&ftrace, " 0) | }", out);
+
+ /* Pop 255 tracked sub_func frames */
+ for (i = 1; i < 256; i++)
+ ftrace_process_fgraph_line(&ftrace, " 0) | }", out);
+
+ /* Pop root frame - must cleanly match root_func on stack, not popped prematurely */
+ ftrace_process_fgraph_line(&ftrace, " 0) | } /* root_func */", out);
+
+ /* Feed one more function entry - it must have base indentation (no indentation drift) */
+ ftrace_process_fgraph_line(&ftrace, " 0) | base_func() {", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ /* Verify base_func starts at standard 2 spaces indentation without drift */
+ if (!strstr(out_buf, "| base_func() {")) {
+ pr_debug("FAILED %s:%d no indentation drift after stack saturation\n",
+ __FILE__, __LINE__);
+ free(out_buf);
+ return TEST_FAIL;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* Test overflow recovery when events are dropped */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out)
+ return TEST_FAIL;
+
+ ftrace_process_fgraph_line(&ftrace, " 0) | root_func() {", out);
+ for (i = 1; i < 256; i++)
+ ftrace_process_fgraph_line(&ftrace, " 0) | sub_func() {", out);
+ for (i = 0; i < 44; i++)
+ ftrace_process_fgraph_line(&ftrace, " 0) | overflow_func() {", out);
+
+ /* Dropped events: root_func directly calls recovery_func */
+ ftrace_process_fgraph_line(&ftrace, " 0) | recovery_func() { /* <-root_func+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace, " 0) 1.000 us | }", out);
+ ftrace_process_fgraph_line(&ftrace, " 0) 5.000 us | } /* root_func */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (!strstr(out_buf, "recovery_func() {")) {
+ pr_debug("FAILED %s:%d recovery_func missing after overflow recovery\n",
+ __FILE__, __LINE__);
+ free(out_buf);
+ return TEST_FAIL;
+ }
+ if (!strstr(out_buf, "} /* root_func */")) {
+ pr_debug("FAILED %s:%d root_func exit missing after overflow recovery\n",
+ __FILE__, __LINE__);
+ free(out_buf);
+ return TEST_FAIL;
+ }
+
+ free(out_buf);
+ return TEST_OK;
+}
+
+static int test_column_layouts(void)
+{
+ static const struct {
+ const char *desc;
+ bool verbose;
+ const char *in;
+ const char *expected;
+ } cases[] = {
+ {
+ "default layout", false,
+ " 7) | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
+ " 7) | wake_up_new_task() {\n",
+ },
+ {
+ "verbose layout", true,
+ " 1234.567890 | 7) bash-1234 | | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
+ " 1234.567890 | 7) bash-1234 | | wake_up_new_task() {\n",
+ },
+ };
+ struct perf_ftrace ftrace;
+ char *out_buf = NULL;
+ size_t out_len = 0;
+ FILE *out;
+ size_t i;
+
+ for (i = 0; i < ARRAY_SIZE(cases); i++) {
+ memset(&ftrace, 0, sizeof(ftrace));
+ ftrace.use_inline = true;
+ ftrace.filter_retaddr = true;
+ ftrace.graph_verbose = cases[i].verbose;
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out)
+ return TEST_FAIL;
+
+ ftrace_process_fgraph_line(&ftrace, cases[i].in, out);
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strcmp(out_buf, cases[i].expected)) {
+ pr_debug("%s: expected '%s', got '%s'\n",
+ cases[i].desc, cases[i].expected, out_buf);
+ free(out_buf);
+ return TEST_FAIL;
+ }
+ free(out_buf);
+ out_buf = NULL;
+ }
+
+ return TEST_OK;
+}
+
+static int test_context_switch_handling(void)
+{
+ struct perf_ftrace ftrace;
+ char *out_buf = NULL;
+ size_t out_len = 0;
+ FILE *out;
+
+ memset(&ftrace, 0, sizeof(ftrace));
+ ftrace.use_inline = true;
+ ftrace_cpu_states__clear();
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out)
+ return TEST_FAIL;
+
+ /* Task 1 (PID 100) starts running on CPU 0 */
+ ftrace_process_fgraph_line(&ftrace, " ------------------------------------------", out);
+ ftrace_process_fgraph_line(&ftrace, " 0) <idle>-0 => task1-100 ", out);
+ ftrace_process_fgraph_line(&ftrace, " ------------------------------------------", out);
+
+ /* Task 1 enters func1 */
+ ftrace_process_fgraph_line(&ftrace, " 0) | func1() {", out);
+
+ /* Context switch to Task 2 (PID 200) on CPU 0 */
+ ftrace_process_fgraph_line(&ftrace, " ------------------------------------------", out);
+ ftrace_process_fgraph_line(&ftrace, " 0) task1-100 => task2-200 ", out);
+ ftrace_process_fgraph_line(&ftrace, " ------------------------------------------", out);
+
+ /* Task 2 executes func2 - should NOT inherit func1's indentation */
+ ftrace_process_fgraph_line(&ftrace, " 0) | func2() {", out);
+ ftrace_process_fgraph_line(&ftrace, " 0) | } /* func2 */", out);
+
+ /* Task 1 migrates and resumes on CPU 1 */
+ ftrace_process_fgraph_line(&ftrace, " ------------------------------------------", out);
+ ftrace_process_fgraph_line(&ftrace, " 1) <idle>-0 => task1-100 ", out);
+ ftrace_process_fgraph_line(&ftrace, " ------------------------------------------", out);
+
+ /* Task 1 exits func1 on CPU 1 - should cleanly match func1 from its preserved stack */
+ ftrace_process_fgraph_line(&ftrace, " 1) | } /* func1 */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ /* Verify func2 in Task 2 started at base indentation "| func2() {" */
+ TEST_ASSERT_VAL("task2 does not inherit task1 call stack",
+ strstr(out_buf, "| func2() {") != NULL);
+
+ /* Verify task1 exit on CPU 1 matches cleanly */
+ TEST_ASSERT_VAL("task1 preserved stack across migration",
+ strstr(out_buf, "1) | } /* func1 */") != NULL);
+
+ free(out_buf);
+ return TEST_OK;
+}
+
static int test__ftrace(struct test_suite *test __maybe_unused, int subtest __maybe_unused)
{
int ret;
@@ -74,7 +537,23 @@ static int test__ftrace(struct test_suite *test __maybe_unused, int subtest __ma
if (ret != TEST_OK)
return ret;
+ ret = test_stack_saturation();
+ if (ret != TEST_OK)
+ return ret;
+
+ ret = test_context_switch_handling();
+ if (ret != TEST_OK)
+ return ret;
+
+ ret = test_column_layouts();
+ if (ret != TEST_OK)
+ return ret;
+
+ ret = test_inline_processing();
+ if (ret != TEST_OK)
+ return ret;
+
return TEST_OK;
}
-DEFINE_SUITE("Ftrace return address processing", ftrace);
+DEFINE_SUITE("Ftrace inline processing", ftrace);
diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
index aaf108369c36..020551a4d7e0 100644
--- a/tools/perf/util/ftrace.c
+++ b/tools/perf/util/ftrace.c
@@ -1,13 +1,176 @@
// SPDX-License-Identifier: GPL-2.0
-#include <linux/compiler.h>
-#include <linux/kernel.h>
-#include <linux/string.h>
+/*
+ * ftrace.c - Ftrace utilities and inlined function processing
+ *
+ * Copyright (C) 2026 Aaron Tomlin <atomlin@atomlin.com>
+ */
#include <stdio.h>
#include <stdlib.h>
+#include <errno.h>
#include <string.h>
#include <ctype.h>
+#include <stdbool.h>
+#include <linux/kernel.h>
+#include <linux/zalloc.h>
+#include <linux/hash.h>
+#include <linux/list.h>
+#include <linux/string.h>
+#include "debug.h"
+#include "dso.h"
+#include "machine.h"
+#include "map.h"
+#include "srcline.h"
+#include "symbol.h"
#include "util/ftrace.h"
+#define FTRACE_MAX_CPUS 4096
+#define FTRACE_MAX_STACK 256
+#define FTRACE_MAX_INLINES 32
+#define FTRACE_PID_HASH_BITS 10
+#define FTRACE_PID_HASH_SIZE (1 << FTRACE_PID_HASH_BITS)
+
+struct ftrace_stack_frame {
+ char *name;
+ bool inlined;
+ int indent;
+};
+
+struct ftrace_task_state {
+ struct hlist_node node;
+ pid_t pid;
+ struct ftrace_stack_frame stack[FTRACE_MAX_STACK];
+ int depth;
+ int inlined_depth;
+ int overflow_depth;
+};
+
+#define ftrace_cpu_state ftrace_task_state
+
+static struct hlist_head ftrace_task_hash[FTRACE_PID_HASH_SIZE];
+static struct ftrace_task_state *ftrace_cpu_tasks[FTRACE_MAX_CPUS];
+
+/**
+ * ftrace_cpu_states__clear - Free allocated CPU and task stack states
+ *
+ * Free all memory associated with the per-task stack tracking states
+ * used for inlined function resolution, including the strings
+ * allocated for stack frames.
+ */
+void ftrace_cpu_states__clear(void)
+{
+ struct ftrace_task_state *ts;
+ struct hlist_node *tmp;
+ int i, j;
+
+ for (i = 0; i < FTRACE_PID_HASH_SIZE; i++) {
+ hlist_for_each_entry_safe(ts, tmp, &ftrace_task_hash[i], node) {
+ for (j = 0; j < ts->depth; j++)
+ free(ts->stack[j].name);
+ hlist_del_init(&ts->node);
+ free(ts);
+ }
+ }
+
+ memset(ftrace_cpu_tasks, 0, sizeof(ftrace_cpu_tasks));
+}
+
+/**
+ * get_task_state - Get or allocate stack state for a given task PID
+ * @pid: Process ID of the task
+ *
+ * Return: A pointer to the task's stack state, or NULL on allocation failure.
+ */
+static struct ftrace_task_state *get_task_state(pid_t pid)
+{
+ struct ftrace_task_state *ts;
+ struct hlist_head *head;
+ u32 key = hash_32((u32)pid, FTRACE_PID_HASH_BITS);
+
+ head = &ftrace_task_hash[key];
+ hlist_for_each_entry(ts, head, node) {
+ if (ts->pid == pid)
+ return ts;
+ }
+
+ ts = zalloc(sizeof(*ts));
+ if (!ts)
+ return NULL;
+
+ ts->pid = pid;
+ hlist_add_head(&ts->node, head);
+ return ts;
+}
+
+/**
+ * get_cpu_state - Get or allocate active task stack state for a given CPU
+ * @cpu: The CPU number
+ *
+ * Return: A pointer to the CPU's active task stack state, or NULL on error or
+ * out-of-bounds CPU index.
+ */
+static struct ftrace_task_state *get_cpu_state(int cpu)
+{
+ if (cpu < 0 || cpu >= FTRACE_MAX_CPUS)
+ return NULL;
+
+ if (!ftrace_cpu_tasks[cpu]) {
+ /*
+ * If no context switch banner has been seen yet on this CPU,
+ * allocate an initial default task state for it using a negative
+ * synthetic PID to avoid collision with real PIDs.
+ */
+ ftrace_cpu_tasks[cpu] = get_task_state(-1 - cpu);
+ }
+
+ return ftrace_cpu_tasks[cpu];
+}
+
+/**
+ * parse_context_switch - Parse context switch banner and update CPU's task
+ * @line: The trace line containing the banner
+ *
+ * Check if @line is a context switch banner (e.g. " 0) <idle>-0 => migrati-19").
+ * If so, extract the CPU number and the newly scheduled task's PID, and
+ * update the active task mapping for that CPU.
+ *
+ * Return: true if @line was a context switch banner, false otherwise.
+ */
+static bool parse_context_switch(const char *line)
+{
+ const char *arrow = strstr(line, " => ");
+ const char *p;
+ const char *next;
+ const char *hyphen;
+ char *endptr;
+ long cpu_val;
+ long pid_val;
+
+ if (!arrow)
+ return false;
+
+ /* Find CPU number before ')' */
+ p = skip_spaces(line);
+ cpu_val = strtol(p, &endptr, 10);
+ if (endptr == p || *endptr != ')')
+ return false;
+
+ if (cpu_val < 0 || cpu_val >= FTRACE_MAX_CPUS)
+ return false;
+
+ /* Parse the target task's PID after the arrow (" => comm-pid") */
+ next = skip_spaces(arrow + 4);
+ hyphen = strrchr(next, '-');
+ if (!hyphen)
+ return false;
+
+ pid_val = strtol(hyphen + 1, &endptr, 10);
+ if (endptr == hyphen + 1)
+ return false;
+
+ ftrace_cpu_tasks[cpu_val] = get_task_state((pid_t)pid_val);
+ return true;
+}
+
/**
* ftrace_parse_retaddr - Parse return address comment from trace line
* @str: The trace line containing the comment
@@ -84,3 +247,596 @@ void ftrace_filter_retaddr(char *str)
memmove(start, end, strlen(end) + 1);
}
+
+/**
+ * ftrace_resolve_inlines - Resolve inlined functions for a return address
+ * @machine: Perf machine context
+ * @sym_name: Name of the caller function
+ * @offset: Instruction offset within the caller function
+ * @inlined_names: Array to store the resolved inline function names
+ * @max_inlines: Maximum number of inline frames to resolve
+ *
+ * Use DWARF debug information from the kernel image to resolve a caller's
+ * return address to its inlined callchain.
+ *
+ * Return: The number of inline functions successfully resolved.
+ */
+int ftrace_resolve_inlines(struct machine *machine,
+ const char *sym_name, u64 offset,
+ const char **inlined_names, int max_inlines)
+{
+ struct map *map = NULL;
+ struct symbol *sym;
+ struct dso *dso;
+ struct inline_node *node;
+ struct inline_list *ilist;
+ u64 ip, addr;
+ int count = 0;
+
+ if (!machine)
+ return 0;
+
+ sym = machine__find_kernel_symbol_by_name(machine, sym_name, &map);
+ if (!sym || !map)
+ return 0;
+
+ dso = map__dso(map);
+ if (!dso) {
+ map__put(map);
+ return 0;
+ }
+
+ ip = sym->start + offset;
+ addr = map__rip_2objdump(map, ip);
+
+ node = inlines__tree_find(dso__inlined_nodes(dso), addr);
+ if (!node) {
+ node = dso__parse_addr_inlines(dso, addr, sym);
+ if (node)
+ inlines__tree_insert(dso__inlined_nodes(dso), node);
+ }
+
+ if (!node) {
+ map__put(map);
+ return 0;
+ }
+
+ list_for_each_entry_reverse(ilist, &node->val, list) {
+ if (ilist->symbol && symbol__inlined(ilist->symbol)) {
+ if (count < max_inlines) {
+ char *name = strdup(ilist->symbol->name);
+
+ if (name)
+ inlined_names[count++] = name;
+ }
+ }
+ }
+
+ map__put(map);
+ return count;
+}
+
+/**
+ * extract_entry_func_name - Extract the function name from an entry record
+ * @str: The trace line's function column string
+ * @buf: Buffer to store the extracted name
+ * @size: Size of the @buf
+ *
+ * Parse the function name out of an entry trace record by stripping away
+ * parentheses or braces.
+ */
+static void extract_entry_func_name(const char *str, char *buf, size_t size)
+{
+ size_t i = 0;
+
+ while (*str && *str != '(' && *str != '{' && !isspace(*str) && i < size - 1)
+ buf[i++] = *str++;
+ buf[i] = '\0';
+}
+
+/**
+ * extract_exit_func_name - Extract the function name from an exit record
+ * @str: The trace line's function column string
+ * @buf: Buffer to store the extracted name
+ * @size: Size of the @buf
+ *
+ * Parse the original function name out of the trailing comment in an
+ * exit trace record.
+ */
+static void extract_exit_func_name(const char *str, char *buf, size_t size)
+{
+ const char *p = strchr(str, '}');
+ size_t i = 0;
+
+ buf[0] = '\0';
+ if (!p)
+ return;
+
+ p = strstr(p, "/*");
+ if (!p)
+ return;
+
+ p = skip_spaces(p + 2);
+ while (*p && !isspace(*p) && *p != '*' && *p != '/' && i < size - 1)
+ buf[i++] = *p++;
+ buf[i] = '\0';
+}
+
+/**
+ * make_blank_prefix - Create a blank prefix mirroring a trace record's context
+ * @line: The original trace record
+ * @pipe_char: Pointer to the final delimiter character of the context columns
+ * @buf: Buffer to store the blank prefix
+ * @size: Size of the @buf
+ *
+ * Generate a prefix matching the context columns of @line (e.g. CPU, duration)
+ * but with the duration field replaced by spaces.
+ */
+static void make_blank_prefix(const struct perf_ftrace *ftrace, const char *line,
+ const char *pipe_char, char *buf, size_t size)
+{
+ size_t len = pipe_char - line + 1;
+ const char *start_delim = NULL;
+ char *c;
+
+ if (!size)
+ return;
+
+ if (len >= size)
+ len = size - 1;
+
+ memcpy(buf, line, len);
+ buf[len] = '\0';
+
+ if (*pipe_char == '|') {
+ if (ftrace->graph_verbose) {
+ /* In verbose mode, duration starts after the comm-pid '|' */
+ for (const char *p = pipe_char - 1; p >= line; p--) {
+ if (*p == '|') {
+ start_delim = p;
+ break;
+ }
+ }
+ } else {
+ /* In default mode, duration starts after CPU ')' */
+ start_delim = strchr(line, ')');
+ }
+
+ if (start_delim && start_delim < pipe_char && start_delim < line + len) {
+ char *end = buf + (pipe_char - line);
+
+ if (end > buf + len)
+ end = buf + len;
+ for (c = buf + (start_delim - line) + 1; c < end; c++)
+ *c = ' ';
+ }
+ }
+}
+
+/**
+ * find_func_column - Locate the function call column in a trace record
+ * @ftrace: Perf ftrace context
+ * @line: The trace record to parse
+ * @cpu: Pointer to store the CPU number
+ *
+ * Locate the context columns that precede the function call column of a
+ * function_graph trace record. Depending on the funcgraph-* options set
+ * up by perf, a record looks like:
+ *
+ * [ abstime |] CPU) [ comm-pid |] [ duration |] func
+ *
+ * The abstime and comm-pid columns are only present with --graph-opts
+ * verbose.
+ *
+ * Return: A pointer to the last character of the context columns (the final
+ * '|' delimiter). Returns NULL if @line is not a trace record.
+ */
+static const char *find_func_column(const struct perf_ftrace *ftrace,
+ const char *line, int *cpu)
+{
+ const char *p = line;
+ char *endptr;
+ long val;
+
+ if (ftrace->graph_verbose) {
+ p = strchr(p, '|');
+ if (!p)
+ return NULL;
+ p++;
+ }
+
+ p = skip_spaces(p);
+ val = strtol(p, &endptr, 10);
+ if (endptr == p || *endptr != ')')
+ return NULL;
+ p = endptr;
+
+ if (ftrace->graph_verbose) {
+ const char *proc_start = p;
+
+ p = strchr(p, '|');
+ if (!p)
+ return NULL;
+
+ if (val >= 0 && val < FTRACE_MAX_CPUS) {
+ const char *h = p - 1;
+
+ while (h >= proc_start && *h != '-')
+ h--;
+ if (h >= proc_start && *h == '-') {
+ long pid_val = strtol(h + 1, NULL, 10);
+ ftrace_cpu_tasks[(int)val] = get_task_state((pid_t)pid_val);
+ }
+ }
+ }
+
+ p = strchr(p + 1, '|');
+ if (!p)
+ return NULL;
+
+ *cpu = (int)val;
+ return p;
+}
+
+/**
+ * print_line - Output a trace line verbatim, filtering return addresses if needed
+ * @ftrace: Perf ftrace context
+ * @line: The trace line to output
+ * @out: File stream to output to
+ *
+ * Output @line to @out. If ftrace->filter_retaddr is set, return address
+ * comments are filtered out first.
+ *
+ * Return: 0 on success, or a negative error code (e.g. -ENOMEM) on failure.
+ */
+static int print_line(struct perf_ftrace *ftrace, const char *line, FILE *out)
+{
+ char *filtered;
+
+ if (!ftrace->filter_retaddr) {
+ fprintf(out, "%s\n", line);
+ return 0;
+ }
+
+ filtered = strdup(line);
+ if (!filtered) {
+ pr_err("Not enough memory\n");
+ return -ENOMEM;
+ }
+
+ ftrace_filter_retaddr(filtered);
+ fprintf(out, "%s\n", filtered);
+ free(filtered);
+ return 0;
+}
+
+/**
+ * ftrace_process_fgraph_line - Process and output a function graph trace line
+ * @ftrace: Perf ftrace context
+ * @line: The trace line to process
+ * @out: File stream to output to
+ *
+ * Process a trace line from the function_graph tracer. If --inline is set,
+ * this function parses the return address comment, resolves it to its
+ * inlined callchain, and synthesises intermediate frames into the streamed
+ * call graph.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FILE *out)
+{
+ char caller_sym[128];
+ char blank_prefix[128];
+ const char *inlined_names[FTRACE_MAX_INLINES];
+ const char *pipe_char;
+ const char *after_pipe;
+ const char *func_start;
+ struct ftrace_cpu_state *cs;
+ u64 caller_offset = 0;
+ int num_inlines = 0;
+ int cpu = -1;
+ int leading_spaces;
+ int extra_indent;
+ int caller_idx;
+ int active_inlines;
+ int common;
+ int match_idx;
+ int i;
+ bool is_exit;
+ bool has_open_brace;
+ bool has_semicolon;
+ bool has_retaddr = false;
+
+ if (!ftrace->use_inline)
+ return print_line(ftrace, line, out);
+
+ pipe_char = find_func_column(ftrace, line, &cpu);
+ if (!pipe_char) {
+ parse_context_switch(line);
+ return print_line(ftrace, line, out);
+ }
+
+ cs = get_cpu_state(cpu);
+ if (!cs)
+ return print_line(ftrace, line, out);
+
+ make_blank_prefix(ftrace, line, pipe_char, blank_prefix, sizeof(blank_prefix));
+
+ after_pipe = pipe_char + 1;
+ func_start = skip_spaces(after_pipe);
+ leading_spaces = func_start - after_pipe;
+
+ if (*func_start == '\0')
+ return print_line(ftrace, line, out);
+
+ is_exit = (*func_start == '}');
+
+ if (is_exit) {
+ char exit_name[128];
+
+ extract_exit_func_name(func_start, exit_name, sizeof(exit_name));
+
+ /* Find matching frame on stack */
+ match_idx = -1;
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined) {
+ if (exit_name[0] == '\0' ||
+ (cs->stack[i].name &&
+ strcmp(cs->stack[i].name, exit_name) == 0)) {
+ match_idx = i;
+ break;
+ }
+ }
+ }
+
+ if (match_idx >= 0) {
+ cs->overflow_depth = 0;
+ } else if (cs->overflow_depth > 0) {
+ cs->overflow_depth--;
+ extra_indent = cs->inlined_depth * 2;
+ fprintf(out, "%.*s%*s%s\n",
+ (int)(pipe_char - line + 1), line,
+ leading_spaces + extra_indent, "", func_start);
+ return 0;
+ }
+
+ if (match_idx >= 0) {
+ /* Close all inlined and orphaned frames above match_idx */
+ while (cs->depth > 0 && cs->depth - 1 > match_idx) {
+ struct ftrace_stack_frame *top = &cs->stack[cs->depth - 1];
+
+ if (top->inlined) {
+ int exit_indent = top->indent;
+
+ if (cs->inlined_depth > 0)
+ cs->inlined_depth--;
+ fprintf(out, "%s%*s} /* %s (inline) */\n",
+ blank_prefix, exit_indent, "", top->name);
+ }
+ free(top->name);
+ cs->depth--;
+ }
+
+ /* Pop the matched real frame */
+ if (cs->depth > 0 && match_idx == cs->depth - 1 &&
+ !cs->stack[cs->depth - 1].inlined) {
+ free(cs->stack[cs->depth - 1].name);
+ cs->depth--;
+ }
+
+ extra_indent = cs->inlined_depth * 2;
+ } else {
+ extra_indent = 0;
+ }
+
+ fprintf(out, "%.*s%*s%s\n",
+ (int)(pipe_char - line + 1), line,
+ leading_spaces + extra_indent, "", func_start);
+ return 0;
+ }
+
+ /* It's an entry or a leaf call */
+ has_open_brace = (strchr(func_start, '{') != NULL);
+ has_semicolon = (strchr(func_start, ';') != NULL);
+
+ if (ftrace_parse_retaddr(func_start, caller_sym, sizeof(caller_sym), &caller_offset)) {
+ has_retaddr = true;
+ num_inlines = ftrace_resolve_inlines(ftrace->machine, caller_sym,
+ caller_offset, inlined_names,
+ FTRACE_MAX_INLINES);
+ }
+
+ /* Determine which inlined frames are currently open above the caller */
+ caller_idx = -1;
+ if (has_retaddr) {
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined && cs->stack[i].name &&
+ strcmp(cs->stack[i].name, caller_sym) == 0) {
+ caller_idx = i;
+ break;
+ }
+ }
+
+ /* If caller is not on the tracked stack, do not synthesise inlines */
+ if (caller_idx < 0) {
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ num_inlines = 0;
+ }
+ }
+
+ active_inlines = 0;
+ if (caller_idx >= 0) {
+ for (i = caller_idx + 1; i < cs->depth; i++) {
+ if (cs->stack[i].inlined)
+ active_inlines++;
+ else
+ break;
+ }
+ }
+
+ /* Find common prefix of inlined functions */
+ common = 0;
+ if (caller_idx >= 0) {
+ while (common < active_inlines && common < num_inlines &&
+ cs->stack[caller_idx + 1 + common].name &&
+ !strcmp(cs->stack[caller_idx + 1 + common].name,
+ inlined_names[common])) {
+ common++;
+ }
+ }
+
+ /* Close stale callee frames and inactive inlined frames above caller */
+ if (caller_idx >= 0)
+ cs->overflow_depth = 0;
+
+ while (caller_idx >= 0 && cs->depth > caller_idx + 1 + common) {
+ struct ftrace_stack_frame *top = &cs->stack[cs->depth - 1];
+
+ if (top->inlined) {
+ int exit_indent = top->indent;
+
+ if (cs->inlined_depth > 0)
+ cs->inlined_depth--;
+ fprintf(out, "%s%*s} /* %s (inline) */\n",
+ blank_prefix, exit_indent, "", top->name);
+ }
+ free(top->name);
+ cs->depth--;
+ }
+
+ /* Open new inlined frames */
+ for (i = common; i < num_inlines; i++) {
+ int entry_indent = leading_spaces + cs->inlined_depth * 2;
+
+ if (!inlined_names[i]) {
+ pr_err("Not enough memory\n");
+ for (int j = 0; j < num_inlines; j++)
+ free((void *)inlined_names[j]);
+ return -ENOMEM;
+ }
+ if (cs->depth >= FTRACE_MAX_STACK)
+ break;
+
+ fprintf(out, "%s%*s%s() { /* (inline) */\n",
+ blank_prefix, entry_indent, "", inlined_names[i]);
+ cs->stack[cs->depth].name = strdup(inlined_names[i]);
+ if (!cs->stack[cs->depth].name) {
+ pr_err("Not enough memory\n");
+ for (int j = 0; j < num_inlines; j++)
+ free((void *)inlined_names[j]);
+ return -ENOMEM;
+ }
+ cs->stack[cs->depth].inlined = true;
+ cs->stack[cs->depth].indent = entry_indent;
+ cs->depth++;
+ cs->inlined_depth++;
+ }
+
+ /* Print current line with updated indentation */
+ extra_indent = cs->inlined_depth * 2;
+
+ if (ftrace->filter_retaddr) {
+ char *filtered = strdup(func_start);
+
+ if (filtered) {
+ ftrace_filter_retaddr(filtered);
+ fprintf(out, "%.*s%*s%s\n",
+ (int)(pipe_char - line + 1), line,
+ leading_spaces + extra_indent, "", filtered);
+ free(filtered);
+ } else {
+ pr_err("Not enough memory\n");
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ return -ENOMEM;
+ }
+ } else {
+ fprintf(out, "%.*s%*s%s\n",
+ (int)(pipe_char - line + 1), line,
+ leading_spaces + extra_indent, "", func_start);
+ }
+
+ /* If non-leaf entry, push onto stack */
+ if (has_open_brace && !has_semicolon) {
+ char entry_name[128];
+
+ extract_entry_func_name(func_start, entry_name, sizeof(entry_name));
+ if (cs->depth < FTRACE_MAX_STACK) {
+ cs->stack[cs->depth].name = strdup(entry_name);
+ if (!cs->stack[cs->depth].name) {
+ pr_err("Not enough memory\n");
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ return -ENOMEM;
+ }
+ cs->stack[cs->depth].inlined = false;
+ cs->stack[cs->depth].indent = leading_spaces + extra_indent;
+ cs->depth++;
+ } else {
+ cs->overflow_depth++;
+ }
+ }
+
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ return 0;
+}
+
+/**
+ * perf_ftrace__setup_inlines - Initialise symbol resolution for inlines
+ * @ftrace: Perf ftrace context
+ *
+ * Initialise symbol tracking and load the kernel image (vmlinux) for
+ * DWARF debug information parsing.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+int perf_ftrace__setup_inlines(struct perf_ftrace *ftrace)
+{
+ struct map *kmap;
+
+ if (!ftrace->use_inline)
+ return 0;
+
+ symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL);
+ symbol_conf.inline_name = true;
+ if (symbol_conf.vmlinux_name)
+ symbol_conf.ignore_vmlinux_buildid = true;
+
+ if (symbol__init(NULL) < 0) {
+ pr_warning("Failed to initialize symbols for inlined functions\n");
+ return -1;
+ }
+
+ ftrace->machine = machine__new_host(NULL);
+ if (!ftrace->machine) {
+ pr_warning("Failed to create host machine for symbols\n");
+ return -1;
+ }
+
+ if (symbol_conf.vmlinux_name) {
+ kmap = machine__kernel_map(ftrace->machine);
+ if (kmap)
+ dso__load_vmlinux(map__dso(kmap), kmap, symbol_conf.vmlinux_name, false);
+ } else {
+ machine__load_vmlinux_path(ftrace->machine);
+ }
+
+ return 0;
+}
+
+/**
+ * perf_ftrace__cleanup_inlines - Clean up resources used for inline resolution
+ * @ftrace: Perf ftrace context
+ *
+ * Free the machine context and the CPU stack tracking states.
+ */
+void perf_ftrace__cleanup_inlines(struct perf_ftrace *ftrace)
+{
+ if (ftrace->machine) {
+ machine__delete(ftrace->machine);
+ ftrace->machine = NULL;
+ }
+ ftrace_cpu_states__clear();
+}
diff --git a/tools/perf/util/ftrace.h b/tools/perf/util/ftrace.h
index 50216d3fef64..404d596bb60f 100644
--- a/tools/perf/util/ftrace.h
+++ b/tools/perf/util/ftrace.h
@@ -1,6 +1,7 @@
#ifndef __PERF_FTRACE_H__
#define __PERF_FTRACE_H__
+#include <stdio.h>
#include <linux/list.h>
#include "target.h"
@@ -41,6 +42,8 @@ struct perf_ftrace {
int graph_tail;
bool filter_retaddr;
bool filter_retaddr_set;
+ bool use_inline;
+ struct machine *machine;
};
struct filter_entry {
@@ -95,7 +98,16 @@ perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace __maybe_unused)
#endif /* HAVE_BPF_SKEL */
+struct machine;
+
bool ftrace_parse_retaddr(const char *str, char *sym_name, size_t sym_len, u64 *offset);
void ftrace_filter_retaddr(char *str);
+int ftrace_resolve_inlines(struct machine *machine,
+ const char *sym_name, u64 offset,
+ const char **inlined_names, int max_inlines);
+int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FILE *out);
+void ftrace_cpu_states__clear(void);
+int perf_ftrace__setup_inlines(struct perf_ftrace *ftrace);
+void perf_ftrace__cleanup_inlines(struct perf_ftrace *ftrace);
#endif /* __PERF_FTRACE_H__ */
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH perf-tools-next v8 4/6] perf ftrace: Support display of inlined functions in function graph tracer
2026-10-18 2:37 ` [PATCH perf-tools-next v8 4/6] perf ftrace: Support display of inlined functions in function graph tracer Aaron Tomlin
@ 2026-10-18 3:00 ` sashiko-bot
0 siblings, 0 replies; 13+ messages in thread
From: sashiko-bot @ 2026-10-18 3:00 UTC (permalink / raw)
To: Aaron Tomlin; +Cc: linux-perf-users
Thank you for your contribution! Sashiko AI review found 5 potential issue(s) to consider:
- [Medium] perf ftrace: find_func_column() misses latency-format column in verbose mode
- [Medium] perf ftrace: select_tracer() overrides function_graph tracer with --inline
- [Medium] perf ftrace: idle tasks across CPUs share a single ftrace_task_state
- [Medium] perf ftrace: perf_ftrace__setup_inlines() ignores vmlinux load failures
- [Medium] perf test ftrace: test_inline_processing() hardcodes build-specific offset
--
commit a1b712dfdc11c697976e8ccf628bd22443d5d421
Author: Aaron Tomlin <atomlin@atomlin.com>
perf ftrace: Support display of inlined functions in function graph tracer
This commit introduces the --inline option to perf ftrace to resolve caller
return addresses via DWARF debug information in vmlinux and synthesize
inlined function frames into the function_graph call stream. It also adds per-
task call stack tracking, the -k/--vmlinux option, and unit tests in
tools/perf/tests/ftrace.c.
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index c715cbebaabe9..cdacf5af14a65 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
[ ... ]
> @@ -1973,6 +1991,20 @@ int cmd_ftrace(int argc, const char **argv)
>
> switch (subcmd) {
> case PERF_FTRACE_TRACE:
> + if (ftrace.use_inline) {
> + if (ftrace.tracer && strcmp(ftrace.tracer, "function_graph") != 0) {
> + pr_err("Error: --inline option is only supported with function_graph tracer\n");
> + ret = -EINVAL;
> + goto out_delete_filters;
> + }
> + ftrace.tracer = "function_graph";
> + }
[Severity: Medium]
Can select_tracer() override ftrace.tracer back to "function" after this
check runs?
In cmd_ftrace(), ftrace.tracer defaults to DEFAULT_TRACER ("function_graph"),
so running perf ftrace -T <func> --inline (or -N <func> --inline) passes this
validation. However, when __cmd_ftrace() starts, it calls select_tracer():
tools/perf/builtin-ftrace.c:select_tracer() {
bool graph = !list_empty(&ftrace->graph_funcs) ||
!list_empty(&ftrace->nograph_funcs);
bool func = !list_empty(&ftrace->filters) ||
!list_empty(&ftrace->notrace);
/* The function_graph has priority over function tracer. */
if (graph)
ftrace->tracer = "function_graph";
else if (func)
ftrace->tracer = "function";
...
}
Because filters or notrace is non-empty and graph_funcs/nograph_funcs are
empty, select_tracer() overwrites ftrace->tracer to "function", causing
__cmd_ftrace() to configure the function tracer while feeding its output into
ftrace_process_fgraph_line(). Conversely, if ftrace.tracer is set to
"function" in perfconfig and the user runs perf ftrace -G <func> --inline,
cmd_ftrace() rejects it before select_tracer() can select "function_graph".
Should the --inline tracer check or enforcement happen in integration with
select_tracer()?
> diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
> index 596cecee844e3..6e20a61860a9e 100644
> --- a/tools/perf/tests/ftrace.c
> +++ b/tools/perf/tests/ftrace.c
> @@ -62,6 +66,465 @@ static int test_filter_retaddr(void)
> return TEST_OK;
> }
>
> +#define TEST_CHECK(text, cond) \
> +do { \
> + if (!(cond)) { \
> + pr_debug("FAILED %s:%d %s\n", __FILE__, __LINE__, text); \
> + ret = TEST_FAIL; \
> + goto out_cleanup; \
> + } \
> +} while (0)
> +
> +static int test_inline_processing(void)
> +{
> + static const char line_entry[] = " 0) | wake_up_new_task() {";
> + static const char line_enqueue[] =
> + " 0) | enqueue_task() { /* <-wake_up_new_task+0x1d1/0x3e0 */";
[ ... ]
> + vmlinux = vmlinux_path__find();
> +
> + /* If vmlinux is present, test resolving symbols */
> + if (vmlinux) {
> + symbol_conf.vmlinux_name = vmlinux;
> + symbol_conf.ignore_vmlinux_buildid = true;
> + ret = perf_ftrace__setup_inlines(&ftrace);
> + if (ret < 0 || !ftrace.machine) {
> + ret = TEST_SKIP;
> + goto out_cleanup;
> + }
> +
> + out = open_memstream(&out_buf, &out_len);
> + if (!out) {
> + ret = TEST_FAIL;
> + goto out_cleanup;
> + }
> +
> + /* Feed simulated function graph trace lines */
> + ftrace_process_fgraph_line(&ftrace, line_entry, out);
> + ftrace_process_fgraph_line(&ftrace, line_enqueue, out);
> + ftrace_process_fgraph_line(&ftrace, line_exit_enqueue, out);
> + ftrace_process_fgraph_line(&ftrace, line_exit_task, out);
> +
> + fclose(out);
> + ftrace_cpu_states__clear();
> +
> + pr_debug("Processed output:\n%s\n", out_buf);
> +
> + /* Verify activate_task entry with inline hint */
> + TEST_CHECK("contains activate_task entry with inline hint",
> + strstr(out_buf, "activate_task() { /* (inline) */") != NULL);
[Severity: Medium]
Will test_inline_processing() fail on systems where the host vmlinux does not
have an inlined activate_task() call specifically at wake_up_new_task+0x1d1?
When vmlinux_path__find() finds a host vmlinux file, the test feeds
line_enqueue with the hardcoded offset wake_up_new_task+0x1d1/0x3e0 and
requires DWARF resolution to produce "activate_task() { /* (inline) */". Because
the instruction offset +0x1d1 (and whether activate_task is inlined) depends
on the architecture, compiler, optimization flags, and kernel configuration,
as well as DWARF support in both vmlinux and the perf build, this check will
return TEST_FAIL on other kernel builds.
Also, symbol_conf.vmlinux_name and symbol_conf.ignore_vmlinux_buildid are
modified here without being restored in out_cleanup.
> diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
> index aaf108369c365..020551a4d7e00 100644
> --- a/tools/perf/util/ftrace.c
> +++ b/tools/perf/util/ftrace.c
[ ... ]
> +static bool parse_context_switch(const char *line)
> +{
> + const char *arrow = strstr(line, " => ");
> + const char *p;
> + const char *next;
> + const char *hyphen;
> + char *endptr;
> + long cpu_val;
> + long pid_val;
> +
> + if (!arrow)
> + return false;
> +
> + /* Find CPU number before ')' */
> + p = skip_spaces(line);
> + cpu_val = strtol(p, &endptr, 10);
> + if (endptr == p || *endptr != ')')
> + return false;
> +
> + if (cpu_val < 0 || cpu_val >= FTRACE_MAX_CPUS)
> + return false;
> +
> + /* Parse the target task's PID after the arrow (" => comm-pid") */
> + next = skip_spaces(arrow + 4);
> + hyphen = strrchr(next, '-');
> + if (!hyphen)
> + return false;
> +
> + pid_val = strtol(hyphen + 1, &endptr, 10);
> + if (endptr == hyphen + 1)
> + return false;
> +
> + ftrace_cpu_tasks[cpu_val] = get_task_state((pid_t)pid_val);
> + return true;
> +}
[Severity: Medium]
What happens in system-wide tracing when multiple CPUs switch to the idle
task (<idle>-0)?
Each CPU has its own swapper thread, all with PID 0, which ftrace formats as
"<idle>-0" in context-switch banners and in the verbose comm-pid column. When
parse_context_switch() or find_func_column() sees "<idle>-0", pid_val is 0 and
get_task_state(0) returns the same struct ftrace_task_state instance for all
idle CPUs.
If multiple CPUs are executing in idle simultaneously, won't their interleaved
trace lines share and corrupt the same stack, depth, and inlined_depth? (Note
that get_cpu_state() uses -1 - cpu before a banner is seen; should pid_val == 0
similarly map to a per-CPU state such as -1 - cpu_val?)
[ ... ]
> +static void make_blank_prefix(const struct perf_ftrace *ftrace, const char *line,
> + const char *pipe_char, char *buf, size_t size)
> +{
> + size_t len = pipe_char - line + 1;
> + const char *start_delim = NULL;
> + char *c;
> +
> + if (!size)
> + return;
> +
> + if (len >= size)
> + len = size - 1;
> +
> + memcpy(buf, line, len);
> + buf[len] = '\0';
> +
> + if (*pipe_char == '|') {
> + if (ftrace->graph_verbose) {
> + /* In verbose mode, duration starts after the comm-pid '|' */
> + for (const char *p = pipe_char - 1; p >= line; p--) {
> + if (*p == '|') {
> + start_delim = p;
> + break;
> + }
> + }
> + } else {
> + /* In default mode, duration starts after CPU ')' */
> + start_delim = strchr(line, ')');
> + }
> +
> + if (start_delim && start_delim < pipe_char && start_delim < line + len) {
> + char *end = buf + (pipe_char - line);
> +
> + if (end > buf + len)
> + end = buf + len;
> + for (c = buf + (start_delim - line) + 1; c < end; c++)
> + *c = ' ';
> + }
> + }
> +}
> +
> +/**
> + * find_func_column - Locate the function call column in a trace record
> + * @ftrace: Perf ftrace context
> + * @line: The trace record to parse
> + * @cpu: Pointer to store the CPU number
> + *
> + * Locate the context columns that precede the function call column of a
> + * function_graph trace record. Depending on the funcgraph-* options set
> + * up by perf, a record looks like:
> + *
> + * [ abstime |] CPU) [ comm-pid |] [ duration |] func
> + *
> + * The abstime and comm-pid columns are only present with --graph-opts
> + * verbose.
> + *
> + * Return: A pointer to the last character of the context columns (the final
> + * '|' delimiter). Returns NULL if @line is not a trace record.
> + */
> +static const char *find_func_column(const struct perf_ftrace *ftrace,
> + const char *line, int *cpu)
> +{
> + const char *p = line;
> + char *endptr;
> + long val;
> +
> + if (ftrace->graph_verbose) {
> + p = strchr(p, '|');
> + if (!p)
> + return NULL;
> + p++;
> + }
> +
> + p = skip_spaces(p);
> + val = strtol(p, &endptr, 10);
> + if (endptr == p || *endptr != ')')
> + return NULL;
> + p = endptr;
> +
> + if (ftrace->graph_verbose) {
> + const char *proc_start = p;
> +
> + p = strchr(p, '|');
> + if (!p)
> + return NULL;
> +
> + if (val >= 0 && val < FTRACE_MAX_CPUS) {
> + const char *h = p - 1;
> +
> + while (h >= proc_start && *h != '-')
> + h--;
> + if (h >= proc_start && *h == '-') {
> + long pid_val = strtol(h + 1, NULL, 10);
> + ftrace_cpu_tasks[(int)val] = get_task_state((pid_t)pid_val);
> + }
> + }
> + }
> +
> + p = strchr(p + 1, '|');
> + if (!p)
> + return NULL;
> +
> + *cpu = (int)val;
> + return p;
> +}
[Severity: Medium]
Does find_func_column() account for the latency-format column when
ftrace->graph_verbose is enabled?
In tools/perf/builtin-ftrace.c, set_tracing_funcgraph_verbose() enables three
tracing options:
static int set_tracing_funcgraph_verbose(struct perf_ftrace *ftrace)
{
if (!ftrace->graph_verbose)
return 0;
if (write_tracing_option_file("funcgraph-proc", "1") < 0)
return -1;
if (write_tracing_option_file("funcgraph-abstime", "1") < 0)
return -1;
if (write_tracing_option_file("latency-format", "1") < 0)
return -1;
return 0;
}
When latency-format is enabled alongside funcgraph-abstime and funcgraph-proc,
the kernel's function_graph tracer emits four '|' delimiters before the
function column:
<abstime> | <cpu>) <comm>-<pid> | <lat-fmt> | <duration> | <func>
However, find_func_column() only advances past three '|' characters in
verbose mode, returning a pointer to the '|' at the end of the latency-format
column instead of the '|' at the end of the duration column.
Consequently, in ftrace_process_fgraph_line():
1. On entry lines with a blank duration column, func_start points to the
duration column's trailing "| func() {", so extract_entry_func_name()
extracts "|" as the function name.
2. On exit lines with a duration (e.g. " 1.234 us | }"), func_start
points to the duration value rather than '}', so is_exit is false and
function exits are never detected.
3. make_blank_prefix() blanks out the latency-format column and truncates the
prefix before the duration column.
[ ... ]
> +int perf_ftrace__setup_inlines(struct perf_ftrace *ftrace)
> +{
> + struct map *kmap;
> +
> + if (!ftrace->use_inline)
> + return 0;
> +
> + symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL);
> + symbol_conf.inline_name = true;
> + if (symbol_conf.vmlinux_name)
> + symbol_conf.ignore_vmlinux_buildid = true;
> +
> + if (symbol__init(NULL) < 0) {
> + pr_warning("Failed to initialize symbols for inlined functions\n");
> + return -1;
> + }
> +
> + ftrace->machine = machine__new_host(NULL);
> + if (!ftrace->machine) {
> + pr_warning("Failed to create host machine for symbols\n");
> + return -1;
> + }
> +
> + if (symbol_conf.vmlinux_name) {
> + kmap = machine__kernel_map(ftrace->machine);
> + if (kmap)
> + dso__load_vmlinux(map__dso(kmap), kmap, symbol_conf.vmlinux_name, false);
> + } else {
> + machine__load_vmlinux_path(ftrace->machine);
> + }
> +
> + return 0;
> +}
[Severity: Medium]
Should perf_ftrace__setup_inlines() check the return values of
dso__load_vmlinux() and machine__load_vmlinux_path()?
Both functions return <= 0 when loading vmlinux fails (for example, if the
user passes a non-existent or invalid path via -k/--vmlinux, or if no vmlinux
with symbols is found on the system). Because the return values are ignored
and perf_ftrace__setup_inlines() unconditionally returns 0,perf ftrace
--inline proceeds without a loaded vmlinux DSO (and test_inline_processing()
does not skip when vmlinux cannot be loaded).
If loading fails, should perf_ftrace__setup_inlines() clean up ftrace->machine
and return an error?
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261018023753.584873-1-atomlin@atomlin.com?part=4
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH perf-tools-next v8 5/6] perf ftrace: Support omitting execution duration in function graph tracer
2026-10-18 2:37 [PATCH perf-tools-next v8 0/6] perf ftrace: Support inlined functions and display enhancements in function graph tracer Aaron Tomlin
` (3 preceding siblings ...)
2026-10-18 2:37 ` [PATCH perf-tools-next v8 4/6] perf ftrace: Support display of inlined functions in function graph tracer Aaron Tomlin
@ 2026-10-18 2:37 ` Aaron Tomlin
2026-10-18 2:51 ` sashiko-bot
2026-10-18 2:37 ` [PATCH perf-tools-next v8 6/6] perf ftrace: Filter extraneous functions " Aaron Tomlin
5 siblings, 1 reply; 13+ messages in thread
From: Aaron Tomlin @ 2026-10-18 2:37 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung
Cc: mark.rutland, alexander.shishkin, jolsa, irogers, adrian.hunter,
atomlin, linux-perf-users, linux-kernel
The Linux kernel's function graph tracer displays the execution duration
of completed functions by default via the funcgraph-duration trace
option. Whilst informative, in scenarios where the user is solely
interested in the call graph topology, call flow sequence, or inlined
hierarchy, the duration column can add visual clutter and widen each
trace line.
Introduce support for omitting function duration by exposing the
noduration sub-option under --graph-opts. Consistent with the existing
nosleep-time and noirqs sub-options, it accepts an optional value, so
noduration=0 explicitly retains the duration column; when specified
more than once, the last occurrence takes effect. When enabled, perf
ftrace configures the kernel's tracing instance option
funcgraph-duration to 0, and restores it to 1 upon resetting tracing
options.
Because the profile subcommand fundamentally requires function execution
duration to compute timing statistics and call counts, combining
noduration with perf ftrace profile is rejected with an explicit error.
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
tools/perf/Documentation/perf-ftrace.txt | 1 +
tools/perf/builtin-ftrace.c | 34 ++++++++++++-
tools/perf/tests/ftrace.c | 64 ++++++++++++++++++++++--
tools/perf/util/ftrace.c | 18 +++++--
tools/perf/util/ftrace.h | 1 +
5 files changed, 109 insertions(+), 9 deletions(-)
diff --git a/tools/perf/Documentation/perf-ftrace.txt b/tools/perf/Documentation/perf-ftrace.txt
index 445c8ccc22d2..a50f3bae86b2 100644
--- a/tools/perf/Documentation/perf-ftrace.txt
+++ b/tools/perf/Documentation/perf-ftrace.txt
@@ -129,6 +129,7 @@ OPTIONS for 'perf ftrace trace'
- retaddr - Show function return address.
- nosleep-time - Measure on-CPU time only for function_graph tracer.
- noirqs - Ignore functions that happen inside interrupt.
+ - noduration - Do not print function duration.
- verbose - Show process names, PIDs, timestamps, etc.
- thresh=<n> - Setup trace duration threshold in microseconds.
- depth=<n> - Set max depth for function graph tracer to follow.
diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index cdacf5af14a6..2309ef88ffdb 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -300,6 +300,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
write_tracing_option_file("func_stack_trace", "0");
write_tracing_option_file("sleep-time", "1");
write_tracing_option_file("funcgraph-irqs", "1");
+ write_tracing_option_file("funcgraph-duration", "1");
write_tracing_option_file("funcgraph-proc", "0");
write_tracing_option_file("funcgraph-abstime", "0");
write_tracing_option_file("funcgraph-tail", "0");
@@ -599,6 +600,26 @@ static int set_tracing_funcgraph_irqs(struct perf_ftrace *ftrace)
return 0;
}
+/**
+ * set_tracing_funcgraph_duration - Set funcgraph-duration tracing option
+ * @ftrace: Perf ftrace context
+ *
+ * Configure the kernel's funcgraph-duration tracing option based on the
+ * ftrace->graph_noduration flag.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
+static int set_tracing_funcgraph_duration(struct perf_ftrace *ftrace)
+{
+ if (!ftrace->graph_noduration)
+ return 0;
+
+ if (write_tracing_option_file("funcgraph-duration", "0") < 0)
+ return -1;
+
+ return 0;
+}
+
static int set_tracing_funcgraph_verbose(struct perf_ftrace *ftrace)
{
if (!ftrace->graph_verbose)
@@ -708,6 +729,11 @@ static int set_tracing_options(struct perf_ftrace *ftrace)
return -1;
}
+ if (set_tracing_funcgraph_duration(ftrace) < 0) {
+ pr_err("failed to set tracing option funcgraph-duration\n");
+ return -1;
+ }
+
if (set_tracing_funcgraph_verbose(ftrace) < 0) {
pr_err("failed to set tracing option funcgraph-proc/funcgraph-abstime\n");
return -1;
@@ -1774,6 +1800,7 @@ static int parse_graph_tracer_opts(const struct option *opt,
{ .name = "retaddr", .value_ptr = &ftrace->graph_retaddr },
{ .name = "nosleep-time", .value_ptr = &ftrace->graph_nosleep_time },
{ .name = "noirqs", .value_ptr = &ftrace->graph_noirqs },
+ { .name = "noduration", .value_ptr = &ftrace->graph_noduration },
{ .name = "verbose", .value_ptr = &ftrace->graph_verbose },
{ .name = "thresh", .value_ptr = &ftrace->graph_thresh },
{ .name = "depth", .value_ptr = &ftrace->graph_depth },
@@ -1863,7 +1890,7 @@ int cmd_ftrace(int argc, const char **argv)
OPT_CALLBACK('g', "nograph-funcs", &ftrace.nograph_funcs, "func",
"Set nograph filter on given functions", parse_filter_func),
OPT_CALLBACK(0, "graph-opts", &ftrace, "options",
- "Graph tracer options, available options: args,retval,retval-hex,retaddr,nosleep-time,noirqs,verbose,thresh=<n>,depth=<n>",
+ "Graph tracer options, available options: args,retval,retval-hex,retaddr,nosleep-time,noirqs,noduration,verbose,thresh=<n>,depth=<n>,tail",
parse_graph_tracer_opts),
OPT_CALLBACK('m', "buffer-size", &ftrace.percpu_buffer_size, "size",
"Size of per cpu buffer, needs to use a B, K, M or G suffix.", parse_buffer_size),
@@ -2070,6 +2097,11 @@ int cmd_ftrace(int argc, const char **argv)
cmd_func = __cmd_latency;
break;
case PERF_FTRACE_PROFILE:
+ if (ftrace.graph_noduration) {
+ pr_err("Error: 'noduration' option cannot be used with profile\n");
+ ret = -EINVAL;
+ goto out_delete_filters;
+ }
cmd_func = __cmd_profile;
break;
case PERF_FTRACE_NONE:
diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
index 6e20a61860a9..d83dee5a050d 100644
--- a/tools/perf/tests/ftrace.c
+++ b/tools/perf/tests/ftrace.c
@@ -97,6 +97,18 @@ static int test_inline_processing(void)
" 1234.567890 | 0) bash-1234 | | activate_task() { /* (inline) */";
static const char line_verbose_inline_exit[] =
" 1234.567890 | 0) bash-1234 | | } /* activate_task (inline) */";
+ static const char line_verbose_nodur_entry[] =
+ " 1234.567890 | 0) bash-1234 | wake_up_new_task() {";
+ static const char line_verbose_nodur_enqueue[] =
+ " 1234.567890 | 0) bash-1234 | enqueue_task() { /* <-wake_up_new_task+0x1d1/0x3e0 */";
+ static const char line_verbose_nodur_exit_enqueue[] =
+ " 1234.567890 | 0) bash-1234 | } /* enqueue_task */";
+ static const char line_verbose_nodur_exit_task[] =
+ " 1234.567890 | 0) bash-1234 | } /* wake_up_new_task */";
+ static const char line_verbose_nodur_inline_entry[] =
+ " 1234.567890 | 0) bash-1234 | activate_task() { /* (inline) */";
+ static const char line_verbose_nodur_inline_exit[] =
+ " 1234.567890 | 0) bash-1234 | } /* activate_task (inline) */";
struct perf_ftrace ftrace;
char *out_buf = NULL;
size_t out_len = 0;
@@ -318,6 +330,33 @@ static int test_inline_processing(void)
free(out_buf);
out_buf = NULL;
ftrace.graph_verbose = false;
+
+ /* Test verbose noduration prefix formatting for synthesised inlines */
+ ftrace.graph_verbose = true;
+ ftrace.graph_noduration = true;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_cleanup;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_verbose_nodur_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_verbose_nodur_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_verbose_nodur_exit_enqueue, out);
+ ftrace_process_fgraph_line(&ftrace, line_verbose_nodur_exit_task, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ TEST_CHECK("verbose noduration inline entry preserves comm-pid",
+ strstr(out_buf, line_verbose_nodur_inline_entry) != NULL);
+ TEST_CHECK("verbose noduration inline exit preserves comm-pid",
+ strstr(out_buf, line_verbose_nodur_inline_exit) != NULL);
+
+ free(out_buf);
+ out_buf = NULL;
+ ftrace.graph_verbose = false;
+ ftrace.graph_noduration = false;
}
out_cleanup:
@@ -422,20 +461,36 @@ static int test_column_layouts(void)
{
static const struct {
const char *desc;
- bool verbose;
+ bool noduration;
+ bool verbose;
const char *in;
const char *expected;
} cases[] = {
{
- "default layout", false,
+ "default layout", false, false,
" 7) | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
" 7) | wake_up_new_task() {\n",
},
{
- "verbose layout", true,
+ "noduration layout", true, false,
+ " 7) housekeeping_any_cpu(); /* <-ring_buffer_unlock_commit+0x1a0/0x1f0 */",
+ " 7) housekeeping_any_cpu();\n",
+ },
+ {
+ "verbose layout", false, true,
" 1234.567890 | 7) bash-1234 | | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
" 1234.567890 | 7) bash-1234 | | wake_up_new_task() {\n",
},
+ {
+ "verbose noduration layout", true, true,
+ " 1234.567890 | 7) bash-1234 | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
+ " 1234.567890 | 7) bash-1234 | wake_up_new_task() {\n",
+ },
+ {
+ "noduration context switch marker", true, false,
+ " 7) bash-1234 => sh-5678",
+ " 7) bash-1234 => sh-5678\n",
+ },
};
struct perf_ftrace ftrace;
char *out_buf = NULL;
@@ -447,7 +502,8 @@ static int test_column_layouts(void)
memset(&ftrace, 0, sizeof(ftrace));
ftrace.use_inline = true;
ftrace.filter_retaddr = true;
- ftrace.graph_verbose = cases[i].verbose;
+ ftrace.graph_noduration = cases[i].noduration;
+ ftrace.graph_verbose = cases[i].verbose;
out = open_memstream(&out_buf, &out_len);
if (!out)
diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
index 020551a4d7e0..66c19116ee64 100644
--- a/tools/perf/util/ftrace.c
+++ b/tools/perf/util/ftrace.c
@@ -388,6 +388,9 @@ static void make_blank_prefix(const struct perf_ftrace *ftrace, const char *line
memcpy(buf, line, len);
buf[len] = '\0';
+ if (ftrace->graph_noduration)
+ return;
+
if (*pipe_char == '|') {
if (ftrace->graph_verbose) {
/* In verbose mode, duration starts after the comm-pid '|' */
@@ -426,10 +429,12 @@ static void make_blank_prefix(const struct perf_ftrace *ftrace, const char *line
* [ abstime |] CPU) [ comm-pid |] [ duration |] func
*
* The abstime and comm-pid columns are only present with --graph-opts
- * verbose.
+ * verbose, whereas the duration column, and thus its '|' delimiter, is
+ * omitted with --graph-opts noduration.
*
* Return: A pointer to the last character of the context columns (the final
- * '|' delimiter). Returns NULL if @line is not a trace record.
+ * '|' delimiter, or the CPU ')' when no delimiter follows it). Returns NULL
+ * if @line is not a trace record (e.g. a header or a context switch marker).
*/
static const char *find_func_column(const struct perf_ftrace *ftrace,
const char *line, int *cpu)
@@ -470,9 +475,14 @@ static const char *find_func_column(const struct perf_ftrace *ftrace,
}
}
- p = strchr(p + 1, '|');
- if (!p)
+ if (!ftrace->graph_noduration) {
+ p = strchr(p + 1, '|');
+ if (!p)
+ return NULL;
+ } else if (strstr(p, " => ")) {
+ /* Context switch marker: " CPU) comm-pid => comm-pid" */
return NULL;
+ }
*cpu = (int)val;
return p;
diff --git a/tools/perf/util/ftrace.h b/tools/perf/util/ftrace.h
index 404d596bb60f..9fc4741d007f 100644
--- a/tools/perf/util/ftrace.h
+++ b/tools/perf/util/ftrace.h
@@ -37,6 +37,7 @@ struct perf_ftrace {
int graph_retaddr;
int graph_nosleep_time;
int graph_noirqs;
+ int graph_noduration;
int graph_verbose;
int graph_thresh;
int graph_tail;
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH perf-tools-next v8 5/6] perf ftrace: Support omitting execution duration in function graph tracer
2026-10-18 2:37 ` [PATCH perf-tools-next v8 5/6] perf ftrace: Support omitting execution duration " Aaron Tomlin
@ 2026-10-18 2:51 ` sashiko-bot
0 siblings, 0 replies; 13+ messages in thread
From: sashiko-bot @ 2026-10-18 2:51 UTC (permalink / raw)
To: Aaron Tomlin; +Cc: linux-perf-users
Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
- [Medium] perf ftrace: missing latency-format column handling in find_func_column()
- [Medium] perf ftrace: IRQ markers misparsed as functions in find_func_column()
--
commit 00347b16046d3c2aa23a7cbbe87ca2f63a2e9477
Author: Aaron Tomlin <atomlin@atomlin.com>
perf ftrace: Support omitting execution duration in function graph tracer
This commit adds the noduration sub-option under --graph-opts to omit
function execution duration in the function_graph tracer by setting the
kernel's funcgraph-duration option to 0. It also updates trace line
parsing for inline function synthesis and rejects using noduration with
perf ftrace profile.
> diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
> index 6e20a61860a9e..d83dee5a050dd 100644
> --- a/tools/perf/tests/ftrace.c
> +++ b/tools/perf/tests/ftrace.c
[ ... ]
> @@ -422,20 +461,36 @@ static int test_column_layouts(void)
> {
> static const struct {
> const char *desc;
> - bool verbose;
> + bool noduration;
> + bool verbose;
> const char *in;
> const char *expected;
> } cases[] = {
> {
> - "default layout", false,
> + "default layout", false, false,
> " 7) | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
> " 7) | wake_up_new_task() {\n",
> },
> {
> - "verbose layout", true,
> + "noduration layout", true, false,
> + " 7) housekeeping_any_cpu(); /* <-ring_buffer_unlock_commit+0x1a0/0x1f0 */",
> + " 7) housekeeping_any_cpu();\n",
> + },
> + {
> + "verbose layout", false, true,
> " 1234.567890 | 7) bash-1234 | | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
> " 1234.567890 | 7) bash-1234 | | wake_up_new_task() {\n",
> },
> + {
> + "verbose noduration layout", true, true,
> + " 1234.567890 | 7) bash-1234 | wake_up_new_task() { /* <-kernel_clone+0x160/0x510 */",
> + " 1234.567890 | 7) bash-1234 | wake_up_new_task() {\n",
> + },
[ ... ]
> diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
> index 020551a4d7e00..66c19116ee64e 100644
> --- a/tools/perf/util/ftrace.c
> +++ b/tools/perf/util/ftrace.c
> @@ -388,6 +388,9 @@ static void make_blank_prefix(const struct perf_ftrace *ftrace, const char *line
> memcpy(buf, line, len);
> buf[len] = '\0';
>
> + if (ftrace->graph_noduration)
> + return;
> +
> if (*pipe_char == '|') {
> if (ftrace->graph_verbose) {
> /* In verbose mode, duration starts after the comm-pid '|' */
> @@ -426,10 +429,12 @@ static void make_blank_prefix(const struct perf_ftrace *ftrace, const char *line
> * [ abstime |] CPU) [ comm-pid |] [ duration |] func
> *
> * The abstime and comm-pid columns are only present with --graph-opts
> - * verbose.
> + * verbose, whereas the duration column, and thus its '|' delimiter, is
> + * omitted with --graph-opts noduration.
> *
> * Return: A pointer to the last character of the context columns (the final
> - * '|' delimiter). Returns NULL if @line is not a trace record.
> + * '|' delimiter, or the CPU ')' when no delimiter follows it). Returns NULL
> + * if @line is not a trace record (e.g. a header or a context switch marker).
> */
> static const char *find_func_column(const struct perf_ftrace *ftrace,
> const char *line, int *cpu)
> @@ -470,9 +475,14 @@ static const char *find_func_column(const struct perf_ftrace *ftrace,
> }
> }
>
> - p = strchr(p + 1, '|');
> - if (!p)
> + if (!ftrace->graph_noduration) {
> + p = strchr(p + 1, '|');
> + if (!p)
> + return NULL;
[Severity: Medium]
When perf ftrace trace is invoked with --inline and --graph-opts
verbose,noduration (or --graph-opts verbose), set_tracing_funcgraph_verbose()
in tools/perf/builtin-ftrace.c enables latency-format in addition to
funcgraph-proc and funcgraph-abstime:
tools/perf/builtin-ftrace.c:set_tracing_funcgraph_verbose() {
if (!ftrace->graph_verbose)
return 0;
if (write_tracing_option_file("funcgraph-proc", "1") < 0)
return -1;
if (write_tracing_option_file("funcgraph-abstime", "1") < 0)
return -1;
if (write_tracing_option_file("latency-format", "1") < 0)
return -1;
return 0;
}
In the kernel's function_graph tracer, TRACE_ITER(LATENCY_FMT) causes
print_graph_prologue() -> print_graph_lat_fmt() in
kernel/trace/trace_functions_graph.c to emit an additional pipe-delimited
column (such as " .... | ") between the comm-pid column and the duration
(or function) column:
kernel/trace/trace_functions_graph.c:print_graph_prologue() {
/* Proc */
if (flags & TRACE_GRAPH_PRINT_PROC) {
print_graph_proc(s, ent->pid);
trace_seq_puts(s, " | ");
}
/* Latency format */
if (tr->trace_flags & TRACE_ITER(LATENCY_FMT))
print_graph_lat_fmt(s, ent);
}
When both ftrace->graph_verbose and ftrace->graph_noduration are enabled,
find_func_column() skips the strchr(p + 1, '|') step and stops at the '|'
delimiter immediately after comm-pid instead of advancing past the
latency-format '|' delimiter. As a result, ftrace_process_fgraph_line()
treats the latency flags as the function column, which breaks function
entry parsing, exit brace detection, indentation, and blank_prefix
truncation in make_blank_prefix().
Should find_func_column(), make_blank_prefix(), and the test cases in
test_inline_processing() and test_column_layouts() account for the
latency-format column when ftrace->graph_verbose is set?
> + } else if (strstr(p, " => ")) {
> + /* Context switch marker: " CPU) comm-pid => comm-pid" */
> return NULL;
> + }
[Severity: Medium]
What happens to IRQ boundary markers ("==========>" and "<==========") when
perf ftrace trace is run with --inline and --graph-opts noduration?
In kernel/trace/trace_functions_graph.c, print_graph_irq() wraps the IRQ
arrows with print_graph_duration():
kernel/trace/trace_functions_graph.c:print_graph_irq() {
/* No overhead */
print_graph_duration(tr, 0, s, flags | FLAGS_FILL_START);
if (type == TRACE_GRAPH_ENT)
trace_seq_puts(s, "==========>");
else
trace_seq_puts(s, "<==========");
print_graph_duration(tr, 0, s, flags | FLAGS_FILL_END);
trace_seq_putc(s, '\n');
}
When funcgraph-duration is disabled, print_graph_duration() returns early
without printing a trailing '|', so the kernel emits lines like
" 0) ==========>" and " 0) <==========".
In find_func_column(), the noduration path filters out " => " context
switch markers, but returns a pointer to ')' for IRQ markers. When an
interrupt occurs while cs->inlined_depth > 0, ftrace_process_fgraph_line()
treats "==========>" as a function call and shifts it by extra_indent.
Should find_func_column() also return NULL for these IRQ markers when
ftrace->graph_noduration is enabled?
>
> *cpu = (int)val;
> return p;
[ ... ]
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261018023753.584873-1-atomlin@atomlin.com?part=5
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH perf-tools-next v8 6/6] perf ftrace: Filter extraneous functions in function graph tracer
2026-10-18 2:37 [PATCH perf-tools-next v8 0/6] perf ftrace: Support inlined functions and display enhancements in function graph tracer Aaron Tomlin
` (4 preceding siblings ...)
2026-10-18 2:37 ` [PATCH perf-tools-next v8 5/6] perf ftrace: Support omitting execution duration " Aaron Tomlin
@ 2026-10-18 2:37 ` Aaron Tomlin
2026-10-18 2:55 ` sashiko-bot
5 siblings, 1 reply; 13+ messages in thread
From: Aaron Tomlin @ 2026-10-18 2:37 UTC (permalink / raw)
To: peterz, mingo, acme, namhyung
Cc: mark.rutland, alexander.shishkin, jolsa, irogers, adrian.hunter,
atomlin, linux-perf-users, linux-kernel
When tracing with the function graph tracer using a targeted filter
(-G or --graph-funcs), users occasionally observe unrelated functions in
the trace output:
1. Top-level functions (e.g. most notably
finish_task_switch.isra.0()) appearing before or alongside the
requested target function. This occurs because clearing and
rewriting set_graph_function creates a transient window where
remote CPUs still observe an empty filter (tracing all
functions) before RCU synchronisation completes, coupled with
perf ftrace sleeping in poll() on trace_pipe right as tracing
starts.
2. Detached/orphan calls inside the target callgraph
(e.g. housekeeping_any_cpu() and arch_irq_work_raise())
appearing as child frames under the target function. These
routines are invoked by rb_wakeups() to notify perf's poll() on
trace_pipe whenever an event is committed, and execute while the
target function is on the CPU.
Resolve these issues deterministically in userspace:
1. Target callgraph state tracking
Track active target callgraph state per-CPU in
tools/perf/util/ftrace.c. Any entry not matching the requested
-G targets is suppressed along with its nested children until
its scope exits. When a target function is matched (at root or
nested depths), record its root depth; for leaf target calls,
immediately reset the filter state upon completion to ensure
subsequent sibling functions remain suppressed. Support glob
wildcard matching (-G 'prefix_*' and compiler clone suffixes).
2. Caller provenance and orphan suppression
Within an active target callgraph, validate that child entries
descend from an active frame on the target stack using their
return addresses (CONFIG_FUNCTION_GRAPH_RETADDR). Any detached
calls originating outside the target hierarchy (such as
ring-buffer wakeups or asynchronous interrupts) are suppressed
along with their subtrees (cs->orphan_depth).
Exits within an orphan subtree are suppressed without querying
the target stack, preventing common helpers (such as
_raw_spin_unlock) from colliding with or prematurely unspooling
active target frames. Dropped-orphan exit recovery is safely
permitted only when the named exit explicitly matches the target
root function itself.
3. Dynamic call stack allocation
Maintain the synthetic call stack as a dynamically reallocated
array, growing on demand via realloc(). This supports arbitrarily
deep call trees without fixed-size truncation, ensures unknown
callers are unambiguously identified as orphans, and guarantees
that standard filter teardown executes cleanly regardless of
stack depth upon target function entry.
4. Automated verification
Add comprehensive regression tests in tools/perf/tests/ftrace.c
(test_graph_target_filter()) validating top-level suppression,
detached orphan suppression, glob matching, deep stack
traversal, dropped-orphan recovery, and orphan exit collision
prevention.
For example:
# tracer: function_graph
#
# CPU FUNCTION CALLS
# | | | | |
3) activate_task() {
3) enqueue_task() {
3) psi_enqueue() { /* (inline) */
3) arch_static_branch_jump() { /* (inline) */
3) enqueue_task_fair() {
3) enqueue_hierarchy() { /* (inline) */
3) update_curr();
3) enqueue_entity() {
3) account_entity_enqueue() { /* (inline) */
3) account_llc_enqueue();
3) } /* account_entity_enqueue (inline) */
3) }
3) } /* enqueue_hierarchy (inline) */
3) place_entity() {
3) avg_vruntime();
3) }
3) __enqueue_entity();
3) dl_server_start();
3) }
3) } /* arch_static_branch_jump (inline) */
3) } /* psi_enqueue (inline) */
3) sched_info_enqueue() { /* (inline) */
3) psi_task_change() {
3) psi_write_begin() { /* (inline) */
3) psi_flags_change();
3) } /* psi_write_begin (inline) */
3) psi_group_change() {
3) record_times();
3) }
3) }
3) } /* sched_info_enqueue (inline) */
3) }
3) }
Signed-off-by: Aaron Tomlin <atomlin@atomlin.com>
---
tools/perf/tests/ftrace.c | 937 ++++++++++++++++++++++++++++++++++++++
tools/perf/util/ftrace.c | 398 ++++++++++++++--
2 files changed, 1294 insertions(+), 41 deletions(-)
diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
index d83dee5a050d..4c70951fd0c0 100644
--- a/tools/perf/tests/ftrace.c
+++ b/tools/perf/tests/ftrace.c
@@ -581,6 +581,939 @@ static int test_context_switch_handling(void)
return TEST_OK;
}
+static int test_graph_target_filter(void)
+{
+ static const char line_idle[] = " 7) 3.000 us | sched_idle_set_state();";
+ static const char line_sw_entry[] = " 6) | finish_task_switch.isra.0() {";
+ static const char line_sw_child[] = " 6) 1.000 us | _raw_spin_unlock();";
+ static const char line_sw_exit[] = " 6) 2.000 us | }";
+ static const char line_tgt_entry[] = " 2) | activate_task() {";
+ static const char line_tgt_child[] = " 2) 1.500 us | enqueue_task();";
+ static const char line_tgt_exit[] = " 2) 2.500 us | }";
+ struct perf_ftrace ftrace;
+ struct filter_entry *entry;
+ struct filter_entry *pos, *tmp;
+ char *out_buf = NULL;
+ size_t out_len = 0;
+ FILE *out;
+ const char *vmlinux;
+ int ret = TEST_OK;
+ int i;
+
+ memset(&ftrace, 0, sizeof(ftrace));
+ INIT_LIST_HEAD(&ftrace.graph_funcs);
+
+ entry = malloc(sizeof(*entry) + strlen("activate_task") + 1);
+ if (!entry)
+ return TEST_FAIL;
+ strcpy(entry->name, "activate_task");
+ list_add_tail(&entry->list, &ftrace.graph_funcs);
+
+ /* 1. Test without --inline */
+ ftrace.use_inline = false;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_idle, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_exit, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+ if (strstr(out_buf, "sched_idle_set_state")) {
+ pr_debug("FAILED: sched_idle_set_state not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "finish_task_switch")) {
+ pr_debug("FAILED: finish_task_switch not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "_raw_spin_unlock")) {
+ pr_debug("FAILED: _raw_spin_unlock not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: activate_task entry missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: enqueue_task child missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 2. Test with --inline */
+ ftrace.use_inline = true;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_idle, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_exit, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ pr_debug("Target filter output with --inline:\n%s\n", out_buf);
+
+ if (strstr(out_buf, "finish_task_switch")) {
+ pr_debug("FAILED: finish_task_switch not suppressed with --inline\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: activate_task entry missing with --inline\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 3. Test suppressing detached orphan calls within target graph */
+ ftrace.use_inline = true;
+ ftrace.filter_retaddr = 1;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_tgt_entry, out);
+ /* Leaf orphan call from ring_buffer_unlock_commit */
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) | housekeeping_any_cpu(); /* <-ring_buffer_unlock_commit+0x1a0/0x1f0 */",
+ out);
+ /* Non-leaf orphan call from irq_work_queue_on */
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) | arch_irq_work_raise() { /* <-irq_work_queue_on+0x81/0xa0 */",
+ out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) | apic_wait_icr_idle() { /* (inline) */",
+ out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) | x2apic_send_IPI_self();",
+ out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) | } /* apic_wait_icr_idle (inline) */",
+ out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) 1.000 us | }",
+ out);
+ /* Genuine child call from activate_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) | enqueue_task() { /* <-activate_task+0x1d1/0x3e0 */",
+ out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 2) 1.500 us | }",
+ out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ pr_debug("Orphan call filter output:\n%s\n", out_buf);
+
+ if (strstr(out_buf, "housekeeping_any_cpu")) {
+ pr_debug("FAILED: housekeeping_any_cpu orphan not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "arch_irq_work_raise")) {
+ pr_debug("FAILED: arch_irq_work_raise orphan not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "x2apic_send_IPI_self")) {
+ pr_debug("FAILED: x2apic_send_IPI_self not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task() {")) {
+ pr_debug("FAILED: genuine enqueue_task missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 4. Test targeted leaf call at depth > 0 resetting filter */
+ ftrace.use_inline = false;
+ list_for_each_entry_safe(pos, tmp, &ftrace.graph_funcs, list) {
+ list_del(&pos->list);
+ free(pos);
+ }
+ entry = malloc(sizeof(*entry) + strlen("enqueue_task") + 1);
+ if (!entry) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ strcpy(entry->name, "enqueue_task");
+ list_add_tail(&entry->list, &ftrace.graph_funcs);
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_sw_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "finish_task_switch")) {
+ pr_debug("FAILED: finish_task_switch not suppressed\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: targeted leaf enqueue_task missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "_raw_spin_unlock")) {
+ pr_debug("FAILED: sibling _raw_spin_unlock not suppressed after leaf target\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 5. Test targeted leaf call at depth > 0 with inlines resetting filter */
+ ftrace.use_inline = true;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_sw_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "finish_task_switch")) {
+ pr_debug("FAILED: finish_task_switch not suppressed with inlines\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: targeted leaf enqueue_task missing with inlines\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "_raw_spin_unlock")) {
+ pr_debug("FAILED: sibling _raw_spin_unlock leaked with inlines\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ vmlinux = vmlinux_path__find();
+
+ /* 6. Test targeted leaf call with synthesized DWARF inlines closing and resetting filter */
+ if (vmlinux) {
+ symbol_conf.vmlinux_name = vmlinux;
+ symbol_conf.ignore_vmlinux_buildid = true;
+ if (perf_ftrace__setup_inlines(&ftrace) == 0 && ftrace.machine) {
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ perf_ftrace__cleanup_inlines(&ftrace);
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | wake_up_new_task() {", out);
+ /* Targeted leaf with caller having inline activate_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.500 us | enqueue_task(); /* <-wake_up_new_task+0x1d1/0x3e0 */",
+ out);
+ /* Sibling call after targeted leaf */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | check_preempt_curr(); /* <-wake_up_new_task+0x0/0x3e0 */",
+ out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* wake_up_new_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+ perf_ftrace__cleanup_inlines(&ftrace);
+
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: targeted leaf enqueue_task missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "} /* activate_task (inline) */")) {
+ pr_debug("FAILED: inline activate_task not closed after leaf target\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "check_preempt_curr")) {
+ pr_debug("FAILED: sibling check_preempt_curr leaked after leaf target\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+ }
+ }
+
+ /* 7. Test recovery when an orphan exit event is dropped by the ring buffer */
+ list_for_each_entry_safe(pos, tmp, &ftrace.graph_funcs, list) {
+ list_del(&pos->list);
+ free(pos);
+ }
+ entry = malloc(sizeof(*entry) + strlen("activate_task") + 1);
+ if (!entry) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ strcpy(entry->name, "activate_task");
+ list_add_tail(&entry->list, &ftrace.graph_funcs);
+
+ /* 7a. Method 1: Target function exits directly after a dropped orphan exit */
+ ftrace.use_inline = false;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ /* Orphan entry: housekeeping_any_cpu called from outside activate_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | housekeeping_any_cpu() { /* <-rb_wakeups+0x10 */",
+ out);
+ /* Exit of housekeeping_any_cpu is DROPPED by ring buffer (never arrives) */
+ /* Target function exits with named closing brace */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: target activate_task entry missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "housekeeping_any_cpu")) {
+ pr_debug("FAILED: orphan housekeeping_any_cpu leaked into output\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "} /* activate_task */")) {
+ pr_debug("FAILED: target activate_task exit swallowed after orphan exit dropped\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 7b. Method 2: Target function calls legitimate child after a dropped orphan exit */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ /* Orphan entry: housekeeping_any_cpu called from outside activate_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | housekeeping_any_cpu() { /* <-rb_wakeups+0x10 */",
+ out);
+ /* Exit of housekeeping_any_cpu is DROPPED by ring buffer */
+ /* Legitimate child call from activate_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.200 us | enqueue_task(); /* <-activate_task+0x1d1/0x3e0 */",
+ out);
+ /* Target function exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "housekeeping_any_cpu")) {
+ pr_debug("FAILED: orphan housekeeping_any_cpu leaked into output\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: legitimate child enqueue_task swallowed after orphan exit dropped\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "} /* activate_task */")) {
+ pr_debug("FAILED: target activate_task exit missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /*
+ * 8. Test target_root_depth synchronisation when target root enters
+ * after dropped sibling exit.
+ */
+ /* 8a. Without --inline */
+ ftrace.use_inline = false;
+ ftrace.filter_retaddr = 1;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ /* Caller at depth 0 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | wake_up_new_task() {", out);
+ /* Extraneous sibling at depth 1 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | stale_sibling() {", out);
+ /* Exit of stale_sibling is DROPPED by ring buffer */
+ /* Target function enters with caller wake_up_new_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() { /* <-wake_up_new_task+0x10 */", out);
+ /* First child of target enters and exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | enqueue_task() { /* <-activate_task+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | }", out);
+ /* Second child of target enters and exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.500 us | check_preempt_curr(); /* <-activate_task+0x30 */", out);
+ /* Target function exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "stale_sibling")) {
+ pr_debug("FAILED: stale_sibling leaked into output (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: activate_task entry missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task() {")) {
+ pr_debug("FAILED: first child enqueue_task missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "check_preempt_curr();")) {
+ pr_debug("FAILED: second child check_preempt_curr missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "} /* activate_task */")) {
+ pr_debug("FAILED: target activate_task exit missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 8b. With --inline */
+ ftrace.use_inline = true;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ /* Caller at depth 0 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | wake_up_new_task() {", out);
+ /* Extraneous sibling at depth 1 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | stale_sibling() {", out);
+ /* Exit of stale_sibling is DROPPED by ring buffer */
+ /* Target function enters with caller wake_up_new_task */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() { /* <-wake_up_new_task+0x10 */", out);
+ /* First child of target enters and exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | enqueue_task() { /* <-activate_task+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | }", out);
+ /* Second child of target enters and exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.500 us | check_preempt_curr(); /* <-activate_task+0x30 */", out);
+ /* Target function exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* activate_task */", out);
+ /* Sibling call after target function */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | after_target(); /* <-wake_up_new_task+0x40 */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "stale_sibling")) {
+ pr_debug("FAILED: stale_sibling leaked into output (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: activate_task entry missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task() {")) {
+ pr_debug("FAILED: first child enqueue_task missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "check_preempt_curr();")) {
+ pr_debug("FAILED: second child check_preempt_curr missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "} /* activate_task */")) {
+ pr_debug("FAILED: target activate_task exit missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "after_target")) {
+ pr_debug("FAILED: after_target leaked into output (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /*
+ * 9. Test deep legitimate subtrees when stack depth exceeds
+ * FTRACE_MAX_STACK (overflow_depth > 0).
+ */
+ /* 9a. Without --inline */
+ ftrace.use_inline = false;
+ ftrace.filter_retaddr = 1;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ for (i = 1; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | sub_func() { /* <-activate_task+0x10 */", out);
+ }
+ /* Push frames beyond FTRACE_MAX_STACK (256) */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | overflow_parent() { /* <-sub_func+0x10 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | overflow_child() { /* <-overflow_parent+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | overflow_leaf(); /* <-overflow_child+0x30 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.200 us | }", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.400 us | }", out);
+ for (i = 1; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | }", out);
+ }
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (!strstr(out_buf, "overflow_parent() {")) {
+ pr_debug("FAILED: overflow_parent entry missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "overflow_child() {")) {
+ pr_debug("FAILED: overflow_child entry missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "overflow_leaf();")) {
+ pr_debug("FAILED: overflow_leaf missing (!inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 9b. With --inline */
+ ftrace.use_inline = true;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ for (i = 1; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | sub_func() { /* <-activate_task+0x10 */", out);
+ }
+ /* Push frames beyond FTRACE_MAX_STACK (256) */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | overflow_parent() { /* <-sub_func+0x10 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | overflow_child() { /* <-overflow_parent+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | overflow_leaf(); /* <-overflow_child+0x30 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.200 us | }", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.400 us | }", out);
+ for (i = 1; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | }", out);
+ }
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.500 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (!strstr(out_buf, "overflow_parent() {")) {
+ pr_debug("FAILED: overflow_parent entry missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "overflow_child() {")) {
+ pr_debug("FAILED: overflow_child entry missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "overflow_leaf();")) {
+ pr_debug("FAILED: overflow_leaf missing (--inline)\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 9. Test glob wildcard filter (-G 'activate_*') */
+ list_for_each_entry_safe(pos, tmp, &ftrace.graph_funcs, list) {
+ list_del(&pos->list);
+ free(pos);
+ }
+ entry = malloc(sizeof(*entry) + strlen("activate_*") + 1);
+ if (!entry) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ strcpy(entry->name, "activate_*");
+ list_add_tail(&entry->list, &ftrace.graph_funcs);
+
+ ftrace.use_inline = false;
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_idle, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_exit, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "finish_task_switch")) {
+ pr_debug("FAILED: finish_task_switch leaked with wildcard\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: wildcard target activate_* missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: wildcard child enqueue_task missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 10. Test wildcard filter matching compiler clone with suffix (-G '*task_switch') */
+ list_for_each_entry_safe(pos, tmp, &ftrace.graph_funcs, list) {
+ list_del(&pos->list);
+ free(pos);
+ }
+ entry = malloc(sizeof(*entry) + strlen("*task_switch") + 1);
+ if (!entry) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ strcpy(entry->name, "*task_switch");
+ list_add_tail(&entry->list, &ftrace.graph_funcs);
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace, line_idle, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_child, out);
+ ftrace_process_fgraph_line(&ftrace, line_sw_exit, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_entry, out);
+ ftrace_process_fgraph_line(&ftrace, line_tgt_exit, out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (!strstr(out_buf, "finish_task_switch.isra.0() {")) {
+ pr_debug("FAILED: wildcard target *task_switch did not match clone\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "_raw_spin_unlock();")) {
+ pr_debug("FAILED: child _raw_spin_unlock missing under clone target\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "activate_task")) {
+ pr_debug("FAILED: unrelated activate_task leaked under *task_switch\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 11. Test that orphan exits sharing name with target active frame do not collide (Issue 1) */
+ list_for_each_entry_safe(pos, tmp, &ftrace.graph_funcs, list) {
+ list_del(&pos->list);
+ free(pos);
+ }
+ entry = malloc(sizeof(*entry) + strlen("activate_task") + 1);
+ if (!entry) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ strcpy(entry->name, "activate_task");
+ list_add_tail(&entry->list, &ftrace.graph_funcs);
+
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | _raw_spin_unlock() { /* <-activate_task+0x10 */", out);
+ /* Detached interrupt enters */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | smp_apic_timer_interrupt() { /* <-irq_work+0x10 */", out);
+ /* Interrupt invokes _raw_spin_unlock and exits with named brace */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | _raw_spin_unlock() { /* <-smp_apic_timer_interrupt+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | } /* _raw_spin_unlock */", out);
+ /* Interrupt exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.000 us | }", out);
+ /* Target's _raw_spin_unlock exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 3.000 us | } /* _raw_spin_unlock */", out);
+ /* Target continues and exits */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 4.000 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "smp_apic_timer_interrupt")) {
+ pr_debug("FAILED: orphan smp_apic_timer_interrupt leaked into output\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: activate_task entry missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "} /* activate_task */")) {
+ pr_debug("FAILED: activate_task exit missing or prematurely unspooled\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 12. Test target function entering at deep stack (depth >= 256) and exiting without leaking sibling (Issue 2) */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ /* Push 256 non-target outer frames */
+ for (i = 0; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | outer_func() {", out);
+ }
+ /* Target function enters at depth 256 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | enqueue_task(); /* <-activate_task+0x10 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.000 us | } /* activate_task */", out);
+ /* Unrelated sibling enters at depth 256 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | unrelated_sibling() {", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | unrelated_child();", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.000 us | }", out);
+ /* Unspool outer frames */
+ for (i = 0; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | }", out);
+ }
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (!strstr(out_buf, "activate_task() {")) {
+ pr_debug("FAILED: deep activate_task entry missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "enqueue_task();")) {
+ pr_debug("FAILED: deep enqueue_task child missing\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (strstr(out_buf, "unrelated_sibling")) {
+ pr_debug("FAILED: unrelated_sibling leaked after deep target exit\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ free(out_buf);
+ out_buf = NULL;
+
+ /* 13. Test orphan suppression during deep stacks (Issue 3) */
+ out = open_memstream(&out_buf, &out_len);
+ if (!out) {
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | activate_task() {", out);
+ for (i = 1; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | sub_func() { /* <-activate_task+0x10 */", out);
+ }
+ /* Target call at depth 256 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | deep_child() { /* <-sub_func+0x10 */", out);
+ /* Detached interrupt fires at depth 257 */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) | smp_apic_timer_interrupt() { /* <-irq_work+0x10 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | irq_sub(); /* <-smp_apic_timer_interrupt+0x20 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.000 us | }", out);
+ /* Target continues */
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | deep_leaf(); /* <-deep_child+0x10 */", out);
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 2.000 us | }", out);
+ for (i = 1; i < 256; i++) {
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 1.000 us | }", out);
+ }
+ ftrace_process_fgraph_line(&ftrace,
+ " 0) 3.000 us | } /* activate_task */", out);
+
+ fclose(out);
+ ftrace_cpu_states__clear();
+
+ if (strstr(out_buf, "smp_apic_timer_interrupt") || strstr(out_buf, "irq_sub")) {
+ pr_debug("FAILED: interrupt leaked during deep stack\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ if (!strstr(out_buf, "deep_leaf();")) {
+ pr_debug("FAILED: legitimate deep_leaf missing during deep stack\n");
+ ret = TEST_FAIL;
+ goto out_free_filter;
+ }
+ free(out_buf);
+ out_buf = NULL;
+
+out_free_filter:
+ if (out_buf)
+ free(out_buf);
+ list_for_each_entry_safe(pos, tmp, &ftrace.graph_funcs, list) {
+ list_del(&pos->list);
+ free(pos);
+ }
+ return ret;
+}
+
static int test__ftrace(struct test_suite *test __maybe_unused, int subtest __maybe_unused)
{
int ret;
@@ -605,6 +1538,10 @@ static int test__ftrace(struct test_suite *test __maybe_unused, int subtest __ma
if (ret != TEST_OK)
return ret;
+ ret = test_graph_target_filter();
+ if (ret != TEST_OK)
+ return ret;
+
ret = test_inline_processing();
if (ret != TEST_OK)
return ret;
diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
index 66c19116ee64..26fc5b83b7b4 100644
--- a/tools/perf/util/ftrace.c
+++ b/tools/perf/util/ftrace.c
@@ -22,6 +22,7 @@
#include "srcline.h"
#include "symbol.h"
#include "util/ftrace.h"
+#include "util/string2.h"
#define FTRACE_MAX_CPUS 4096
#define FTRACE_MAX_STACK 256
@@ -38,10 +39,13 @@ struct ftrace_stack_frame {
struct ftrace_task_state {
struct hlist_node node;
pid_t pid;
- struct ftrace_stack_frame stack[FTRACE_MAX_STACK];
+ struct ftrace_stack_frame *stack;
+ int max_stack;
int depth;
int inlined_depth;
- int overflow_depth;
+ bool in_target;
+ int target_root_depth;
+ int orphan_depth;
};
#define ftrace_cpu_state ftrace_task_state
@@ -66,6 +70,7 @@ void ftrace_cpu_states__clear(void)
hlist_for_each_entry_safe(ts, tmp, &ftrace_task_hash[i], node) {
for (j = 0; j < ts->depth; j++)
free(ts->stack[j].name);
+ free(ts->stack);
hlist_del_init(&ts->node);
free(ts);
}
@@ -96,11 +101,35 @@ static struct ftrace_task_state *get_task_state(pid_t pid)
if (!ts)
return NULL;
+ ts->stack = calloc(FTRACE_MAX_STACK, sizeof(*ts->stack));
+ if (!ts->stack) {
+ free(ts);
+ return NULL;
+ }
+ ts->max_stack = FTRACE_MAX_STACK;
+
ts->pid = pid;
hlist_add_head(&ts->node, head);
return ts;
}
+static int ftrace_task_state__grow_stack(struct ftrace_task_state *cs)
+{
+ int new_max = cs->max_stack ? cs->max_stack * 2 : FTRACE_MAX_STACK;
+ struct ftrace_stack_frame *new_stack;
+
+ new_stack = realloc(cs->stack, new_max * sizeof(*new_stack));
+ if (!new_stack) {
+ pr_err("Not enough memory to resize ftrace stack\n");
+ return -ENOMEM;
+ }
+
+ memset(new_stack + cs->max_stack, 0, (new_max - cs->max_stack) * sizeof(*new_stack));
+ cs->stack = new_stack;
+ cs->max_stack = new_max;
+ return 0;
+}
+
/**
* get_cpu_state - Get or allocate active task stack state for a given CPU
* @cpu: The CPU number
@@ -520,6 +549,64 @@ static int print_line(struct perf_ftrace *ftrace, const char *line, FILE *out)
return 0;
}
+static inline bool ftrace_has_graph_funcs(const struct perf_ftrace *ftrace)
+{
+ return ftrace->graph_funcs.next != NULL && !list_empty(&ftrace->graph_funcs);
+}
+
+/**
+ * ftrace_is_graph_func - Check if a function matches the graph_funcs target filter
+ * @ftrace: Perf ftrace context
+ * @func_name: The function name to test
+ *
+ * Return: true if ftrace->graph_funcs is empty or @func_name matches a
+ * target filter (including glob and compiler clone matches), false otherwise.
+ */
+static bool ftrace_is_graph_func(const struct perf_ftrace *ftrace, const char *func_name)
+{
+ struct filter_entry *pos;
+ char base_name[128];
+ const char *dot;
+
+ if (!ftrace_has_graph_funcs(ftrace))
+ return true;
+
+ /* Extract base symbol name by stripping compiler clone suffixes (.isra.*, .part.*, etc.) */
+ dot = strchr(func_name, '.');
+ if (dot) {
+ size_t base_len = dot - func_name;
+
+ if (base_len < sizeof(base_name)) {
+ memcpy(base_name, func_name, base_len);
+ base_name[base_len] = '\0';
+ } else {
+ dot = NULL;
+ }
+ }
+
+ list_for_each_entry(pos, &ftrace->graph_funcs, list) {
+ /* Fast path: non-glob patterns */
+ if (!strisglob(pos->name)) {
+ size_t len = strlen(pos->name);
+
+ if (strcmp(pos->name, func_name) == 0)
+ return true;
+ if (!strncmp(pos->name, func_name, len) &&
+ (func_name[len] == '\0' || func_name[len] == '.'))
+ return true;
+ continue;
+ }
+
+ /* Glob pattern matching */
+ if (strglobmatch(func_name, pos->name))
+ return true;
+ if (dot && strglobmatch(base_name, pos->name))
+ return true;
+ }
+
+ return false;
+}
+
/**
* ftrace_process_fgraph_line - Process and output a function graph trace line
* @ftrace: Perf ftrace context
@@ -556,8 +643,10 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
bool has_open_brace;
bool has_semicolon;
bool has_retaddr = false;
+ bool has_filter = ftrace_has_graph_funcs(ftrace);
+ bool is_target_root = false;
- if (!ftrace->use_inline)
+ if (!has_filter && !ftrace->use_inline)
return print_line(ftrace, line, out);
pipe_char = find_func_column(ftrace, line, &cpu);
@@ -586,28 +675,49 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
extract_exit_func_name(func_start, exit_name, sizeof(exit_name));
- /* Find matching frame on stack */
+ if (cs->orphan_depth > 0) {
+ /*
+ * If an orphan exit was dropped by the kernel ring buffer,
+ * and the target root function itself exits with its named
+ * closing brace, the orphan has definitively ended. Clear
+ * stale orphan_depth and proceed with matching the target root.
+ * Otherwise, all exits (bare or callee names) belong to the orphan.
+ */
+ if (has_filter && cs->in_target && exit_name[0] != '\0' &&
+ cs->target_root_depth < cs->depth &&
+ cs->stack[cs->target_root_depth].name &&
+ strcmp(cs->stack[cs->target_root_depth].name, exit_name) == 0) {
+ cs->orphan_depth = 0;
+ } else {
+ cs->orphan_depth--;
+ return 0;
+ }
+ }
+
+ /* Find matching frame on stack if exit is named */
match_idx = -1;
- for (i = cs->depth - 1; i >= 0; i--) {
- if (!cs->stack[i].inlined) {
- if (exit_name[0] == '\0' ||
- (cs->stack[i].name &&
- strcmp(cs->stack[i].name, exit_name) == 0)) {
+ if (exit_name[0] != '\0') {
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined && cs->stack[i].name &&
+ strcmp(cs->stack[i].name, exit_name) == 0) {
match_idx = i;
break;
}
}
}
- if (match_idx >= 0) {
- cs->overflow_depth = 0;
- } else if (cs->overflow_depth > 0) {
- cs->overflow_depth--;
- extra_indent = cs->inlined_depth * 2;
- fprintf(out, "%.*s%*s%s\n",
- (int)(pipe_char - line + 1), line,
- leading_spaces + extra_indent, "", func_start);
- return 0;
+ /* If bare brace ('}'), match the topmost non-inlined frame */
+ if (match_idx < 0) {
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined) {
+ if (exit_name[0] == '\0' ||
+ (cs->stack[i].name &&
+ strcmp(cs->stack[i].name, exit_name) == 0)) {
+ match_idx = i;
+ break;
+ }
+ }
+ }
}
if (match_idx >= 0) {
@@ -620,8 +730,9 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
if (cs->inlined_depth > 0)
cs->inlined_depth--;
- fprintf(out, "%s%*s} /* %s (inline) */\n",
- blank_prefix, exit_indent, "", top->name);
+ if (!has_filter || cs->in_target)
+ fprintf(out, "%s%*s} /* %s (inline) */\n",
+ blank_prefix, exit_indent, "", top->name);
}
free(top->name);
cs->depth--;
@@ -639,9 +750,36 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
extra_indent = 0;
}
- fprintf(out, "%.*s%*s%s\n",
- (int)(pipe_char - line + 1), line,
- leading_spaces + extra_indent, "", func_start);
+ if (!has_filter || cs->in_target) {
+ if (!ftrace->use_inline) {
+ print_line(ftrace, line, out);
+ } else {
+ fprintf(out, "%.*s%*s%s\n",
+ (int)(pipe_char - line + 1), line,
+ leading_spaces + extra_indent, "", func_start);
+ }
+ }
+
+ if (has_filter && cs->in_target) {
+ while (cs->depth > cs->target_root_depth &&
+ cs->stack[cs->depth - 1].inlined) {
+ struct ftrace_stack_frame *top = &cs->stack[cs->depth - 1];
+ int exit_indent = top->indent;
+
+ if (cs->inlined_depth > 0)
+ cs->inlined_depth--;
+ fprintf(out, "%s%*s} /* %s (inline) */\n",
+ blank_prefix, exit_indent, "", top->name);
+ free(top->name);
+ cs->depth--;
+ }
+
+ if (cs->depth <= cs->target_root_depth) {
+ cs->in_target = false;
+ cs->target_root_depth = 0;
+ cs->orphan_depth = 0;
+ }
+ }
return 0;
}
@@ -649,6 +787,143 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
has_open_brace = (strchr(func_start, '{') != NULL);
has_semicolon = (strchr(func_start, ';') != NULL);
+ if (cs->orphan_depth > 0) {
+ /*
+ * If an orphan exit was dropped by the kernel ring buffer,
+ * check whether this entry descends from an active target
+ * frame on the stack. If so, the orphan has completed; clear
+ * orphan_depth and resume processing.
+ */
+ if (has_filter && cs->in_target && cs->depth > cs->target_root_depth) {
+ if (ftrace_parse_retaddr(func_start, caller_sym,
+ sizeof(caller_sym), &caller_offset)) {
+ caller_idx = -1;
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined && cs->stack[i].name &&
+ strcmp(cs->stack[i].name, caller_sym) == 0) {
+ caller_idx = i;
+ break;
+ }
+ }
+ if (caller_idx >= cs->target_root_depth)
+ cs->orphan_depth = 0;
+ }
+ }
+
+ if (cs->orphan_depth > 0) {
+ if (has_open_brace && !has_semicolon)
+ cs->orphan_depth++;
+ return 0;
+ }
+ }
+
+ if (has_filter) {
+ if (cs->depth == 0 || !cs->in_target) {
+ char entry_name[128];
+
+ extract_entry_func_name(func_start, entry_name, sizeof(entry_name));
+ if (ftrace_is_graph_func(ftrace, entry_name)) {
+ cs->in_target = true;
+ cs->target_root_depth = cs->depth;
+ cs->orphan_depth = 0;
+ is_target_root = true;
+ } else if (cs->depth == 0) {
+ cs->in_target = false;
+ cs->orphan_depth = 0;
+ }
+ }
+
+ if (!cs->in_target) {
+ /* Non-matching call tree; track depth if entering scope, suppress output */
+ if (has_open_brace && !has_semicolon) {
+ char entry_name[128];
+
+ extract_entry_func_name(func_start, entry_name, sizeof(entry_name));
+ if (cs->depth >= cs->max_stack &&
+ ftrace_task_state__grow_stack(cs) < 0)
+ return -ENOMEM;
+
+ cs->stack[cs->depth].name = strdup(entry_name);
+ if (!cs->stack[cs->depth].name) {
+ pr_err("Not enough memory\n");
+ return -ENOMEM;
+ }
+ cs->stack[cs->depth].inlined = false;
+ cs->stack[cs->depth].indent = leading_spaces;
+ cs->depth++;
+ }
+ return 0;
+ }
+ }
+
+ if (!ftrace->use_inline) {
+ if (is_target_root) {
+ if (ftrace_parse_retaddr(func_start, caller_sym,
+ sizeof(caller_sym), &caller_offset)) {
+ caller_idx = -1;
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined && cs->stack[i].name &&
+ strcmp(cs->stack[i].name, caller_sym) == 0) {
+ caller_idx = i;
+ break;
+ }
+ }
+ if (caller_idx >= 0) {
+ while (cs->depth > caller_idx + 1) {
+ free(cs->stack[cs->depth - 1].name);
+ cs->depth--;
+ }
+ }
+ }
+ cs->target_root_depth = cs->depth;
+ }
+
+ if (has_filter && cs->in_target && cs->depth > cs->target_root_depth) {
+ if (ftrace_parse_retaddr(func_start, caller_sym,
+ sizeof(caller_sym), &caller_offset)) {
+ caller_idx = -1;
+ for (i = cs->depth - 1; i >= 0; i--) {
+ if (!cs->stack[i].inlined && cs->stack[i].name &&
+ strcmp(cs->stack[i].name, caller_sym) == 0) {
+ caller_idx = i;
+ break;
+ }
+ }
+ if (caller_idx < cs->target_root_depth) {
+ if (has_open_brace && !has_semicolon)
+ cs->orphan_depth = 1;
+ return 0;
+ }
+ }
+ }
+
+ if (has_open_brace && !has_semicolon) {
+ char entry_name[128];
+
+ extract_entry_func_name(func_start, entry_name, sizeof(entry_name));
+ if (cs->depth >= cs->max_stack &&
+ ftrace_task_state__grow_stack(cs) < 0)
+ return -ENOMEM;
+
+ cs->stack[cs->depth].name = strdup(entry_name);
+ if (!cs->stack[cs->depth].name) {
+ pr_err("Not enough memory\n");
+ return -ENOMEM;
+ }
+ cs->stack[cs->depth].inlined = false;
+ cs->stack[cs->depth].indent = leading_spaces;
+ cs->depth++;
+ }
+ print_line(ftrace, line, out);
+ if (has_filter && cs->in_target && is_target_root &&
+ has_semicolon && !has_open_brace) {
+ cs->in_target = false;
+ cs->target_root_depth = 0;
+ cs->orphan_depth = 0;
+ }
+ return 0;
+ }
+
if (ftrace_parse_retaddr(func_start, caller_sym, sizeof(caller_sym), &caller_offset)) {
has_retaddr = true;
num_inlines = ftrace_resolve_inlines(ftrace->machine, caller_sym,
@@ -675,6 +950,21 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
}
}
+ /*
+ * When inside a target graph filter, child functions must descend
+ * from an active frame on the target stack. Suppress detached
+ * calls originating outside the target hierarchy.
+ */
+ if (has_filter && cs->in_target && cs->depth > cs->target_root_depth) {
+ if (has_retaddr && caller_idx < cs->target_root_depth) {
+ if (has_open_brace && !has_semicolon)
+ cs->orphan_depth = 1;
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ return 0;
+ }
+ }
+
active_inlines = 0;
if (caller_idx >= 0) {
for (i = caller_idx + 1; i < cs->depth; i++) {
@@ -697,9 +987,6 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
}
/* Close stale callee frames and inactive inlined frames above caller */
- if (caller_idx >= 0)
- cs->overflow_depth = 0;
-
while (caller_idx >= 0 && cs->depth > caller_idx + 1 + common) {
struct ftrace_stack_frame *top = &cs->stack[cs->depth - 1];
@@ -708,13 +995,17 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
if (cs->inlined_depth > 0)
cs->inlined_depth--;
- fprintf(out, "%s%*s} /* %s (inline) */\n",
- blank_prefix, exit_indent, "", top->name);
+ if (!has_filter || (cs->in_target && !is_target_root))
+ fprintf(out, "%s%*s} /* %s (inline) */\n",
+ blank_prefix, exit_indent, "", top->name);
}
free(top->name);
cs->depth--;
}
+ if (is_target_root)
+ cs->target_root_depth = cs->depth;
+
/* Open new inlined frames */
for (i = common; i < num_inlines; i++) {
int entry_indent = leading_spaces + cs->inlined_depth * 2;
@@ -725,7 +1016,8 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
free((void *)inlined_names[j]);
return -ENOMEM;
}
- if (cs->depth >= FTRACE_MAX_STACK)
+ if (cs->depth >= cs->max_stack &&
+ ftrace_task_state__grow_stack(cs) < 0)
break;
fprintf(out, "%s%*s%s() { /* (inline) */\n",
@@ -772,20 +1064,44 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
char entry_name[128];
extract_entry_func_name(func_start, entry_name, sizeof(entry_name));
- if (cs->depth < FTRACE_MAX_STACK) {
- cs->stack[cs->depth].name = strdup(entry_name);
- if (!cs->stack[cs->depth].name) {
- pr_err("Not enough memory\n");
- for (i = 0; i < num_inlines; i++)
- free((void *)inlined_names[i]);
- return -ENOMEM;
+ if (cs->depth >= cs->max_stack &&
+ ftrace_task_state__grow_stack(cs) < 0) {
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ return -ENOMEM;
+ }
+
+ cs->stack[cs->depth].name = strdup(entry_name);
+ if (!cs->stack[cs->depth].name) {
+ pr_err("Not enough memory\n");
+ for (i = 0; i < num_inlines; i++)
+ free((void *)inlined_names[i]);
+ return -ENOMEM;
+ }
+ cs->stack[cs->depth].inlined = false;
+ cs->stack[cs->depth].indent = leading_spaces + extra_indent;
+ cs->depth++;
+ }
+
+ if (has_filter && cs->in_target && is_target_root &&
+ has_semicolon && !has_open_brace) {
+ while (cs->depth > cs->target_root_depth) {
+ struct ftrace_stack_frame *top = &cs->stack[cs->depth - 1];
+
+ if (top->inlined) {
+ int exit_indent = top->indent;
+
+ if (cs->inlined_depth > 0)
+ cs->inlined_depth--;
+ fprintf(out, "%s%*s} /* %s (inline) */\n",
+ blank_prefix, exit_indent, "", top->name);
}
- cs->stack[cs->depth].inlined = false;
- cs->stack[cs->depth].indent = leading_spaces + extra_indent;
- cs->depth++;
- } else {
- cs->overflow_depth++;
+ free(top->name);
+ cs->depth--;
}
+ cs->in_target = false;
+ cs->target_root_depth = 0;
+ cs->orphan_depth = 0;
}
for (i = 0; i < num_inlines; i++)
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* Re: [PATCH perf-tools-next v8 6/6] perf ftrace: Filter extraneous functions in function graph tracer
2026-10-18 2:37 ` [PATCH perf-tools-next v8 6/6] perf ftrace: Filter extraneous functions " Aaron Tomlin
@ 2026-10-18 2:55 ` sashiko-bot
0 siblings, 0 replies; 13+ messages in thread
From: sashiko-bot @ 2026-10-18 2:55 UTC (permalink / raw)
To: Aaron Tomlin; +Cc: linux-perf-users
Thank you for your contribution! Sashiko AI review found 6 potential issue(s) to consider:
- [Medium] perf ftrace: funcgraph-retaddr not enabled for -G in cmd_ftrace()
- [Medium] perf ftrace: premature inline frame unwind in ftrace_process_fgraph_line()
- [Medium] perf ftrace: inlines skew target_root_depth in ftrace_process_fgraph_line()
- [Medium] perf ftrace: non-target exit lines leaked in ftrace_process_fgraph_line()
- [Medium] perf ftrace: ftrace_process_fgraph_line() treats ancestor callers as orphans
- [Medium] perf tests ftrace: hardcoded vmlinux offset in test_graph_target_filter()
--
commit eca2ef148505a6e43726e07f9ad0e28ae131e40c
Author: Aaron Tomlin <atomlin@atomlin.com>
perf ftrace: Filter extraneous functions in function graph tracer
This commit adds userspace callgraph tracking and return-address provenance
filtering to perf ftrace so that top-level functions outside -G /
--graph-funcs targets and detached orphan subtrees inside target callgraphs
are suppressed. It also switches the per-task call stack to grow
dynamically and adds regression tests in tools/perf/tests/ftrace.c.
> diff --git a/tools/perf/tests/ftrace.c b/tools/perf/tests/ftrace.c
> index d83dee5a050dd..4c70951fd0c0c 100644
> --- a/tools/perf/tests/ftrace.c
> +++ b/tools/perf/tests/ftrace.c
> @@ -581,6 +581,939 @@ static int test_context_switch_handling(void)
[ ... ]
> + vmlinux = vmlinux_path__find();
> +
> + /* 6. Test targeted leaf call with synthesized DWARF inlines closing and resetting filter */
> + if (vmlinux) {
> + symbol_conf.vmlinux_name = vmlinux;
> + symbol_conf.ignore_vmlinux_buildid = true;
> + if (perf_ftrace__setup_inlines(&ftrace) == 0 && ftrace.machine) {
> + out = open_memstream(&out_buf, &out_len);
> + if (!out) {
> + perf_ftrace__cleanup_inlines(&ftrace);
> + ret = TEST_FAIL;
> + goto out_free_filter;
> + }
> +
> + ftrace_process_fgraph_line(&ftrace,
> + " 0) | wake_up_new_task() {", out);
> + /* Targeted leaf with caller having inline activate_task */
> + ftrace_process_fgraph_line(&ftrace,
> + " 0) 1.500 us | enqueue_task(); /* <-wake_up_new_task+0x1d1/0x3e0 */",
> + out);
> + /* Sibling call after targeted leaf */
> + ftrace_process_fgraph_line(&ftrace,
> + " 0) 1.000 us | check_preempt_curr(); /* <-wake_up_new_task+0x0/0x3e0 */",
> + out);
> + ftrace_process_fgraph_line(&ftrace,
> + " 0) 2.500 us | } /* wake_up_new_task */", out);
> +
> + fclose(out);
> + ftrace_cpu_states__clear();
> + perf_ftrace__cleanup_inlines(&ftrace);
> +
> + if (!strstr(out_buf, "enqueue_task();")) {
> + pr_debug("FAILED: targeted leaf enqueue_task missing\n");
> + ret = TEST_FAIL;
> + goto out_free_filter;
> + }
> + if (!strstr(out_buf, "} /* activate_task (inline) */")) {
> + pr_debug("FAILED: inline activate_task not closed after leaf target\n");
> + ret = TEST_FAIL;
> + goto out_free_filter;
> + }
[Severity: Medium]
In test_graph_target_filter(), when vmlinux_path__find() finds a host
vmlinux image, perf_ftrace__setup_inlines() loads that binary regardless of
architecture, compiler, Kconfig, or whether DWARF debug info is present.
The test then passes the hardcoded return address offset
wake_up_new_task+0x1d1/0x3e0 to ftrace_process_fgraph_line() and returns
TEST_FAIL if out_buf does not contain "} /* activate_task (inline) */".
Won't this cause perf test to fail on any system with a readable vmlinux
where wake_up_new_task+0x1d1 does not resolve via DWARF to an inlined
activate_task() frame?
[ ... ]
> diff --git a/tools/perf/util/ftrace.c b/tools/perf/util/ftrace.c
> index 66c19116ee64e..26fc5b83b7b41 100644
> --- a/tools/perf/util/ftrace.c
> +++ b/tools/perf/util/ftrace.c
[ ... ]
> @@ -586,28 +675,49 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
>
> extract_exit_func_name(func_start, exit_name, sizeof(exit_name));
>
> - /* Find matching frame on stack */
> + if (cs->orphan_depth > 0) {
> + /*
> + * If an orphan exit was dropped by the kernel ring buffer,
> + * and the target root function itself exits with its named
> + * closing brace, the orphan has definitively ended. Clear
> + * stale orphan_depth and proceed with matching the target root.
> + * Otherwise, all exits (bare or callee names) belong to the orphan.
> + */
> + if (has_filter && cs->in_target && exit_name[0] != '\0' &&
> + cs->target_root_depth < cs->depth &&
> + cs->stack[cs->target_root_depth].name &&
> + strcmp(cs->stack[cs->target_root_depth].name, exit_name) == 0) {
> + cs->orphan_depth = 0;
> + } else {
> + cs->orphan_depth--;
> + return 0;
> + }
> + }
[Severity: Medium]
In ftrace_process_fgraph_line(), this dropped-orphan exit recovery check
compares exit_name against cs->stack[cs->target_root_depth].name.
However, when a target root function enters at depth > 0 with
ftrace->use_inline enabled and its caller has inlined frames,
ftrace_process_fgraph_line() sets cs->target_root_depth before pushing the
synthesized inlined frames onto cs->stack:
if (is_target_root)
cs->target_root_depth = cs->depth;
/* Open new inlined frames */
for (i = common; i < num_inlines; i++) {
...
cs->stack[cs->depth].name = strdup(inlined_names[i]);
...
cs->stack[cs->depth].inlined = true;
cs->stack[cs->depth].indent = entry_indent;
cs->depth++;
cs->inlined_depth++;
}
The non-inlined target root frame is then pushed above those synthesized
frames, leaving cs->stack[cs->target_root_depth].name holding the outermost
inlined caller frame's name rather than the target root function's name.
If an orphan call's exit event is dropped in this state and the target root
subsequently exits with its named closing brace, won't this strcmp() fail to
match exit_name, causing the target root's exit to be swallowed as part of
the orphan?
[ ... ]
> @@ -639,9 +750,36 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
> extra_indent = 0;
> }
>
> - fprintf(out, "%.*s%*s%s\n",
> - (int)(pipe_char - line + 1), line,
> - leading_spaces + extra_indent, "", func_start);
> + if (!has_filter || cs->in_target) {
> + if (!ftrace->use_inline) {
> + print_line(ftrace, line, out);
> + } else {
> + fprintf(out, "%.*s%*s%s\n",
> + (int)(pipe_char - line + 1), line,
> + leading_spaces + extra_indent, "", func_start);
> + }
> + }
[Severity: Medium]
Suppose an outer non-target function enters at depth 0 (stored in
cs->stack[0] with its entry line suppressed) and calls a target function at
depth 1 (cs->target_root_depth = 1, cs->in_target = true).
If the target function's exit event is dropped by the ring buffer and the
outer function exits with "} /* outer_func */", match_idx is 0 (which is
less than cs->target_root_depth) and cs->depth is unwound to 0. Similarly,
if an unmatched named exit arrives while cs->in_target is true, match_idx
is -1.
Because ftrace_process_fgraph_line() prints the exit line whenever
cs->in_target is true before checking whether cs->depth <=
cs->target_root_depth, won't this leak the closing brace line for the
suppressed ancestor (or unmatched function) into the output?
> +
> + if (has_filter && cs->in_target) {
> + while (cs->depth > cs->target_root_depth &&
> + cs->stack[cs->depth - 1].inlined) {
> + struct ftrace_stack_frame *top = &cs->stack[cs->depth - 1];
> + int exit_indent = top->indent;
> +
> + if (cs->inlined_depth > 0)
> + cs->inlined_depth--;
> + fprintf(out, "%s%*s} /* %s (inline) */\n",
> + blank_prefix, exit_indent, "", top->name);
> + free(top->name);
> + cs->depth--;
> + }
> +
> + if (cs->depth <= cs->target_root_depth) {
> + cs->in_target = false;
> + cs->target_root_depth = 0;
> + cs->orphan_depth = 0;
> + }
> + }
> return 0;
> }
[Severity: Medium]
When both -G (has_filter) and --inline (ftrace->use_inline) are active and a
non-leaf child function is called from within a synthesized inlined scope
inside the target callgraph, the stack holds the caller's .inlined = true
frame(s) followed by the child callee's .inlined = false frame.
When that child callee exits, its real frame is popped above, leaving the
caller's still-active .inlined frame(s) at cs->stack[cs->depth - 1] with
cs->depth > cs->target_root_depth.
Doesn't this while loop run unconditionally on every exit event inside the
target callgraph, prematurely popping and printing "} /* ... (inline) */"
for the caller's active inlined frames whenever a child function returns
(and then re-opening them on the next sibling call)?
[ ... ]
> + if (!ftrace->use_inline) {
> + if (is_target_root) {
> + if (ftrace_parse_retaddr(func_start, caller_sym,
> + sizeof(caller_sym), &caller_offset)) {
> + caller_idx = -1;
> + for (i = cs->depth - 1; i >= 0; i--) {
> + if (!cs->stack[i].inlined && cs->stack[i].name &&
> + strcmp(cs->stack[i].name, caller_sym) == 0) {
> + caller_idx = i;
> + break;
> + }
> + }
> + if (caller_idx >= 0) {
> + while (cs->depth > caller_idx + 1) {
> + free(cs->stack[cs->depth - 1].name);
> + cs->depth--;
> + }
> + }
> + }
> + cs->target_root_depth = cs->depth;
> + }
> +
> + if (has_filter && cs->in_target && cs->depth > cs->target_root_depth) {
> + if (ftrace_parse_retaddr(func_start, caller_sym,
> + sizeof(caller_sym), &caller_offset)) {
> + caller_idx = -1;
> + for (i = cs->depth - 1; i >= 0; i--) {
> + if (!cs->stack[i].inlined && cs->stack[i].name &&
> + strcmp(cs->stack[i].name, caller_sym) == 0) {
> + caller_idx = i;
> + break;
> + }
> + }
> + if (caller_idx < cs->target_root_depth) {
> + if (has_open_brace && !has_semicolon)
> + cs->orphan_depth = 1;
> + return 0;
> + }
> + }
> + }
[Severity: Medium]
In the !ftrace->use_inline path (as well as the dropped-orphan recovery
above), ftrace_process_fgraph_line() relies on ftrace_parse_retaddr()
finding return address comments in the trace stream to suppress detached
orphan calls and synchronize target_root_depth.
However, in tools/perf/builtin-ftrace.c, set_tracing_funcgraph_retaddr()
only enables funcgraph-retaddr when ftrace->graph_retaddr or
ftrace->use_inline is set:
static int set_tracing_funcgraph_retaddr(struct perf_ftrace *ftrace)
{
if (ftrace->graph_retaddr || ftrace->use_inline) {
if (write_tracing_option_file("funcgraph-retaddr", "1") < 0) {
and cmd_ftrace() only sets ftrace.filter_retaddr = true when
ftrace.use_inline && !ftrace.graph_retaddr:
if (!ftrace.filter_retaddr_set) {
if (ftrace.use_inline && !ftrace.graph_retaddr)
ftrace.filter_retaddr = true;
else
ftrace.filter_retaddr = false;
}
When a user runs perf ftrace -G <func> without --inline or --graph-opts
retaddr, funcgraph-retaddr is disabled in tracefs, so ftrace_parse_retaddr()
always returns false. Should builtin-ftrace.c also enable funcgraph-retaddr
(and filter_retaddr when !ftrace.graph_retaddr) when ftrace.graph_funcs is
non-empty?
[ ... ]
> @@ -675,6 +950,21 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
> }
> }
>
> + /*
> + * When inside a target graph filter, child functions must descend
> + * from an active frame on the target stack. Suppress detached
> + * calls originating outside the target hierarchy.
> + */
> + if (has_filter && cs->in_target && cs->depth > cs->target_root_depth) {
> + if (has_retaddr && caller_idx < cs->target_root_depth) {
> + if (has_open_brace && !has_semicolon)
> + cs->orphan_depth = 1;
> + for (i = 0; i < num_inlines; i++)
> + free((void *)inlined_names[i]);
> + return 0;
> + }
> + }
[Severity: Medium]
What happens if a target function enters at a nested depth
(cs->target_root_depth > 0, beneath a suppressed outer ancestor on
cs->stack), and the target root's exit event is dropped by the ring buffer?
When the outer ancestor subsequently makes another function call,
caller_idx resolves to that ancestor's frame on cs->stack (0 <= caller_idx <
cs->target_root_depth), proving the target root has already returned.
Because both this check in the ftrace->use_inline path and the corresponding
check in the !ftrace->use_inline path test caller_idx <
cs->target_root_depth instead of caller_idx < 0, they treat the ancestor's
subsequent call as a detached orphan and return early without unwinding the
stale target frames or resetting cs->in_target.
Should 0 <= caller_idx < cs->target_root_depth unwind the completed target
frames down to caller_idx + 1 and reset cs->in_target before evaluating the
new entry?
[ ... ]
> @@ -708,13 +995,17 @@ int ftrace_process_fgraph_line(struct perf_ftrace *ftrace, const char *line, FIL
>
> if (cs->inlined_depth > 0)
> cs->inlined_depth--;
> - fprintf(out, "%s%*s} /* %s (inline) */\n",
> - blank_prefix, exit_indent, "", top->name);
> + if (!has_filter || (cs->in_target && !is_target_root))
> + fprintf(out, "%s%*s} /* %s (inline) */\n",
> + blank_prefix, exit_indent, "", top->name);
> }
> free(top->name);
> cs->depth--;
> }
[ ... ]
--
Sashiko AI review · https://sashiko.dev/#/patchset/20261018023753.584873-1-atomlin@atomlin.com?part=6
^ permalink raw reply [flat|nested] 13+ messages in thread