* [PATCH v4 1/2] perf debug: Add function symbols to dump_stack
@ 2025-06-23 16:19 Ian Rogers
2025-06-23 16:19 ` [PATCH v4 2/2] perf srcline: Lower verbosity on addr2line debug messages Ian Rogers
2025-06-24 17:45 ` [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Namhyung Kim
0 siblings, 2 replies; 4+ messages in thread
From: Ian Rogers @ 2025-06-23 16:19 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Arnaldo Carvalho de Melo,
Namhyung Kim, Mark Rutland, Alexander Shishkin, Jiri Olsa,
Ian Rogers, Adrian Hunter, Kan Liang, James Clark, Howard Chu,
Yicong Yang, linux-perf-users, linux-kernel
Symbolize stack traces by creating a live machine. Add this
functionality to dump_stack and switch dump_stack users to use
it. Switch TUI to use it. Add stack traces to the child test function
which can be useful to diagnose blocked code.
Example output:
```
$ perf test -vv PERF_RECORD_
...
7: PERF_RECORD_* events & perf_sample fields:
7: PERF_RECORD_* events & perf_sample fields : Running (1 active)
^C
Signal (2) while running tests.
Terminating tests with the same signal
Internal test harness failure. Completing any started tests:
: 7: PERF_RECORD_* events & perf_sample fields:
---- unexpected signal (2) ----
#0 0x5628ad5570a3 in child_test_sig_handler builtin-test.c:0
#1 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
#2 0x7f561de99687 in __internal_syscall_cancel cancellation.c:64
#3 0x7f561dee5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72
#4 0x7f561def1393 in __nanosleep nanosleep.c:26
#5 0x7f561df02d68 in __sleep sleep.c:55
#6 0x5628ad5679ab in test__PERF_RECORD perf-record.c:0
#7 0x5628ad556fb0 in run_test_child builtin-test.c:0
#8 0x5628ad4f318d in start_command run-command.c:127
#9 0x5628ad557ef3 in __cmd_test builtin-test.c:0
#10 0x5628ad5585bf in cmd_test ??:0
#11 0x5628ad4e5bb0 in run_builtin perf.c:0
#12 0x5628ad4e5ecb in handle_internal_command perf.c:0
#13 0x5628ad461383 in main ??:0
#14 0x7f561de33ca8 in __libc_start_call_main libc_start_call_main.h:74
#15 0x7f561de33d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
#16 0x5628ad4619d1 in _start ??:0
---- unexpected signal (2) ----
#0 0x5628ad5570a3 in child_test_sig_handler builtin-test.c:0
#1 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
#2 0x7f561dea3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45
#3 0x7f561de49fd9 in __GI___sigprocmask sigprocmask.c:26
#4 0x7f561df2601b in __longjmp_chk longjmp.c:36
#5 0x5628ad5570c0 in print_test_result.isra.0 builtin-test.c:0
#6 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
#7 0x7f561de99687 in __internal_syscall_cancel cancellation.c:64
#8 0x7f561dee5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72
#9 0x7f561def1393 in __nanosleep nanosleep.c:26
#10 0x7f561df02d68 in __sleep sleep.c:55
#11 0x5628ad5679ab in test__PERF_RECORD perf-record.c:0
#12 0x5628ad556fb0 in run_test_child builtin-test.c:0
#13 0x5628ad4f318d in start_command run-command.c:127
#14 0x5628ad557ef3 in __cmd_test builtin-test.c:0
#15 0x5628ad5585bf in cmd_test ??:0
#16 0x5628ad4e5bb0 in run_builtin perf.c:0
#17 0x5628ad4e5ecb in handle_internal_command perf.c:0
#18 0x5628ad461383 in main ??:0
#19 0x7f561de33ca8 in __libc_start_call_main libc_start_call_main.h:74
#20 0x7f561de33d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
#21 0x5628ad4619d1 in _start ??:0
7: PERF_RECORD_* events & perf_sample fields : Skip (permissions)
```
Signed-off-by: Ian Rogers <irogers@google.com>
---
v4: Additional addr_location__exit (Namhyung)
v3: Rebase
v2: Fix NO_BACKTRACE=1 build (Arnaldo)
---
tools/perf/tests/builtin-test.c | 15 +++++++-
tools/perf/ui/tui/setup.c | 2 +-
tools/perf/util/debug.c | 66 +++++++++++++++++++++++++++------
tools/perf/util/debug.h | 1 +
4 files changed, 71 insertions(+), 13 deletions(-)
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 45d3d8b3317a..80375ca39a37 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -6,6 +6,9 @@
*/
#include <fcntl.h>
#include <errno.h>
+#ifdef HAVE_BACKTRACE_SUPPORT
+#include <execinfo.h>
+#endif
#include <poll.h>
#include <unistd.h>
#include <setjmp.h>
@@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;
static void child_test_sig_handler(int sig)
{
+#ifdef HAVE_BACKTRACE_SUPPORT
+ void *stackdump[32];
+ size_t stackdump_size;
+#endif
+
+ fprintf(stderr, "\n---- unexpected signal (%d) ----\n", sig);
+#ifdef HAVE_BACKTRACE_SUPPORT
+ stackdump_size = backtrace(stackdump, ARRAY_SIZE(stackdump));
+ __dump_stack(stderr, stackdump, stackdump_size);
+#endif
siglongjmp(run_test_jmp_buf, sig);
}
@@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process)
err = sigsetjmp(run_test_jmp_buf, 1);
if (err) {
- fprintf(stderr, "\n---- unexpected signal (%d) ----\n", err);
+ /* Received signal. */
err = err > 0 ? -err : -1;
goto err_out;
}
diff --git a/tools/perf/ui/tui/setup.c b/tools/perf/ui/tui/setup.c
index 16c6eff4d241..022534eed68c 100644
--- a/tools/perf/ui/tui/setup.c
+++ b/tools/perf/ui/tui/setup.c
@@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)
printf("-------- backtrace --------\n");
size = backtrace(stackdump, ARRAY_SIZE(stackdump));
- backtrace_symbols_fd(stackdump, size, STDOUT_FILENO);
+ __dump_stack(stdout, stackdump, size);
exit(0);
}
diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c
index f9ef7d045c92..efc9d2c6448d 100644
--- a/tools/perf/util/debug.c
+++ b/tools/perf/util/debug.c
@@ -14,11 +14,18 @@
#ifdef HAVE_BACKTRACE_SUPPORT
#include <execinfo.h>
#endif
+#include "addr_location.h"
#include "color.h"
-#include "event.h"
#include "debug.h"
+#include "event.h"
+#include "machine.h"
+#include "map.h"
#include "print_binary.h"
+#include "srcline.h"
+#include "symbol.h"
+#include "synthetic-events.h"
#include "target.h"
+#include "thread.h"
#include "trace-event.h"
#include "ui/helpline.h"
#include "ui/ui.h"
@@ -298,21 +305,58 @@ void perf_debug_setup(void)
libapi_set_print(pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper);
}
+void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size)
+{
+ /* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */
+ pid_t pid = getpid();
+ struct machine *machine = machine__new_live(/*kernel_maps=*/false, pid);
+ struct thread *thread = NULL;
+
+ if (machine)
+ thread = machine__find_thread(machine, pid, pid);
+
+ if (!machine || !thread) {
+ /*
+ * Backtrace functions are async signal safe. Fall back on them
+ * if machine/thread creation fails.
+ */
+ backtrace_symbols_fd(stackdump, stackdump_size, fileno(file));
+ machine__delete(machine);
+ return;
+ }
+
+ for (size_t i = 0; i < stackdump_size; i++) {
+ struct addr_location al;
+ u64 addr = (u64)stackdump[i];
+
+ addr_location__init(&al);
+ if (!thread__find_map(thread, PERF_RECORD_MISC_USER, addr, &al)) {
+ addr_location__exit(&al);
+ continue;
+ }
+
+ al.sym = map__find_symbol(al.map, al.addr);
+ if (al.sym)
+ fprintf(file, " #%zd %p in %s ", i, stackdump[i], al.sym->name);
+ else
+ fprintf(file, " #%zd %p ", i, stackdump[i]);
+
+ map__fprintf_srcline(al.map, al.addr, "", file);
+ fprintf(file, "\n");
+ addr_location__exit(&al);
+ }
+ thread__put(thread);
+ machine__delete(machine);
+}
+
/* Obtain a backtrace and print it to stdout. */
#ifdef HAVE_BACKTRACE_SUPPORT
void dump_stack(void)
{
- void *array[16];
- size_t size = backtrace(array, ARRAY_SIZE(array));
- char **strings = backtrace_symbols(array, size);
- size_t i;
-
- printf("Obtained %zd stack frames.\n", size);
-
- for (i = 0; i < size; i++)
- printf("%s\n", strings[i]);
+ void *stackdump[32];
+ size_t size = backtrace(stackdump, ARRAY_SIZE(stackdump));
- free(strings);
+ __dump_stack(stdout, stackdump, size);
}
#else
void dump_stack(void) {}
diff --git a/tools/perf/util/debug.h b/tools/perf/util/debug.h
index a4026d1fd6a3..6b737e195ce1 100644
--- a/tools/perf/util/debug.h
+++ b/tools/perf/util/debug.h
@@ -85,6 +85,7 @@ void debug_set_display_time(bool set);
void perf_debug_setup(void);
int perf_quiet_option(void);
+void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size);
void dump_stack(void);
void sighandler_dump_stack(int sig);
--
2.50.0.rc2.761.g2dc52ea45b-goog
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH v4 2/2] perf srcline: Lower verbosity on addr2line debug messages
2025-06-23 16:19 [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Ian Rogers
@ 2025-06-23 16:19 ` Ian Rogers
2025-06-24 17:45 ` [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Namhyung Kim
1 sibling, 0 replies; 4+ messages in thread
From: Ian Rogers @ 2025-06-23 16:19 UTC (permalink / raw)
To: Peter Zijlstra, Ingo Molnar, Arnaldo Carvalho de Melo,
Namhyung Kim, Mark Rutland, Alexander Shishkin, Jiri Olsa,
Ian Rogers, Adrian Hunter, Kan Liang, James Clark, Howard Chu,
Yicong Yang, linux-perf-users, linux-kernel
Lower non-error debug messages to verbose 3 or larger.
Signed-off-by: Ian Rogers <irogers@google.com>
---
tools/perf/util/srcline.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/tools/perf/util/srcline.c b/tools/perf/util/srcline.c
index f32d0d4f4bc9..3e3449e35dd4 100644
--- a/tools/perf/util/srcline.c
+++ b/tools/perf/util/srcline.c
@@ -524,12 +524,12 @@ static enum a2l_style addr2line_configure(struct child_process *a2l, const char
style = LLVM;
cached = true;
lines = 1;
- pr_debug("Detected LLVM addr2line style\n");
+ pr_debug3("Detected LLVM addr2line style\n");
} else if (ch == '0') {
style = GNU_BINUTILS;
cached = true;
lines = 3;
- pr_debug("Detected binutils addr2line style\n");
+ pr_debug3("Detected binutils addr2line style\n");
} else {
if (!symbol_conf.disable_add2line_warn) {
char *output = NULL;
@@ -595,7 +595,7 @@ static int read_addr2line_record(struct io *io,
if (io__getline(io, &line, &line_len) < 0 || !line_len)
goto error;
- pr_debug("%s %s: addr2line read address for sentinel: %s", __func__, dso_name, line);
+ pr_debug3("%s %s: addr2line read address for sentinel: %s", __func__, dso_name, line);
if (style == LLVM && line_len == 2 && line[0] == ',') {
/* Found the llvm-addr2line sentinel character. */
zfree(&line);
@@ -641,7 +641,7 @@ static int read_addr2line_record(struct io *io,
if (first && (io__getline(io, &line, &line_len) < 0 || !line_len))
goto error;
- pr_debug("%s %s: addr2line read line: %s", __func__, dso_name, line);
+ pr_debug3("%s %s: addr2line read line: %s", __func__, dso_name, line);
if (function != NULL)
*function = strdup(strim(line));
@@ -652,7 +652,7 @@ static int read_addr2line_record(struct io *io,
if (io__getline(io, &line, &line_len) < 0 || !line_len)
goto error;
- pr_debug("%s %s: addr2line filename:number : %s", __func__, dso_name, line);
+ pr_debug3("%s %s: addr2line filename:number : %s", __func__, dso_name, line);
if (filename_split(line, line_nr == NULL ? &dummy_line_nr : line_nr) == 0 &&
style == GNU_BINUTILS) {
ret = 0;
--
2.50.0.rc2.761.g2dc52ea45b-goog
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH v4 1/2] perf debug: Add function symbols to dump_stack
2025-06-23 16:19 [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Ian Rogers
2025-06-23 16:19 ` [PATCH v4 2/2] perf srcline: Lower verbosity on addr2line debug messages Ian Rogers
@ 2025-06-24 17:45 ` Namhyung Kim
2025-06-24 20:39 ` Ian Rogers
1 sibling, 1 reply; 4+ messages in thread
From: Namhyung Kim @ 2025-06-24 17:45 UTC (permalink / raw)
To: Ian Rogers
Cc: Peter Zijlstra, Ingo Molnar, Arnaldo Carvalho de Melo,
Mark Rutland, Alexander Shishkin, Jiri Olsa, Adrian Hunter,
Kan Liang, James Clark, Howard Chu, Yicong Yang, linux-perf-users,
linux-kernel
Hi Ian,
On Mon, Jun 23, 2025 at 09:19:28AM -0700, Ian Rogers wrote:
> Symbolize stack traces by creating a live machine. Add this
> functionality to dump_stack and switch dump_stack users to use
> it. Switch TUI to use it. Add stack traces to the child test function
> which can be useful to diagnose blocked code.
>
> Example output:
> ```
> $ perf test -vv PERF_RECORD_
> ...
> 7: PERF_RECORD_* events & perf_sample fields:
> 7: PERF_RECORD_* events & perf_sample fields : Running (1 active)
> ^C
> Signal (2) while running tests.
> Terminating tests with the same signal
> Internal test harness failure. Completing any started tests:
> : 7: PERF_RECORD_* events & perf_sample fields:
>
> ---- unexpected signal (2) ----
> #0 0x5628ad5570a3 in child_test_sig_handler builtin-test.c:0
> #1 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
> #2 0x7f561de99687 in __internal_syscall_cancel cancellation.c:64
> #3 0x7f561dee5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72
> #4 0x7f561def1393 in __nanosleep nanosleep.c:26
> #5 0x7f561df02d68 in __sleep sleep.c:55
> #6 0x5628ad5679ab in test__PERF_RECORD perf-record.c:0
> #7 0x5628ad556fb0 in run_test_child builtin-test.c:0
> #8 0x5628ad4f318d in start_command run-command.c:127
> #9 0x5628ad557ef3 in __cmd_test builtin-test.c:0
> #10 0x5628ad5585bf in cmd_test ??:0
> #11 0x5628ad4e5bb0 in run_builtin perf.c:0
> #12 0x5628ad4e5ecb in handle_internal_command perf.c:0
> #13 0x5628ad461383 in main ??:0
> #14 0x7f561de33ca8 in __libc_start_call_main libc_start_call_main.h:74
> #15 0x7f561de33d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
> #16 0x5628ad4619d1 in _start ??:0
>
> ---- unexpected signal (2) ----
> #0 0x5628ad5570a3 in child_test_sig_handler builtin-test.c:0
> #1 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
> #2 0x7f561dea3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45
> #3 0x7f561de49fd9 in __GI___sigprocmask sigprocmask.c:26
> #4 0x7f561df2601b in __longjmp_chk longjmp.c:36
> #5 0x5628ad5570c0 in print_test_result.isra.0 builtin-test.c:0
> #6 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
> #7 0x7f561de99687 in __internal_syscall_cancel cancellation.c:64
> #8 0x7f561dee5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72
> #9 0x7f561def1393 in __nanosleep nanosleep.c:26
> #10 0x7f561df02d68 in __sleep sleep.c:55
> #11 0x5628ad5679ab in test__PERF_RECORD perf-record.c:0
> #12 0x5628ad556fb0 in run_test_child builtin-test.c:0
> #13 0x5628ad4f318d in start_command run-command.c:127
> #14 0x5628ad557ef3 in __cmd_test builtin-test.c:0
> #15 0x5628ad5585bf in cmd_test ??:0
> #16 0x5628ad4e5bb0 in run_builtin perf.c:0
> #17 0x5628ad4e5ecb in handle_internal_command perf.c:0
> #18 0x5628ad461383 in main ??:0
> #19 0x7f561de33ca8 in __libc_start_call_main libc_start_call_main.h:74
> #20 0x7f561de33d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
> #21 0x5628ad4619d1 in _start ??:0
> 7: PERF_RECORD_* events & perf_sample fields : Skip (permissions)
> ```
>
> Signed-off-by: Ian Rogers <irogers@google.com>
Unfortunately it has some build errors so I had to remove it from the
tree. I'll keep the patch 2/2 btw.
The NO_BACKTRACE=1 build caused a trouble like below.
CC /tmp/tmp.huqYLzBWIi/util/debug.o
util/debug.c: In function '__dump_stack':
util/debug.c:323:17: error: implicit declaration of function 'backtrace_symbols_fd' [-Wimplicit-function-declaration]
323 | backtrace_symbols_fd(stackdump, stackdump_size, fileno(file));
| ^~~~~~~~~~~~~~~~~~~~
make[6]: *** [/usr/local/google/home/namhyung/project/linux/tools/build/Makefile.build:86: /tmp/tmp.huqYLzBWIi/util/debug.o] Error 1
make[5]: *** [/usr/local/google/home/namhyung/project/linux/tools/build/Makefile.build:142: util] Error 2
make[4]: *** [Makefile.perf:798: /tmp/tmp.huqYLzBWIi/perf-util-in.o] Error 2
make[4]: *** Waiting for unfinished jobs....
make[3]: *** [Makefile.perf:290: sub-make] Error 2
make[2]: *** [Makefile:76: all] Error 2
make[1]: *** [tests/make:341: make_no_backtrace_O] Error 1
make: *** [Makefile:109: build-test] Error 2
And on 32-bit, it also shows:
CC /build/util/debug.o
util/debug.c: In function '__dump_stack':
util/debug.c:331:28: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast]
331 | u64 addr = (u64)stackdump[i];
| ^
cc1: all warnings being treated as errors
make[4]: *** [/linux/tools/build/Makefile.build:85: /build/util/debug.o] Error 1
make[4]: *** Waiting for unfinished jobs....
make[3]: *** [/linux/tools/build/Makefile.build:142: util] Error 2
make[2]: *** [Makefile.perf:798: /build/perf-util-in.o] Error 2
make[2]: *** Waiting for unfinished jobs....
make[1]: *** [Makefile.perf:290: sub-make] Error 2
make: *** [Makefile:76: all] Error 2
Thanks,
Namhyung
> ---
> v4: Additional addr_location__exit (Namhyung)
> v3: Rebase
> v2: Fix NO_BACKTRACE=1 build (Arnaldo)
> ---
> tools/perf/tests/builtin-test.c | 15 +++++++-
> tools/perf/ui/tui/setup.c | 2 +-
> tools/perf/util/debug.c | 66 +++++++++++++++++++++++++++------
> tools/perf/util/debug.h | 1 +
> 4 files changed, 71 insertions(+), 13 deletions(-)
>
> diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
> index 45d3d8b3317a..80375ca39a37 100644
> --- a/tools/perf/tests/builtin-test.c
> +++ b/tools/perf/tests/builtin-test.c
> @@ -6,6 +6,9 @@
> */
> #include <fcntl.h>
> #include <errno.h>
> +#ifdef HAVE_BACKTRACE_SUPPORT
> +#include <execinfo.h>
> +#endif
> #include <poll.h>
> #include <unistd.h>
> #include <setjmp.h>
> @@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;
>
> static void child_test_sig_handler(int sig)
> {
> +#ifdef HAVE_BACKTRACE_SUPPORT
> + void *stackdump[32];
> + size_t stackdump_size;
> +#endif
> +
> + fprintf(stderr, "\n---- unexpected signal (%d) ----\n", sig);
> +#ifdef HAVE_BACKTRACE_SUPPORT
> + stackdump_size = backtrace(stackdump, ARRAY_SIZE(stackdump));
> + __dump_stack(stderr, stackdump, stackdump_size);
> +#endif
> siglongjmp(run_test_jmp_buf, sig);
> }
>
> @@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process)
>
> err = sigsetjmp(run_test_jmp_buf, 1);
> if (err) {
> - fprintf(stderr, "\n---- unexpected signal (%d) ----\n", err);
> + /* Received signal. */
> err = err > 0 ? -err : -1;
> goto err_out;
> }
> diff --git a/tools/perf/ui/tui/setup.c b/tools/perf/ui/tui/setup.c
> index 16c6eff4d241..022534eed68c 100644
> --- a/tools/perf/ui/tui/setup.c
> +++ b/tools/perf/ui/tui/setup.c
> @@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)
>
> printf("-------- backtrace --------\n");
> size = backtrace(stackdump, ARRAY_SIZE(stackdump));
> - backtrace_symbols_fd(stackdump, size, STDOUT_FILENO);
> + __dump_stack(stdout, stackdump, size);
>
> exit(0);
> }
> diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c
> index f9ef7d045c92..efc9d2c6448d 100644
> --- a/tools/perf/util/debug.c
> +++ b/tools/perf/util/debug.c
> @@ -14,11 +14,18 @@
> #ifdef HAVE_BACKTRACE_SUPPORT
> #include <execinfo.h>
> #endif
> +#include "addr_location.h"
> #include "color.h"
> -#include "event.h"
> #include "debug.h"
> +#include "event.h"
> +#include "machine.h"
> +#include "map.h"
> #include "print_binary.h"
> +#include "srcline.h"
> +#include "symbol.h"
> +#include "synthetic-events.h"
> #include "target.h"
> +#include "thread.h"
> #include "trace-event.h"
> #include "ui/helpline.h"
> #include "ui/ui.h"
> @@ -298,21 +305,58 @@ void perf_debug_setup(void)
> libapi_set_print(pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper);
> }
>
> +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size)
> +{
> + /* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */
> + pid_t pid = getpid();
> + struct machine *machine = machine__new_live(/*kernel_maps=*/false, pid);
> + struct thread *thread = NULL;
> +
> + if (machine)
> + thread = machine__find_thread(machine, pid, pid);
> +
> + if (!machine || !thread) {
> + /*
> + * Backtrace functions are async signal safe. Fall back on them
> + * if machine/thread creation fails.
> + */
> + backtrace_symbols_fd(stackdump, stackdump_size, fileno(file));
> + machine__delete(machine);
> + return;
> + }
> +
> + for (size_t i = 0; i < stackdump_size; i++) {
> + struct addr_location al;
> + u64 addr = (u64)stackdump[i];
> +
> + addr_location__init(&al);
> + if (!thread__find_map(thread, PERF_RECORD_MISC_USER, addr, &al)) {
> + addr_location__exit(&al);
> + continue;
> + }
> +
> + al.sym = map__find_symbol(al.map, al.addr);
> + if (al.sym)
> + fprintf(file, " #%zd %p in %s ", i, stackdump[i], al.sym->name);
> + else
> + fprintf(file, " #%zd %p ", i, stackdump[i]);
> +
> + map__fprintf_srcline(al.map, al.addr, "", file);
> + fprintf(file, "\n");
> + addr_location__exit(&al);
> + }
> + thread__put(thread);
> + machine__delete(machine);
> +}
> +
> /* Obtain a backtrace and print it to stdout. */
> #ifdef HAVE_BACKTRACE_SUPPORT
> void dump_stack(void)
> {
> - void *array[16];
> - size_t size = backtrace(array, ARRAY_SIZE(array));
> - char **strings = backtrace_symbols(array, size);
> - size_t i;
> -
> - printf("Obtained %zd stack frames.\n", size);
> -
> - for (i = 0; i < size; i++)
> - printf("%s\n", strings[i]);
> + void *stackdump[32];
> + size_t size = backtrace(stackdump, ARRAY_SIZE(stackdump));
>
> - free(strings);
> + __dump_stack(stdout, stackdump, size);
> }
> #else
> void dump_stack(void) {}
> diff --git a/tools/perf/util/debug.h b/tools/perf/util/debug.h
> index a4026d1fd6a3..6b737e195ce1 100644
> --- a/tools/perf/util/debug.h
> +++ b/tools/perf/util/debug.h
> @@ -85,6 +85,7 @@ void debug_set_display_time(bool set);
> void perf_debug_setup(void);
> int perf_quiet_option(void);
>
> +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size);
> void dump_stack(void);
> void sighandler_dump_stack(int sig);
>
> --
> 2.50.0.rc2.761.g2dc52ea45b-goog
>
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH v4 1/2] perf debug: Add function symbols to dump_stack
2025-06-24 17:45 ` [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Namhyung Kim
@ 2025-06-24 20:39 ` Ian Rogers
0 siblings, 0 replies; 4+ messages in thread
From: Ian Rogers @ 2025-06-24 20:39 UTC (permalink / raw)
To: Namhyung Kim
Cc: Peter Zijlstra, Ingo Molnar, Arnaldo Carvalho de Melo,
Mark Rutland, Alexander Shishkin, Jiri Olsa, Adrian Hunter,
Kan Liang, James Clark, Howard Chu, Yicong Yang, linux-perf-users,
linux-kernel
On Tue, Jun 24, 2025 at 10:45 AM Namhyung Kim <namhyung@kernel.org> wrote:
>
> Hi Ian,
>
> On Mon, Jun 23, 2025 at 09:19:28AM -0700, Ian Rogers wrote:
> > Symbolize stack traces by creating a live machine. Add this
> > functionality to dump_stack and switch dump_stack users to use
> > it. Switch TUI to use it. Add stack traces to the child test function
> > which can be useful to diagnose blocked code.
> >
> > Example output:
> > ```
> > $ perf test -vv PERF_RECORD_
> > ...
> > 7: PERF_RECORD_* events & perf_sample fields:
> > 7: PERF_RECORD_* events & perf_sample fields : Running (1 active)
> > ^C
> > Signal (2) while running tests.
> > Terminating tests with the same signal
> > Internal test harness failure. Completing any started tests:
> > : 7: PERF_RECORD_* events & perf_sample fields:
> >
> > ---- unexpected signal (2) ----
> > #0 0x5628ad5570a3 in child_test_sig_handler builtin-test.c:0
> > #1 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
> > #2 0x7f561de99687 in __internal_syscall_cancel cancellation.c:64
> > #3 0x7f561dee5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72
> > #4 0x7f561def1393 in __nanosleep nanosleep.c:26
> > #5 0x7f561df02d68 in __sleep sleep.c:55
> > #6 0x5628ad5679ab in test__PERF_RECORD perf-record.c:0
> > #7 0x5628ad556fb0 in run_test_child builtin-test.c:0
> > #8 0x5628ad4f318d in start_command run-command.c:127
> > #9 0x5628ad557ef3 in __cmd_test builtin-test.c:0
> > #10 0x5628ad5585bf in cmd_test ??:0
> > #11 0x5628ad4e5bb0 in run_builtin perf.c:0
> > #12 0x5628ad4e5ecb in handle_internal_command perf.c:0
> > #13 0x5628ad461383 in main ??:0
> > #14 0x7f561de33ca8 in __libc_start_call_main libc_start_call_main.h:74
> > #15 0x7f561de33d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
> > #16 0x5628ad4619d1 in _start ??:0
> >
> > ---- unexpected signal (2) ----
> > #0 0x5628ad5570a3 in child_test_sig_handler builtin-test.c:0
> > #1 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
> > #2 0x7f561dea3a14 in pthread_sigmask@GLIBC_2.2.5 pthread_sigmask.c:45
> > #3 0x7f561de49fd9 in __GI___sigprocmask sigprocmask.c:26
> > #4 0x7f561df2601b in __longjmp_chk longjmp.c:36
> > #5 0x5628ad5570c0 in print_test_result.isra.0 builtin-test.c:0
> > #6 0x7f561de49df0 in __restore_rt libc_sigaction.c:0
> > #7 0x7f561de99687 in __internal_syscall_cancel cancellation.c:64
> > #8 0x7f561dee5f7a in clock_nanosleep@GLIBC_2.2.5 clock_nanosleep.c:72
> > #9 0x7f561def1393 in __nanosleep nanosleep.c:26
> > #10 0x7f561df02d68 in __sleep sleep.c:55
> > #11 0x5628ad5679ab in test__PERF_RECORD perf-record.c:0
> > #12 0x5628ad556fb0 in run_test_child builtin-test.c:0
> > #13 0x5628ad4f318d in start_command run-command.c:127
> > #14 0x5628ad557ef3 in __cmd_test builtin-test.c:0
> > #15 0x5628ad5585bf in cmd_test ??:0
> > #16 0x5628ad4e5bb0 in run_builtin perf.c:0
> > #17 0x5628ad4e5ecb in handle_internal_command perf.c:0
> > #18 0x5628ad461383 in main ??:0
> > #19 0x7f561de33ca8 in __libc_start_call_main libc_start_call_main.h:74
> > #20 0x7f561de33d65 in __libc_start_main@@GLIBC_2.34 libc-start.c:128
> > #21 0x5628ad4619d1 in _start ??:0
> > 7: PERF_RECORD_* events & perf_sample fields : Skip (permissions)
> > ```
> >
> > Signed-off-by: Ian Rogers <irogers@google.com>
>
> Unfortunately it has some build errors so I had to remove it from the
> tree. I'll keep the patch 2/2 btw.
>
> The NO_BACKTRACE=1 build caused a trouble like below.
>
> CC /tmp/tmp.huqYLzBWIi/util/debug.o
> util/debug.c: In function '__dump_stack':
> util/debug.c:323:17: error: implicit declaration of function 'backtrace_symbols_fd' [-Wimplicit-function-declaration]
> 323 | backtrace_symbols_fd(stackdump, stackdump_size, fileno(file));
> | ^~~~~~~~~~~~~~~~~~~~
> make[6]: *** [/usr/local/google/home/namhyung/project/linux/tools/build/Makefile.build:86: /tmp/tmp.huqYLzBWIi/util/debug.o] Error 1
> make[5]: *** [/usr/local/google/home/namhyung/project/linux/tools/build/Makefile.build:142: util] Error 2
> make[4]: *** [Makefile.perf:798: /tmp/tmp.huqYLzBWIi/perf-util-in.o] Error 2
> make[4]: *** Waiting for unfinished jobs....
> make[3]: *** [Makefile.perf:290: sub-make] Error 2
> make[2]: *** [Makefile:76: all] Error 2
> make[1]: *** [tests/make:341: make_no_backtrace_O] Error 1
> make: *** [Makefile:109: build-test] Error 2
>
> And on 32-bit, it also shows:
>
> CC /build/util/debug.o
> util/debug.c: In function '__dump_stack':
> util/debug.c:331:28: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast]
> 331 | u64 addr = (u64)stackdump[i];
> | ^
>
> cc1: all warnings being treated as errors
> make[4]: *** [/linux/tools/build/Makefile.build:85: /build/util/debug.o] Error 1
> make[4]: *** Waiting for unfinished jobs....
> make[3]: *** [/linux/tools/build/Makefile.build:142: util] Error 2
> make[2]: *** [Makefile.perf:798: /build/perf-util-in.o] Error 2
> make[2]: *** Waiting for unfinished jobs....
> make[1]: *** [Makefile.perf:290: sub-make] Error 2
> make: *** [Makefile:76: all] Error 2
>
> Thanks,
> Namhyung
Sorry Namhyung, I accidentally rebased the v1 rather than the v2
patch. I'll address the issues and repost as v5.
Thanks,
Ian
> > ---
> > v4: Additional addr_location__exit (Namhyung)
> > v3: Rebase
> > v2: Fix NO_BACKTRACE=1 build (Arnaldo)
> > ---
> > tools/perf/tests/builtin-test.c | 15 +++++++-
> > tools/perf/ui/tui/setup.c | 2 +-
> > tools/perf/util/debug.c | 66 +++++++++++++++++++++++++++------
> > tools/perf/util/debug.h | 1 +
> > 4 files changed, 71 insertions(+), 13 deletions(-)
> >
> > diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
> > index 45d3d8b3317a..80375ca39a37 100644
> > --- a/tools/perf/tests/builtin-test.c
> > +++ b/tools/perf/tests/builtin-test.c
> > @@ -6,6 +6,9 @@
> > */
> > #include <fcntl.h>
> > #include <errno.h>
> > +#ifdef HAVE_BACKTRACE_SUPPORT
> > +#include <execinfo.h>
> > +#endif
> > #include <poll.h>
> > #include <unistd.h>
> > #include <setjmp.h>
> > @@ -231,6 +234,16 @@ static jmp_buf run_test_jmp_buf;
> >
> > static void child_test_sig_handler(int sig)
> > {
> > +#ifdef HAVE_BACKTRACE_SUPPORT
> > + void *stackdump[32];
> > + size_t stackdump_size;
> > +#endif
> > +
> > + fprintf(stderr, "\n---- unexpected signal (%d) ----\n", sig);
> > +#ifdef HAVE_BACKTRACE_SUPPORT
> > + stackdump_size = backtrace(stackdump, ARRAY_SIZE(stackdump));
> > + __dump_stack(stderr, stackdump, stackdump_size);
> > +#endif
> > siglongjmp(run_test_jmp_buf, sig);
> > }
> >
> > @@ -244,7 +257,7 @@ static int run_test_child(struct child_process *process)
> >
> > err = sigsetjmp(run_test_jmp_buf, 1);
> > if (err) {
> > - fprintf(stderr, "\n---- unexpected signal (%d) ----\n", err);
> > + /* Received signal. */
> > err = err > 0 ? -err : -1;
> > goto err_out;
> > }
> > diff --git a/tools/perf/ui/tui/setup.c b/tools/perf/ui/tui/setup.c
> > index 16c6eff4d241..022534eed68c 100644
> > --- a/tools/perf/ui/tui/setup.c
> > +++ b/tools/perf/ui/tui/setup.c
> > @@ -108,7 +108,7 @@ static void ui__signal_backtrace(int sig)
> >
> > printf("-------- backtrace --------\n");
> > size = backtrace(stackdump, ARRAY_SIZE(stackdump));
> > - backtrace_symbols_fd(stackdump, size, STDOUT_FILENO);
> > + __dump_stack(stdout, stackdump, size);
> >
> > exit(0);
> > }
> > diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c
> > index f9ef7d045c92..efc9d2c6448d 100644
> > --- a/tools/perf/util/debug.c
> > +++ b/tools/perf/util/debug.c
> > @@ -14,11 +14,18 @@
> > #ifdef HAVE_BACKTRACE_SUPPORT
> > #include <execinfo.h>
> > #endif
> > +#include "addr_location.h"
> > #include "color.h"
> > -#include "event.h"
> > #include "debug.h"
> > +#include "event.h"
> > +#include "machine.h"
> > +#include "map.h"
> > #include "print_binary.h"
> > +#include "srcline.h"
> > +#include "symbol.h"
> > +#include "synthetic-events.h"
> > #include "target.h"
> > +#include "thread.h"
> > #include "trace-event.h"
> > #include "ui/helpline.h"
> > #include "ui/ui.h"
> > @@ -298,21 +305,58 @@ void perf_debug_setup(void)
> > libapi_set_print(pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper);
> > }
> >
> > +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size)
> > +{
> > + /* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */
> > + pid_t pid = getpid();
> > + struct machine *machine = machine__new_live(/*kernel_maps=*/false, pid);
> > + struct thread *thread = NULL;
> > +
> > + if (machine)
> > + thread = machine__find_thread(machine, pid, pid);
> > +
> > + if (!machine || !thread) {
> > + /*
> > + * Backtrace functions are async signal safe. Fall back on them
> > + * if machine/thread creation fails.
> > + */
> > + backtrace_symbols_fd(stackdump, stackdump_size, fileno(file));
> > + machine__delete(machine);
> > + return;
> > + }
> > +
> > + for (size_t i = 0; i < stackdump_size; i++) {
> > + struct addr_location al;
> > + u64 addr = (u64)stackdump[i];
> > +
> > + addr_location__init(&al);
> > + if (!thread__find_map(thread, PERF_RECORD_MISC_USER, addr, &al)) {
> > + addr_location__exit(&al);
> > + continue;
> > + }
> > +
> > + al.sym = map__find_symbol(al.map, al.addr);
> > + if (al.sym)
> > + fprintf(file, " #%zd %p in %s ", i, stackdump[i], al.sym->name);
> > + else
> > + fprintf(file, " #%zd %p ", i, stackdump[i]);
> > +
> > + map__fprintf_srcline(al.map, al.addr, "", file);
> > + fprintf(file, "\n");
> > + addr_location__exit(&al);
> > + }
> > + thread__put(thread);
> > + machine__delete(machine);
> > +}
> > +
> > /* Obtain a backtrace and print it to stdout. */
> > #ifdef HAVE_BACKTRACE_SUPPORT
> > void dump_stack(void)
> > {
> > - void *array[16];
> > - size_t size = backtrace(array, ARRAY_SIZE(array));
> > - char **strings = backtrace_symbols(array, size);
> > - size_t i;
> > -
> > - printf("Obtained %zd stack frames.\n", size);
> > -
> > - for (i = 0; i < size; i++)
> > - printf("%s\n", strings[i]);
> > + void *stackdump[32];
> > + size_t size = backtrace(stackdump, ARRAY_SIZE(stackdump));
> >
> > - free(strings);
> > + __dump_stack(stdout, stackdump, size);
> > }
> > #else
> > void dump_stack(void) {}
> > diff --git a/tools/perf/util/debug.h b/tools/perf/util/debug.h
> > index a4026d1fd6a3..6b737e195ce1 100644
> > --- a/tools/perf/util/debug.h
> > +++ b/tools/perf/util/debug.h
> > @@ -85,6 +85,7 @@ void debug_set_display_time(bool set);
> > void perf_debug_setup(void);
> > int perf_quiet_option(void);
> >
> > +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size);
> > void dump_stack(void);
> > void sighandler_dump_stack(int sig);
> >
> > --
> > 2.50.0.rc2.761.g2dc52ea45b-goog
> >
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2025-06-24 20:39 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2025-06-23 16:19 [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Ian Rogers
2025-06-23 16:19 ` [PATCH v4 2/2] perf srcline: Lower verbosity on addr2line debug messages Ian Rogers
2025-06-24 17:45 ` [PATCH v4 1/2] perf debug: Add function symbols to dump_stack Namhyung Kim
2025-06-24 20:39 ` Ian Rogers
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).