[PATCH v1 1/2] perf record: Fix teardown hang on system-wide multi-threaded sessions

[PATCH v1 1/2] perf record: Fix teardown hang on system-wide multi-threaded sessions

[Ian Rogers]

Under system-wide (-a) parallel streaming mode (--threads=cpu),
background recording threads can be inundated by a continuous
firehose of hardware samples generated by the OS. In this state,
a background thread's local hit count remains unequal to its
sample count, causing it to bypass the blocking fdarray__poll()
call entirely on each iteration of its recording loop.

Because the termination check relies on the POLLHUP event status
populated specifically by fdarray__poll(), bypassing it prevents
the background thread from ever recognizing that its control pipe
was closed by the main thread. This traps the background thread
in an infinite recording loop, hanging the main thread indefinitely
as it awaits a termination acknowledgment that never arrives.

Ensure teardown completion by adding explicit evlist__disable()
calls in the main thread's cleanup paths at out_child: and
out_child_no_flush:. Additionally, patch fdarray__filter() to
respect the fdarray_flag__nonfilterable flag, preventing it
from incorrectly setting the background thread's control pipe
file descriptor to -1 and clearing its revents mask upon
processing termination POLLHUP signals.

Fixes: f94563fac269 ("perf record: fix poll storm when monitored threads exit")
Assisted-by: Antigravity:gemini-3.5-flash
Signed-off-by: Ian Rogers <irogers@google.com>
---
 tools/lib/api/fd/array.c    | 6 ++++--
 tools/perf/builtin-record.c | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c
index ffe8272af59b..16a047f1906e 100644
--- a/tools/lib/api/fd/array.c
+++ b/tools/lib/api/fd/array.c
@@ -115,6 +115,9 @@ int fdarray__filter(struct fdarray *fda, short revents,
 		return 0;
 
 	for (fd = 0; fd < fda->nr; ++fd) {
+		if (fda->priv[fd].flags & fdarray_flag__nonfilterable)
+			continue;
+
 		if (!fda->entries[fd].events)
 			continue;
 
@@ -132,8 +135,7 @@ int fdarray__filter(struct fdarray *fda, short revents,
 			continue;
 		}
 
-		if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable))
-			++nr;
+		++nr;
 	}
 
 	return nr;
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index ebd3ed0c9b3e..d1276382b77a 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -2890,11 +2890,13 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 		record__synthesize_workload(rec, true);
 
 out_child:
+	evlist__disable(rec->evlist);
 	record__stop_threads(rec);
 	record__mmap_read_all(rec, true);
 	goto out_free_threads;
 out_child_no_flush:
 	/* mmap read already failed — retrying would just fail again */
+	evlist__disable(rec->evlist);
 	record__stop_threads(rec);
 out_free_threads:
 	record__free_thread_data(rec);
-- 
2.55.0.795.g602f6c329a-goog

[PATCH v1 2/2] perf cap: If capability is missing still perform root test

[Ian Rogers]

Refine the capability checking logic in perf_cap__capable() to in the
event the capability is missing do the fallback used_root
test. Previously this was only performed when the capability syscall
failed.

Fixes: e25ebda78e23 ("perf cap: Tidy up and improve capability testing")
Assisted-by: Antigravity:gemini-3.5-flash
Signed-off-by: Ian Rogers <irogers@google.com>
---
 tools/perf/util/cap.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c
index ac6d1d9a523d..a2baecbc3145 100644
--- a/tools/perf/util/cap.c
+++ b/tools/perf/util/cap.c
@@ -20,6 +20,7 @@ bool perf_cap__capable(int cap, bool *used_root)
 	};
 	struct __user_cap_data_struct data[MAX_LINUX_CAPABILITY_U32S] = {};
 	__u32 cap_val;
+	bool has_cap = false;
 
 	*used_root = false;
 	while (syscall(SYS_capget, &header, &data[0]) == -1) {
@@ -37,12 +38,21 @@ bool perf_cap__capable(int cap, bool *used_root)
 	if (cap >= 32) {
 		if (header.version == _LINUX_CAPABILITY_VERSION_3) {
 			cap_val = data[1].effective;
+			has_cap = (cap_val & (1U << (cap & 0x1f))) != 0;
 		} else {
 			/* Capability beyond 32 is requested but only 32 are supported. */
-			return false;
+			has_cap = false;
 		}
 	} else {
 		cap_val = data[0].effective;
+		has_cap = (cap_val & (1U << (cap & 0x1f))) != 0;
 	}
-	return (cap_val & (1 << (cap & 0x1f))) != 0;
+
+	if (has_cap) {
+		*used_root = false;
+		return true;
+	}
+
+	*used_root = true;
+	return geteuid() == 0;
 }
-- 
2.55.0.795.g602f6c329a-goog

Re: [PATCH v1 2/2] perf cap: If capability is missing still perform root test

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] The capability fallback logic overrides valid capability checks for root users and incorrectly short-circuits sequential capability checks for non-root users by improperly setting the `used_root` flag.
--

--- Patch [2]: [PATCH v1 2/2] perf cap: If capability is missing still perform root test ---
commit 8515c6747c6ffb6d27af14829f03c002334e5c67
Author: Ian Rogers <irogers@google.com>

perf cap: If capability is missing still perform root test

This commit refines the capability checking logic in perf_cap__capable() to
fallback to the used_root test if a capability is missing, rather than only
doing so when the capget syscall fails.

> diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c
> --- a/tools/perf/util/cap.c
> +++ b/tools/perf/util/cap.c

[ ... ]

> @@ -37,12 +38,21 @@ bool perf_cap__capable(int cap, bool *used_root)
>  	if (cap >= 32) {
>  		if (header.version == _LINUX_CAPABILITY_VERSION_3) {
>  			cap_val = data[1].effective;
> +			has_cap = (cap_val & (1U << (cap & 0x1f))) != 0;
>  		} else {
>  			/* Capability beyond 32 is requested but only 32 are supported. */
> -			return false;
> +			has_cap = false;
>  		}
>  	} else {
>  		cap_val = data[0].effective;
> +		has_cap = (cap_val & (1U << (cap & 0x1f))) != 0;
>  	}
> -	return (cap_val & (1 << (cap & 0x1f))) != 0;
> +
> +	if (has_cap) {
> +		*used_root = false;
> +		return true;
> +	}
> +
> +	*used_root = true;
> +	return geteuid() == 0;
>  }

[Severity: High]
Does setting *used_root = true unconditionally here when a capability is
missing break sequential capability checks for non-root users?

For example, if an unprivileged user lacks CAP_SYS_ADMIN, this will return
false but set used_root to true. In callers like perf_event_paranoid_check():

tools/perf/util/util.c:perf_event_paranoid_check() {
    ...
    if (!used_root && perf_cap__capable(CAP_PERFMON, &used_root))
    ...
}

If a previous check for CAP_SYS_ADMIN failed and poisoned the used_root
flag, won't it cause the subsequent check for CAP_PERFMON to be skipped
entirely?

This pattern appears in other locations as well, such as
check_bpf_filter_capable() where a failed CAP_BPF check might skip the
fallback for non-root users to access root-pinned BPF filters because
!used_root evaluates to false.

Additionally, could this override valid capability drops for root users
in containers? Because geteuid() == 0 evaluates to true, it seems this might
falsely claim root users still possess the capability even if it was
explicitly dropped, potentially leading to later failures instead of
graceful fallbacks.

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260710053927.1864021-1-irogers@google.com?part=2

Re: [PATCH v1 2/2] perf cap: If capability is missing still perform root test

[Namhyung Kim]

On Fri, Jul 10, 2026 at 05:47:10AM +0000, sashiko-bot@kernel.org wrote:
> Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
> - [High] The capability fallback logic overrides valid capability checks for root users and incorrectly short-circuits sequential capability checks for non-root users by improperly setting the `used_root` flag.
> --
> 
> --- Patch [2]: [PATCH v1 2/2] perf cap: If capability is missing still perform root test ---
> commit 8515c6747c6ffb6d27af14829f03c002334e5c67
> Author: Ian Rogers <irogers@google.com>
> 
> perf cap: If capability is missing still perform root test
> 
> This commit refines the capability checking logic in perf_cap__capable() to
> fallback to the used_root test if a capability is missing, rather than only
> doing so when the capget syscall fails.

I think we can process this independently.  It's not related to patch 1,
right?

> 
> > diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c
> > --- a/tools/perf/util/cap.c
> > +++ b/tools/perf/util/cap.c
> 
> [ ... ]
> 
> > @@ -37,12 +38,21 @@ bool perf_cap__capable(int cap, bool *used_root)
> >  	if (cap >= 32) {
> >  		if (header.version == _LINUX_CAPABILITY_VERSION_3) {
> >  			cap_val = data[1].effective;
> > +			has_cap = (cap_val & (1U << (cap & 0x1f))) != 0;
> >  		} else {
> >  			/* Capability beyond 32 is requested but only 32 are supported. */
> > -			return false;
> > +			has_cap = false;
> >  		}
> >  	} else {
> >  		cap_val = data[0].effective;
> > +		has_cap = (cap_val & (1U << (cap & 0x1f))) != 0;
> >  	}
> > -	return (cap_val & (1 << (cap & 0x1f))) != 0;
> > +
> > +	if (has_cap) {
> > +		*used_root = false;
> > +		return true;
> > +	}
> > +
> > +	*used_root = true;
> > +	return geteuid() == 0;
> >  }
> 
> [Severity: High]
> Does setting *used_root = true unconditionally here when a capability is
> missing break sequential capability checks for non-root users?
> 
> For example, if an unprivileged user lacks CAP_SYS_ADMIN, this will return
> false but set used_root to true. In callers like perf_event_paranoid_check():
> 
> tools/perf/util/util.c:perf_event_paranoid_check() {
>     ...
>     if (!used_root && perf_cap__capable(CAP_PERFMON, &used_root))
>     ...
> }
> 
> If a previous check for CAP_SYS_ADMIN failed and poisoned the used_root
> flag, won't it cause the subsequent check for CAP_PERFMON to be skipped
> entirely?
> 
> This pattern appears in other locations as well, such as
> check_bpf_filter_capable() where a failed CAP_BPF check might skip the
> fallback for non-root users to access root-pinned BPF filters because
> !used_root evaluates to false.
> 
> Additionally, could this override valid capability drops for root users
> in containers? Because geteuid() == 0 evaluates to true, it seems this might
> falsely claim root users still possess the capability even if it was
> explicitly dropped, potentially leading to later failures instead of
> graceful fallbacks.

Can we simply remove the used_root parameter from perf_cap__capable()?
Normally I think root would have the capability.  If not, maybe we need
to reject the operation as it may happen in a container setup.

Thanks,
Namhyung

Re: [PATCH v1 1/2] perf record: Fix teardown hang on system-wide multi-threaded sessions

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [High] The patch unconditionally skips `fdarray_flag__nonfilterable` file descriptors in `fdarray__filter()`, causing a memory leak of `perf_mmap` references and an infinite busy loop if a system-wide event or other nonfilterable file descriptor receives an error (e.g., POLLERR, POLLHUP).
--

--- Patch [1]: [PATCH v1 1/2] perf record: Fix teardown hang on system-wide multi-threaded sessions ---
commit 6940fed42e3ec7ba75ed80d3ab8c1cc4f0aad35d
Author: Ian Rogers <irogers@google.com>

perf record: Fix teardown hang on system-wide multi-threaded sessions

This patch attempts to resolve a hang during teardown in system-wide streaming
mode by adding explicit evlist disables. It also modifies the fdarray filter
logic to bypass nonfilterable file descriptors when processing POLLHUP signals.

> diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c
> index f0f195207fca9..3bfcd2aa7c07f 100644
> --- a/tools/lib/api/fd/array.c
> +++ b/tools/lib/api/fd/array.c
> @@ -115,6 +115,9 @@ int fdarray__filter(struct fdarray *fda, short revents,
>  		return 0;
>  
>  	for (fd = 0; fd < fda->nr; ++fd) {
> +		if (fda->priv[fd].flags & fdarray_flag__nonfilterable)
> +			continue;
> +

[Severity: High]
Does skipping the filter logic for non-filterable file descriptors here cause
an infinite poll loop if a system-wide event receives an error?

In tools/lib/perf/evlist.c:mmap_per_evsel(), system-wide event file
descriptors are marked with fdarray_flag__nonfilterable:

    flgs = evsel->system_wide ? fdarray_flag__nonfilterable : fdarray_flag__default;

If one of these file descriptors encounters an error like POLLERR or POLLHUP,
it seems this new continue statement bypasses the block that clears the events
mask and invokes the entry_destructor.

Could this cause poll() to continuously return immediately with the same
error, leading to a CPU busy loop and leaking perf_mmap references since the
destructor is skipped?

>  		if (!fda->entries[fd].events)
>  			continue;
>  
> @@ -126,8 +129,7 @@ int fdarray__filter(struct fdarray *fda, short revents,
>  			continue;
>  		}
>  
> -		if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable))
> -			++nr;
> +		++nr;
>  	}
>  
>  	return nr;

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260710053927.1864021-1-irogers@google.com?part=1

Re: [PATCH v1 1/2] perf record: Fix teardown hang on system-wide multi-threaded sessions

[Namhyung Kim]

Hi Ian,

On Thu, Jul 09, 2026 at 10:39:06PM -0700, Ian Rogers wrote:
> Under system-wide (-a) parallel streaming mode (--threads=cpu),
> background recording threads can be inundated by a continuous
> firehose of hardware samples generated by the OS. In this state,
> a background thread's local hit count remains unequal to its
> sample count, causing it to bypass the blocking fdarray__poll()
> call entirely on each iteration of its recording loop.
> 
> Because the termination check relies on the POLLHUP event status
> populated specifically by fdarray__poll(), bypassing it prevents
> the background thread from ever recognizing that its control pipe
> was closed by the main thread. This traps the background thread
> in an infinite recording loop, hanging the main thread indefinitely
> as it awaits a termination acknowledgment that never arrives.
> 
> Ensure teardown completion by adding explicit evlist__disable()
> calls in the main thread's cleanup paths at out_child: and
> out_child_no_flush:. Additionally, patch fdarray__filter() to
> respect the fdarray_flag__nonfilterable flag, preventing it
> from incorrectly setting the background thread's control pipe
> file descriptor to -1 and clearing its revents mask upon
> processing termination POLLHUP signals.
> 
> Fixes: f94563fac269 ("perf record: fix poll storm when monitored threads exit")
> Assisted-by: Antigravity:gemini-3.5-flash
> Signed-off-by: Ian Rogers <irogers@google.com>

I didn't know you already sent the fix so I was doing the same.  It
seems your fix is better as it added evlist__disable().  I'll test this
and pick up if it's ok.

Thanks,
Namhyung

> ---
>  tools/lib/api/fd/array.c    | 6 ++++--
>  tools/perf/builtin-record.c | 2 ++
>  2 files changed, 6 insertions(+), 2 deletions(-)
> 
> diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c
> index ffe8272af59b..16a047f1906e 100644
> --- a/tools/lib/api/fd/array.c
> +++ b/tools/lib/api/fd/array.c
> @@ -115,6 +115,9 @@ int fdarray__filter(struct fdarray *fda, short revents,
>  		return 0;
>  
>  	for (fd = 0; fd < fda->nr; ++fd) {
> +		if (fda->priv[fd].flags & fdarray_flag__nonfilterable)
> +			continue;
> +
>  		if (!fda->entries[fd].events)
>  			continue;
>  
> @@ -132,8 +135,7 @@ int fdarray__filter(struct fdarray *fda, short revents,
>  			continue;
>  		}
>  
> -		if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable))
> -			++nr;
> +		++nr;
>  	}
>  
>  	return nr;
> diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> index ebd3ed0c9b3e..d1276382b77a 100644
> --- a/tools/perf/builtin-record.c
> +++ b/tools/perf/builtin-record.c
> @@ -2890,11 +2890,13 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
>  		record__synthesize_workload(rec, true);
>  
>  out_child:
> +	evlist__disable(rec->evlist);
>  	record__stop_threads(rec);
>  	record__mmap_read_all(rec, true);
>  	goto out_free_threads;
>  out_child_no_flush:
>  	/* mmap read already failed — retrying would just fail again */
> +	evlist__disable(rec->evlist);
>  	record__stop_threads(rec);
>  out_free_threads:
>  	record__free_thread_data(rec);
> -- 
> 2.55.0.795.g602f6c329a-goog
> 

Re: (subset) [PATCH v1 1/2] perf record: Fix teardown hang on system-wide multi-threaded sessions

[Namhyung Kim]

On Thu, 09 Jul 2026 22:39:06 -0700, Ian Rogers wrote:
> Under system-wide (-a) parallel streaming mode (--threads=cpu),
> background recording threads can be inundated by a continuous
> firehose of hardware samples generated by the OS. In this state,
> a background thread's local hit count remains unequal to its
> sample count, causing it to bypass the blocking fdarray__poll()
> call entirely on each iteration of its recording loop.
> 
> [...]
Applied to perf-tools-next, thanks!

Best regards,
Namhyung