[PATCH v4 0/2] Minor fixes error handling of perf stat

[PATCH v4 0/2] Minor fixes error handling of perf stat

[Levi Yun]

This patchset fixes two issues that were seen when running
  "perf stat -r" with perf_event_paranoid=3

   1. failed with Too many open files.
     $ perf stat -r 1044 -- false
     ...
     failed to create 'go' pipe: Too many open files
     failed to prepare workload: Too many open files
     ...

   2. repating error message
     $ perf stat -r 1044 -- false
     Error:
     Access to performance monitoring and observability operations is limited.
     ...
     (repating with same error message 1044 times).

v4:
  - Move comments to run_perf_stat().

v3:
  - Fix comments.

v2:
  - Add some comments.

Levi Yun (2):
  perf stat: Close cork_fd when create_perf_stat_counter() failed
  perf stat: Stop repeating when ref_perf_stat() returns -1

 tools/perf/builtin-stat.c | 11 ++++++++++-
 tools/perf/util/evlist.c  | 14 +++++++++++++-
 tools/perf/util/evlist.h  |  1 +
 3 files changed, 24 insertions(+), 2 deletions(-)

--
LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}

[PATCH v4 1/2] perf stat: Close cork_fd when create_perf_stat_counter() failed

[Levi Yun]

When create_perf_stat_counter() failed, it doesn't close workload.cork_fd
open in evlist__prepare_workload(). This could make too many open file
error while __run_perf_stat() repeats.

Introduce evlist__cancel_workload to close workload.cork_fd and
wait workload.child_pid until exit to clear child process
when create_perf_stat_counter() is failed with COUNTER_FATAL.

Signed-off-by: Levi Yun <yeoreum.yun@arm.com>
Reviewed-by: James Clark <james.clark@linaro.org>
---
 tools/perf/builtin-stat.c |  4 ++++
 tools/perf/util/evlist.c  | 14 +++++++++++++-
 tools/perf/util/evlist.h  |  1 +
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
index 661832756a24..954eb37ce7b8 100644
--- a/tools/perf/builtin-stat.c
+++ b/tools/perf/builtin-stat.c
@@ -763,6 +763,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
 
 			switch (stat_handle_error(counter)) {
 			case COUNTER_FATAL:
+				if (forks)
+					evlist__cancel_workload(evsel_list);
 				return -1;
 			case COUNTER_RETRY:
 				goto try_again;
@@ -804,6 +806,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
 
 				switch (stat_handle_error(counter)) {
 				case COUNTER_FATAL:
+					if (forks)
+						evlist__cancel_workload(evsel_list);
 					return -1;
 				case COUNTER_RETRY:
 					goto try_again_reset;
diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c
index 3a719edafc7a..51a221679c92 100644
--- a/tools/perf/util/evlist.c
+++ b/tools/perf/util/evlist.c
@@ -46,6 +46,7 @@
 #include <sys/mman.h>
 #include <sys/prctl.h>
 #include <sys/timerfd.h>
+#include <sys/wait.h>
 
 #include <linux/bitops.h>
 #include <linux/hash.h>
@@ -1465,7 +1466,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
 		 * For cancelling the workload without actually running it,
 		 * the parent will just close workload.cork_fd, without writing
 		 * anything, i.e. read will return zero and we just exit()
-		 * here.
+		 * here (See evlist__cancel_workload()).
 		 */
 		if (ret != 1) {
 			if (ret == -1)
@@ -1546,6 +1547,17 @@ int evlist__start_workload(struct evlist *evlist)
 	return 0;
 }
 
+void evlist__cancel_workload(struct evlist *evlist)
+{
+	int status;
+
+	if (evlist->workload.cork_fd > 0) {
+		close(evlist->workload.cork_fd);
+		evlist->workload.cork_fd = -1;
+		waitpid(evlist->workload.pid, &status, WNOHANG);
+	}
+}
+
 int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample)
 {
 	struct evsel *evsel = evlist__event2evsel(evlist, event);
diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h
index cb91dc9117a2..12f929ffdf92 100644
--- a/tools/perf/util/evlist.h
+++ b/tools/perf/util/evlist.h
@@ -184,6 +184,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target,
 			     const char *argv[], bool pipe_output,
 			     void (*exec_error)(int signo, siginfo_t *info, void *ucontext));
 int evlist__start_workload(struct evlist *evlist);
+void evlist__cancel_workload(struct evlist *evlist);
 
 struct option;
 
-- 
LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}

Re: [PATCH v4 1/2] perf stat: Close cork_fd when create_perf_stat_counter() failed

[Namhyung Kim]

Hello,

On Fri, Sep 13, 2024 at 02:59:06PM +0100, Levi Yun wrote:
> When create_perf_stat_counter() failed, it doesn't close workload.cork_fd
> open in evlist__prepare_workload(). This could make too many open file
> error while __run_perf_stat() repeats.
> 
> Introduce evlist__cancel_workload to close workload.cork_fd and
> wait workload.child_pid until exit to clear child process
> when create_perf_stat_counter() is failed with COUNTER_FATAL.
> 
> Signed-off-by: Levi Yun <yeoreum.yun@arm.com>
> Reviewed-by: James Clark <james.clark@linaro.org>
> ---
>  tools/perf/builtin-stat.c |  4 ++++
>  tools/perf/util/evlist.c  | 14 +++++++++++++-
>  tools/perf/util/evlist.h  |  1 +
>  3 files changed, 18 insertions(+), 1 deletion(-)
> 
> diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
> index 661832756a24..954eb37ce7b8 100644
> --- a/tools/perf/builtin-stat.c
> +++ b/tools/perf/builtin-stat.c
> @@ -763,6 +763,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
>  
>  			switch (stat_handle_error(counter)) {
>  			case COUNTER_FATAL:
> +				if (forks)
> +					evlist__cancel_workload(evsel_list);
>  				return -1;
>  			case COUNTER_RETRY:
>  				goto try_again;
> @@ -804,6 +806,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
>  
>  				switch (stat_handle_error(counter)) {
>  				case COUNTER_FATAL:
> +					if (forks)
> +						evlist__cancel_workload(evsel_list);
>  					return -1;

I don't think you covered all the places it can return before starting
the workload.  You'd better have an error handling code at the end with
a label and then goto there.

Thanks,
Namhyung


>  				case COUNTER_RETRY:
>  					goto try_again_reset;
> diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c
> index 3a719edafc7a..51a221679c92 100644
> --- a/tools/perf/util/evlist.c
> +++ b/tools/perf/util/evlist.c
> @@ -46,6 +46,7 @@
>  #include <sys/mman.h>
>  #include <sys/prctl.h>
>  #include <sys/timerfd.h>
> +#include <sys/wait.h>
>  
>  #include <linux/bitops.h>
>  #include <linux/hash.h>
> @@ -1465,7 +1466,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
>  		 * For cancelling the workload without actually running it,
>  		 * the parent will just close workload.cork_fd, without writing
>  		 * anything, i.e. read will return zero and we just exit()
> -		 * here.
> +		 * here (See evlist__cancel_workload()).
>  		 */
>  		if (ret != 1) {
>  			if (ret == -1)
> @@ -1546,6 +1547,17 @@ int evlist__start_workload(struct evlist *evlist)
>  	return 0;
>  }
>  
> +void evlist__cancel_workload(struct evlist *evlist)
> +{
> +	int status;
> +
> +	if (evlist->workload.cork_fd > 0) {
> +		close(evlist->workload.cork_fd);
> +		evlist->workload.cork_fd = -1;
> +		waitpid(evlist->workload.pid, &status, WNOHANG);
> +	}
> +}
> +
>  int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample)
>  {
>  	struct evsel *evsel = evlist__event2evsel(evlist, event);
> diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h
> index cb91dc9117a2..12f929ffdf92 100644
> --- a/tools/perf/util/evlist.h
> +++ b/tools/perf/util/evlist.h
> @@ -184,6 +184,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target,
>  			     const char *argv[], bool pipe_output,
>  			     void (*exec_error)(int signo, siginfo_t *info, void *ucontext));
>  int evlist__start_workload(struct evlist *evlist);
> +void evlist__cancel_workload(struct evlist *evlist);
>  
>  struct option;
>  
> -- 
> LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}
> 

Re: [PATCH v4 1/2] perf stat: Close cork_fd when create_perf_stat_counter() failed

[Yeo Reum Yun]

Hello,

> I don't think you covered all the places it can return before starting
> the workload.  You'd better have an error handling code at the end with
> a label and then goto there.

Right. I'll change it.

Thanks

________________________________________
From: Namhyung Kim <namhyung@kernel.org>
Sent: 24 September 2024 20:29
To: Yeo Reum Yun
Cc: peterz@infradead.org; mingo@redhat.com; acme@kernel.org; Mark Rutland; alexander.shishkin@linux.intel.com; jolsa@kernel.org; irogers@google.com; james.clark@linaro.org; asmadeus@codewreck.org; linux-perf-users@vger.kernel.org; linux-kernel@vger.kernel.org; nd
Subject: Re: [PATCH v4 1/2] perf stat: Close cork_fd when create_perf_stat_counter() failed

Hello,

On Fri, Sep 13, 2024 at 02:59:06PM +0100, Levi Yun wrote:
> When create_perf_stat_counter() failed, it doesn't close workload.cork_fd
> open in evlist__prepare_workload(). This could make too many open file
> error while __run_perf_stat() repeats.
>
> Introduce evlist__cancel_workload to close workload.cork_fd and
> wait workload.child_pid until exit to clear child process
> when create_perf_stat_counter() is failed with COUNTER_FATAL.
>
> Signed-off-by: Levi Yun <yeoreum.yun@arm.com>
> Reviewed-by: James Clark <james.clark@linaro.org>
> ---
>  tools/perf/builtin-stat.c |  4 ++++
>  tools/perf/util/evlist.c  | 14 +++++++++++++-
>  tools/perf/util/evlist.h  |  1 +
>  3 files changed, 18 insertions(+), 1 deletion(-)
>
> diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
> index 661832756a24..954eb37ce7b8 100644
> --- a/tools/perf/builtin-stat.c
> +++ b/tools/perf/builtin-stat.c
> @@ -763,6 +763,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
>
>                       switch (stat_handle_error(counter)) {
>                       case COUNTER_FATAL:
> +                             if (forks)
> +                                     evlist__cancel_workload(evsel_list);
>                               return -1;
>                       case COUNTER_RETRY:
>                               goto try_again;
> @@ -804,6 +806,8 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
>
>                               switch (stat_handle_error(counter)) {
>                               case COUNTER_FATAL:
> +                                     if (forks)
> +                                             evlist__cancel_workload(evsel_list);
>                                       return -1;

I don't think you covered all the places it can return before starting
the workload.  You'd better have an error handling code at the end with
a label and then goto there.

Thanks,
Namhyung


>                               case COUNTER_RETRY:
>                                       goto try_again_reset;
> diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c
> index 3a719edafc7a..51a221679c92 100644
> --- a/tools/perf/util/evlist.c
> +++ b/tools/perf/util/evlist.c
> @@ -46,6 +46,7 @@
>  #include <sys/mman.h>
>  #include <sys/prctl.h>
>  #include <sys/timerfd.h>
> +#include <sys/wait.h>
>
>  #include <linux/bitops.h>
>  #include <linux/hash.h>
> @@ -1465,7 +1466,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
>                * For cancelling the workload without actually running it,
>                * the parent will just close workload.cork_fd, without writing
>                * anything, i.e. read will return zero and we just exit()
> -              * here.
> +              * here (See evlist__cancel_workload()).
>                */
>               if (ret != 1) {
>                       if (ret == -1)
> @@ -1546,6 +1547,17 @@ int evlist__start_workload(struct evlist *evlist)
>       return 0;
>  }
>
> +void evlist__cancel_workload(struct evlist *evlist)
> +{
> +     int status;
> +
> +     if (evlist->workload.cork_fd > 0) {
> +             close(evlist->workload.cork_fd);
> +             evlist->workload.cork_fd = -1;
> +             waitpid(evlist->workload.pid, &status, WNOHANG);
> +     }
> +}
> +
>  int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample)
>  {
>       struct evsel *evsel = evlist__event2evsel(evlist, event);
> diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h
> index cb91dc9117a2..12f929ffdf92 100644
> --- a/tools/perf/util/evlist.h
> +++ b/tools/perf/util/evlist.h
> @@ -184,6 +184,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target,
>                            const char *argv[], bool pipe_output,
>                            void (*exec_error)(int signo, siginfo_t *info, void *ucontext));
>  int evlist__start_workload(struct evlist *evlist);
> +void evlist__cancel_workload(struct evlist *evlist);
>
>  struct option;
>
> --
> LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}
>

[PATCH v4 2/2] perf stat: Stop repeating when ref_perf_stat() returns -1

[Levi Yun]

Exit when run_perf_stat() returns an error to avoid continuously
repeating the same error message. It's not expected that COUNTER_FATAL
or internal errors are recoverable so there's no point in retrying.

This fixes the following flood of error messages for permission issues,
for example when perf_event_paranoid==3:
  perf stat -r 1044 -- false

  Error:
  Access to performance monitoring and observability operations is limited.
  ...
  Error:
  Access to performance monitoring and observability operations is limited.
  ...
  (repeating for 1044 times).

Signed-off-by: Levi Yun <yeoreum.yun@arm.com>
Reviewed-by: James Clark <james.clark@linaro.org>
---
 tools/perf/builtin-stat.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
index 954eb37ce7b8..8b7fc1e3c390 100644
--- a/tools/perf/builtin-stat.c
+++ b/tools/perf/builtin-stat.c
@@ -948,6 +948,14 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
 	return WEXITSTATUS(status);
 }
 
+/*
+ * Returns -1 for fatal errors which signifies to not continue
+ * when in repeat mode.
+ *
+ * Returns < -1 error codes when stat record is used. These
+ * result in the stat information being displayed, but writing
+ * to the file fails and is non fatal.
+ */
 static int run_perf_stat(int argc, const char **argv, int run_idx)
 {
 	int ret;
@@ -2875,7 +2883,10 @@ int cmd_stat(int argc, const char **argv)
 			evlist__reset_prev_raw_counts(evsel_list);
 
 		status = run_perf_stat(argc, argv, run_idx);
-		if (forever && status != -1 && !interval) {
+		if (status == -1)
+			break;
+
+		if (forever && !interval) {
 			print_counters(NULL, argc, argv);
 			perf_stat__reset_stats();
 		}
-- 
LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}

Re: [PATCH v4 0/2] Minor fixes error handling of perf stat

[Yeo Reum Yun]

Gentle ping.

________________________________________
From: Levi Yun <yeoreum.yun@arm.com>
Sent: 13 September 2024 14:59
To: peterz@infradead.org; mingo@redhat.com; acme@kernel.org; namhyung@kernel.org; Mark Rutland; alexander.shishkin@linux.intel.com; jolsa@kernel.org; irogers@google.com; james.clark@linaro.org; asmadeus@codewreck.org
Cc: linux-perf-users@vger.kernel.org; linux-kernel@vger.kernel.org; nd; Yeo Reum Yun
Subject: [PATCH v4 0/2] Minor fixes error handling of perf stat

This patchset fixes two issues that were seen when running
  "perf stat -r" with perf_event_paranoid=3

   1. failed with Too many open files.
     $ perf stat -r 1044 -- false
     ...
     failed to create 'go' pipe: Too many open files
     failed to prepare workload: Too many open files
     ...

   2. repating error message
     $ perf stat -r 1044 -- false
     Error:
     Access to performance monitoring and observability operations is limited.
     ...
     (repating with same error message 1044 times).

v4:
  - Move comments to run_perf_stat().

v3:
  - Fix comments.

v2:
  - Add some comments.

Levi Yun (2):
  perf stat: Close cork_fd when create_perf_stat_counter() failed
  perf stat: Stop repeating when ref_perf_stat() returns -1

 tools/perf/builtin-stat.c | 11 ++++++++++-
 tools/perf/util/evlist.c  | 14 +++++++++++++-
 tools/perf/util/evlist.h  |  1 +
 3 files changed, 24 insertions(+), 2 deletions(-)

--
LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}