[PATCHSET 0/5] perf tools: Show per-event lost sample count (v1)

[PATCHSET 0/5] perf tools: Show per-event lost sample count (v1)

[Namhyung Kim]

Hello,

Now we have PERF_FORMAT_LOST support, add it to perf record and report so that
it can show number of lost samples per event.  This can be useful if you want
to reconstruct number of events from the samples like when using -c option.

Currently it adds PERF_RECORD_LOST_SAMPLES at the end of perf data after reading
event values by read(2).  The perf record unconditionally sets the lost bit if
the kernel supports it.  Users can see the number with `perf report --stat`.

You can get the code from 'perf/report-lost-v1' brach on

  git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.git

Thanks,
Namhyung


Namhyung Kim (5):
  perf tools: Print LOST read format in the verbose mode
  perf record: Set PERF_FORMAT_LOST by default
  perf record: Read and inject LOST_SAMPLES events
  perf hist: Add nr_lost_samples to hist_stats
  perf report: Show per-event LOST SAMPLES stat

 tools/perf/builtin-record.c               | 60 +++++++++++++++++++++++
 tools/perf/builtin-report.c               | 17 +++++++
 tools/perf/util/events_stats.h            |  1 +
 tools/perf/util/evsel.c                   | 10 +++-
 tools/perf/util/evsel.h                   |  1 +
 tools/perf/util/hist.c                    |  8 +++
 tools/perf/util/hist.h                    |  1 +
 tools/perf/util/perf_event_attr_fprintf.c |  2 +-
 8 files changed, 98 insertions(+), 2 deletions(-)


base-commit: 2f4c2fbd2ff5e3176b0121443f995808ee498bd0
-- 
2.37.2.789.g6183377224-goog

[PATCH 1/5] perf tools: Print LOST read format in the verbose mode

[Namhyung Kim]

So that we can see it with:

  $ perf record -vv pwd
  ...
  perf_event_attr:
    size                             128
    { sample_period, sample_freq }   4000
    sample_type                      IP|TID|TIME|PERIOD
    read_format                      ID|LOST
    disabled                         1
    inherit                          1
    exclude_kernel                   1
    freq                             1
    enable_on_exec                   1
    precise_ip                       3
    sample_id_all                    1
    exclude_guest                    1

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/util/perf_event_attr_fprintf.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/perf/util/perf_event_attr_fprintf.c b/tools/perf/util/perf_event_attr_fprintf.c
index 4b0db27b7199..7e5e7b30510d 100644
--- a/tools/perf/util/perf_event_attr_fprintf.c
+++ b/tools/perf/util/perf_event_attr_fprintf.c
@@ -64,7 +64,7 @@ static void __p_read_format(char *buf, size_t size, u64 value)
 #define bit_name(n) { PERF_FORMAT_##n, #n }
 	struct bit_names bits[] = {
 		bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING),
-		bit_name(ID), bit_name(GROUP),
+		bit_name(ID), bit_name(GROUP), bit_name(LOST),
 		{ .name = NULL, }
 	};
 #undef bit_name
-- 
2.37.2.789.g6183377224-goog

[PATCH 2/5] perf record: Set PERF_FORMAT_LOST by default

[Namhyung Kim]

As we want to see the number of lost samples in the perf report, set the
LOST format when it configs evsel.  On old kernels, it'd fallback to
disable it.

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/util/evsel.c | 10 +++++++++-
 tools/perf/util/evsel.h |  1 +
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index e1bc76ece117..5776bfa70f11 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -1161,6 +1161,7 @@ void evsel__config(struct evsel *evsel, struct record_opts *opts,
 	attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
 	attr->inherit	    = !opts->no_inherit;
 	attr->write_backward = opts->overwrite ? 1 : 0;
+	attr->read_format   = PERF_FORMAT_LOST;
 
 	evsel__set_sample_bit(evsel, IP);
 	evsel__set_sample_bit(evsel, TID);
@@ -1856,6 +1857,8 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
 
 static void evsel__disable_missing_features(struct evsel *evsel)
 {
+	if (perf_missing_features.read_lost)
+		evsel->core.attr.read_format &= ~PERF_FORMAT_LOST;
 	if (perf_missing_features.weight_struct) {
 		evsel__set_sample_bit(evsel, WEIGHT);
 		evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
@@ -1907,7 +1910,12 @@ bool evsel__detect_missing_features(struct evsel *evsel)
 	 * Must probe features in the order they were added to the
 	 * perf_event_attr interface.
 	 */
-	if (!perf_missing_features.weight_struct &&
+	if (!perf_missing_features.read_lost &&
+	    (evsel->core.attr.read_format & PERF_FORMAT_LOST)) {
+		perf_missing_features.read_lost = true;
+		pr_debug2("switching off PERF_FORMAT_LOST support\n");
+		return true;
+	} else if (!perf_missing_features.weight_struct &&
 	    (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
 		perf_missing_features.weight_struct = true;
 		pr_debug2("switching off weight struct support\n");
diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
index d927713b513e..989865e16aad 100644
--- a/tools/perf/util/evsel.h
+++ b/tools/perf/util/evsel.h
@@ -188,6 +188,7 @@ struct perf_missing_features {
 	bool data_page_size;
 	bool code_page_size;
 	bool weight_struct;
+	bool read_lost;
 };
 
 extern struct perf_missing_features perf_missing_features;
-- 
2.37.2.789.g6183377224-goog

[PATCH 3/5] perf record: Read and inject LOST_SAMPLES events

[Namhyung Kim]

When there are lost samples, it can read the number of PERF_FORMAT_LOST and
convert it to PERF_RECORD_LOST_SAMPLES and write to the data file at the end.

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/builtin-record.c | 60 +++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index bce8c941d558..cb9881543a07 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -10,6 +10,7 @@
 
 #include "util/build-id.h"
 #include <subcmd/parse-options.h>
+#include <internal/xyarray.h>
 #include "util/parse-events.h"
 #include "util/config.h"
 
@@ -1852,6 +1853,64 @@ record__switch_output(struct record *rec, bool at_exit)
 	return fd;
 }
 
+static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
+					struct perf_record_lost_samples *lost,
+					int size, int cpu_idx, int thread_idx)
+{
+	struct perf_counts_values count;
+	struct perf_sample_id *sid;
+	struct perf_sample sample = {};
+
+	if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
+		pr_err("read LOST count failed\n");
+		return;
+	}
+
+	if (count.lost == 0)
+		return;
+
+	lost->lost = count.lost;
+	if (evsel->core.ids) {
+		sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
+		sample.id = sid->id;
+	}
+
+	perf_event__synthesize_id_sample((void *)(lost + 1),
+					 evsel->core.attr.sample_type, &sample);
+	record__write(rec, NULL, lost, size);
+}
+
+static void record__read_lost_samples(struct record *rec)
+{
+	struct perf_session *session = rec->session;
+	struct machine *machine = &session->machines.host;
+	struct perf_record_lost_samples *lost;
+	struct evsel *evsel;
+	int size = sizeof(*lost) + machine->id_hdr_size;
+
+	lost = zalloc(size);
+	lost->header.type = PERF_RECORD_LOST_SAMPLES;
+	lost->header.size = size;
+
+	evlist__for_each_entry(session->evlist, evsel) {
+		struct xyarray *xy = evsel->core.sample_id;
+
+		if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
+		    xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
+			pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
+			continue;
+		}
+
+		for (int x = 0; x < xyarray__max_x(xy); x++) {
+			for (int y = 0; y < xyarray__max_y(xy); y++) {
+				__record__read_lost_samples(rec, evsel, lost,
+							    size, x, y);
+			}
+		}
+	}
+
+}
+
 static volatile int workload_exec_errno;
 
 /*
@@ -2710,6 +2769,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 	if (rec->off_cpu)
 		rec->bytes_written += off_cpu_write(rec->session);
 
+	record__read_lost_samples(rec);
 	record__synthesize(rec, true);
 	/* this will be recalculated during process_buildids() */
 	rec->samples = 0;
-- 
2.37.2.789.g6183377224-goog

Re: [PATCH 3/5] perf record: Read and inject LOST_SAMPLES events

[Adrian Hunter]

On 1/09/22 00:03, Namhyung Kim wrote:
> When there are lost samples, it can read the number of PERF_FORMAT_LOST and
> convert it to PERF_RECORD_LOST_SAMPLES and write to the data file at the end.
> 
> Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> ---
>  tools/perf/builtin-record.c | 60 +++++++++++++++++++++++++++++++++++++
>  1 file changed, 60 insertions(+)
> 
> diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> index bce8c941d558..cb9881543a07 100644
> --- a/tools/perf/builtin-record.c
> +++ b/tools/perf/builtin-record.c
> @@ -10,6 +10,7 @@
>  
>  #include "util/build-id.h"
>  #include <subcmd/parse-options.h>
> +#include <internal/xyarray.h>
>  #include "util/parse-events.h"
>  #include "util/config.h"
>  
> @@ -1852,6 +1853,64 @@ record__switch_output(struct record *rec, bool at_exit)
>  	return fd;
>  }
>  
> +static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
> +					struct perf_record_lost_samples *lost,
> +					int size, int cpu_idx, int thread_idx)
> +{
> +	struct perf_counts_values count;
> +	struct perf_sample_id *sid;
> +	struct perf_sample sample = {};
> +
> +	if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
> +		pr_err("read LOST count failed\n");
> +		return;
> +	}
> +
> +	if (count.lost == 0)
> +		return;
> +
> +	lost->lost = count.lost;
> +	if (evsel->core.ids) {
> +		sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
> +		sample.id = sid->id;
> +	}
> +
> +	perf_event__synthesize_id_sample((void *)(lost + 1),
> +					 evsel->core.attr.sample_type, &sample);

The ID sample size can vary with sample_type and is not necessarily the same as
machine->id_hdr_size.

The following might be more robust:

	id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1), evsel->core.attr.sample_type, &sample);
	lost->header.size = sizeof(*lost) + id_hdr_size;

> +	record__write(rec, NULL, lost, size);
> +}
> +
> +static void record__read_lost_samples(struct record *rec)
> +{
> +	struct perf_session *session = rec->session;
> +	struct machine *machine = &session->machines.host;
> +	struct perf_record_lost_samples *lost;
> +	struct evsel *evsel;
> +	int size = sizeof(*lost) + machine->id_hdr_size;

  -	int size = sizeof(*lost) + machine->id_hdr_size;

> +
> +	lost = zalloc(size);

	lost = zalloc(PERF_SAMPLE_MAX_SIZE);


> +	lost->header.type = PERF_RECORD_LOST_SAMPLES;
> +	lost->header.size = size;

  -	lost->header.size = size;

> +
> +	evlist__for_each_entry(session->evlist, evsel) {
> +		struct xyarray *xy = evsel->core.sample_id;
> +
> +		if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
> +		    xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
> +			pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
> +			continue;
> +		}
> +
> +		for (int x = 0; x < xyarray__max_x(xy); x++) {
> +			for (int y = 0; y < xyarray__max_y(xy); y++) {
> +				__record__read_lost_samples(rec, evsel, lost,
> +							    size, x, y);
> +			}
> +		}
> +	}
> +
> +}
> +
>  static volatile int workload_exec_errno;
>  
>  /*
> @@ -2710,6 +2769,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
>  	if (rec->off_cpu)
>  		rec->bytes_written += off_cpu_write(rec->session);
>  
> +	record__read_lost_samples(rec);
>  	record__synthesize(rec, true);
>  	/* this will be recalculated during process_buildids() */
>  	rec->samples = 0;

Re: [PATCH 3/5] perf record: Read and inject LOST_SAMPLES events

[Namhyung Kim]

Hi Adrian,

On Thu, Sep 1, 2022 at 4:23 AM Adrian Hunter <adrian.hunter@intel.com> wrote:
>
> On 1/09/22 00:03, Namhyung Kim wrote:
> > When there are lost samples, it can read the number of PERF_FORMAT_LOST and
> > convert it to PERF_RECORD_LOST_SAMPLES and write to the data file at the end.
> >
> > Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> > ---
> >  tools/perf/builtin-record.c | 60 +++++++++++++++++++++++++++++++++++++
> >  1 file changed, 60 insertions(+)
> >
> > diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> > index bce8c941d558..cb9881543a07 100644
> > --- a/tools/perf/builtin-record.c
> > +++ b/tools/perf/builtin-record.c
> > @@ -10,6 +10,7 @@
> >
> >  #include "util/build-id.h"
> >  #include <subcmd/parse-options.h>
> > +#include <internal/xyarray.h>
> >  #include "util/parse-events.h"
> >  #include "util/config.h"
> >
> > @@ -1852,6 +1853,64 @@ record__switch_output(struct record *rec, bool at_exit)
> >       return fd;
> >  }
> >
> > +static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
> > +                                     struct perf_record_lost_samples *lost,
> > +                                     int size, int cpu_idx, int thread_idx)
> > +{
> > +     struct perf_counts_values count;
> > +     struct perf_sample_id *sid;
> > +     struct perf_sample sample = {};
> > +
> > +     if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
> > +             pr_err("read LOST count failed\n");
> > +             return;
> > +     }
> > +
> > +     if (count.lost == 0)
> > +             return;
> > +
> > +     lost->lost = count.lost;
> > +     if (evsel->core.ids) {
> > +             sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
> > +             sample.id = sid->id;
> > +     }
> > +
> > +     perf_event__synthesize_id_sample((void *)(lost + 1),
> > +                                      evsel->core.attr.sample_type, &sample);
>
> The ID sample size can vary with sample_type and is not necessarily the same as
> machine->id_hdr_size.
>
> The following might be more robust:
>
>         id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1), evsel->core.attr.sample_type, &sample);
>         lost->header.size = sizeof(*lost) + id_hdr_size;

Will change.

Thanks for your review!
Namhyung


>
> > +     record__write(rec, NULL, lost, size);
> > +}
> > +
> > +static void record__read_lost_samples(struct record *rec)
> > +{
> > +     struct perf_session *session = rec->session;
> > +     struct machine *machine = &session->machines.host;
> > +     struct perf_record_lost_samples *lost;
> > +     struct evsel *evsel;
> > +     int size = sizeof(*lost) + machine->id_hdr_size;
>
>   -     int size = sizeof(*lost) + machine->id_hdr_size;
>
> > +
> > +     lost = zalloc(size);
>
>         lost = zalloc(PERF_SAMPLE_MAX_SIZE);
>
>
> > +     lost->header.type = PERF_RECORD_LOST_SAMPLES;
> > +     lost->header.size = size;
>
>   -     lost->header.size = size;
>
> > +
> > +     evlist__for_each_entry(session->evlist, evsel) {
> > +             struct xyarray *xy = evsel->core.sample_id;
> > +
> > +             if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
> > +                 xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
> > +                     pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
> > +                     continue;
> > +             }
> > +
> > +             for (int x = 0; x < xyarray__max_x(xy); x++) {
> > +                     for (int y = 0; y < xyarray__max_y(xy); y++) {
> > +                             __record__read_lost_samples(rec, evsel, lost,
> > +                                                         size, x, y);
> > +                     }
> > +             }
> > +     }
> > +
> > +}
> > +
> >  static volatile int workload_exec_errno;
> >
> >  /*
> > @@ -2710,6 +2769,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
> >       if (rec->off_cpu)
> >               rec->bytes_written += off_cpu_write(rec->session);
> >
> > +     record__read_lost_samples(rec);
> >       record__synthesize(rec, true);
> >       /* this will be recalculated during process_buildids() */
> >       rec->samples = 0;
>

[PATCH 4/5] perf hist: Add nr_lost_samples to hist_stats

[Namhyung Kim]

This is a preparation to display accurate lost sample counts for
each evsel.

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/util/events_stats.h | 1 +
 tools/perf/util/hist.c         | 5 +++++
 tools/perf/util/hist.h         | 1 +
 3 files changed, 7 insertions(+)

diff --git a/tools/perf/util/events_stats.h b/tools/perf/util/events_stats.h
index 040ab9d0a803..8fecc9fbaecc 100644
--- a/tools/perf/util/events_stats.h
+++ b/tools/perf/util/events_stats.h
@@ -47,6 +47,7 @@ struct hists_stats {
 	u64 total_non_filtered_period;
 	u32 nr_samples;
 	u32 nr_non_filtered_samples;
+	u32 nr_lost_samples;
 };
 
 void events_stats__inc(struct events_stats *stats, u32 type);
diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
index 698add038cec..8cab049f7119 100644
--- a/tools/perf/util/hist.c
+++ b/tools/perf/util/hist.c
@@ -2335,6 +2335,11 @@ void hists__inc_nr_samples(struct hists *hists, bool filtered)
 		hists->stats.nr_non_filtered_samples++;
 }
 
+void hists__inc_nr_lost_samples(struct hists *hists, u32 lost)
+{
+	hists->stats.nr_lost_samples += lost;
+}
+
 static struct hist_entry *hists__add_dummy_entry(struct hists *hists,
 						 struct hist_entry *pair)
 {
diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h
index 508428b2c1b2..c7a7a3fa0b87 100644
--- a/tools/perf/util/hist.h
+++ b/tools/perf/util/hist.h
@@ -201,6 +201,7 @@ void hists__reset_stats(struct hists *hists);
 void hists__inc_stats(struct hists *hists, struct hist_entry *h);
 void hists__inc_nr_events(struct hists *hists);
 void hists__inc_nr_samples(struct hists *hists, bool filtered);
+void hists__inc_nr_lost_samples(struct hists *hists, u32 lost);
 
 size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows,
 		      int max_cols, float min_pcnt, FILE *fp,
-- 
2.37.2.789.g6183377224-goog

[PATCH 5/5] perf report: Show per-event LOST SAMPLES stat

[Namhyung Kim]

Display lost samples with --stat (if not zero):

  $ perf report --stat
    Aggregated stats:
             TOTAL events:         64
              COMM events:          2  ( 3.1%)
              EXIT events:          1  ( 1.6%)
            SAMPLE events:         26  (40.6%)
             MMAP2 events:          4  ( 6.2%)
      LOST_SAMPLES events:          1  ( 1.6%)
              ATTR events:          2  ( 3.1%)
    FINISHED_ROUND events:          1  ( 1.6%)
          ID_INDEX events:          1  ( 1.6%)
        THREAD_MAP events:          1  ( 1.6%)
           CPU_MAP events:          1  ( 1.6%)
      EVENT_UPDATE events:          2  ( 3.1%)
         TIME_CONV events:          1  ( 1.6%)
           FEATURE events:         20  (31.2%)
     FINISHED_INIT events:          1  ( 1.6%)
  cycles:uH stats:
            SAMPLE events:         14
      LOST_SAMPLES events:          1
  instructions:uH stats:
            SAMPLE events:         12

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
 tools/perf/builtin-report.c | 17 +++++++++++++++++
 tools/perf/util/hist.c      |  3 +++
 2 files changed, 20 insertions(+)

diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c
index 91ed41cc7d88..8361890176c2 100644
--- a/tools/perf/builtin-report.c
+++ b/tools/perf/builtin-report.c
@@ -752,6 +752,22 @@ static int count_sample_event(struct perf_tool *tool __maybe_unused,
 	return 0;
 }
 
+static int count_lost_samples_event(struct perf_tool *tool,
+				    union perf_event *event,
+				    struct perf_sample *sample,
+				    struct machine *machine __maybe_unused)
+{
+	struct report *rep = container_of(tool, struct report, tool);
+	struct evsel *evsel;
+
+	evsel = evlist__id2evsel(rep->session->evlist, sample->id);
+	if (evsel) {
+		hists__inc_nr_lost_samples(evsel__hists(evsel),
+					   event->lost_samples.lost);
+	}
+	return 0;
+}
+
 static int process_attr(struct perf_tool *tool __maybe_unused,
 			union perf_event *event,
 			struct evlist **pevlist);
@@ -761,6 +777,7 @@ static void stats_setup(struct report *rep)
 	memset(&rep->tool, 0, sizeof(rep->tool));
 	rep->tool.attr = process_attr;
 	rep->tool.sample = count_sample_event;
+	rep->tool.lost_samples = count_lost_samples_event;
 	rep->tool.no_warn = true;
 }
 
diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
index 8cab049f7119..cc363ba20c73 100644
--- a/tools/perf/util/hist.c
+++ b/tools/perf/util/hist.c
@@ -2689,6 +2689,9 @@ size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp,
 		ret += fprintf(fp, "%s stats:\n", evsel__name(pos));
 		ret += fprintf(fp, "%16s events: %10d\n",
 			       "SAMPLE", hists->stats.nr_samples);
+		if (hists->stats.nr_lost_samples)
+			ret += fprintf(fp, "%16s events: %10d\n",
+				       "LOST_SAMPLES", hists->stats.nr_lost_samples);
 	}
 
 	return ret;
-- 
2.37.2.789.g6183377224-goog