[PATCH v4 0/6] tracing/osnoise: Track IPIs

[PATCH v4 0/6] tracing/osnoise: Track IPIs

[Valentin Schneider]

Hi folks,

So I've seen a few times now reports of latency spikes caused by IPIs, usually
because of isolation misconfiguration, but only detected at the tail of end
e.g. a 24h timerlat run.

It's not because those IPIs are rare, but rather that they don't by themselves
cause a monitered CPU to reach the latency threshold, it's usually a combined
interference that gets us there.

I'd like to make it easier to detect such misconfigurations and thus IPIs
hitting supposedly-isolated CPUs. I initially kludged a timerlat option to stop
tracing as soon as an IPI was sent to a monitored CPU, regardless of the latency
threshold. It sort of did the trick, but Tomáš convinced me timerlat wasn't
really the place for that.

Patches are available at

https://gitlab.com/vschneid/linux.git -b mainline/tracing/osnoise-ipi-userspace/v4

Cheers,
Valentin

Revisions
=========

v3 -> v4
++++++++

o Rebased onto v7.2-rc4
o Fixed changelog typo

v2 -> v3
++++++++

o Dropped the short-form -i option to leave it free 
o Re-arranged top header printing 
o Fixed tracefs_event_file_write() return value handling

o Changed IPI filtering to allow it to gracefully fail on older kernels
o Added filter clearing for -e events to ensure a known state


v1 -> v2
++++++++

o Dropped the in-kernel osnoise_sample changes and made it all userspace

Valentin Schneider (6):
  rtla/osnoise: Add IPI tracking cmdline option
  rtla/osnoise: Record IPI count in osnoise top
  rtla/osnoise: Leverage IPI event filters when tracing a subset of CPUs
  rtla/osnoise: Allow IPI filters to gracefully fail
  rtla: Unconditionally clean any pre-existing filters for user-provided
    events
  rtla/osnoise: Trace IPI events when recording a trace file

 Documentation/tools/rtla/rtla-osnoise-top.rst |   4 +
 tools/tracing/rtla/src/cli.c                  |   1 +
 tools/tracing/rtla/src/cli_p.h                |   3 +
 tools/tracing/rtla/src/common.c               |   2 +-
 tools/tracing/rtla/src/common.h               |   3 +-
 tools/tracing/rtla/src/osnoise.c              |  72 +++++++-
 tools/tracing/rtla/src/osnoise.h              |   4 +
 tools/tracing/rtla/src/osnoise_top.c          | 155 +++++++++++++++++-
 tools/tracing/rtla/src/trace.c                |   4 +
 9 files changed, 244 insertions(+), 4 deletions(-)

--
2.55.0

[PATCH v4 1/6] rtla/osnoise: Add IPI tracking cmdline option

[Valentin Schneider]

Later commits will add IPI tracking to osnoise top. To avoid breaking
existing scripts, this new feature will be gated behind a new --ipi option.

Suggested-by: Tomas Glozar <tglozar@redhat.com>
Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
 Documentation/tools/rtla/rtla-osnoise-top.rst | 4 ++++
 tools/tracing/rtla/src/cli.c                  | 1 +
 tools/tracing/rtla/src/cli_p.h                | 3 +++
 tools/tracing/rtla/src/common.h               | 1 +
 4 files changed, 9 insertions(+)

diff --git a/Documentation/tools/rtla/rtla-osnoise-top.rst b/Documentation/tools/rtla/rtla-osnoise-top.rst
index b91c02ac2bbe1..b948a813d45cb 100644
--- a/Documentation/tools/rtla/rtla-osnoise-top.rst
+++ b/Documentation/tools/rtla/rtla-osnoise-top.rst
@@ -28,6 +28,10 @@ OPTIONS
 =======
 .. include:: common_osnoise_options.txt
 
+**--ipi**
+
+	Track sources of IPIs.
+
 .. include:: common_top_options.txt
 
 .. include:: common_options.txt
diff --git a/tools/tracing/rtla/src/cli.c b/tools/tracing/rtla/src/cli.c
index c5279c9875310..eb1e76a6b0dea 100644
--- a/tools/tracing/rtla/src/cli.c
+++ b/tools/tracing/rtla/src/cli.c
@@ -78,6 +78,7 @@ struct common_params *osnoise_top_parse_args(int argc, char **argv)
 		RTLA_OPT_STOP_TOTAL('S', "stop-total", "total sample"),
 		OSNOISE_OPT_THRESHOLD,
 		RTLA_OPT_TRACE_OUTPUT("osnoise", opt_osnoise_trace_output_cb),
+		OSNOISE_OPT_IPI,
 
 	OPT_GROUP("Event Configuration:"),
 		RTLA_OPT_EVENT,
diff --git a/tools/tracing/rtla/src/cli_p.h b/tools/tracing/rtla/src/cli_p.h
index 3c939de9abf02..d120fac424e30 100644
--- a/tools/tracing/rtla/src/cli_p.h
+++ b/tools/tracing/rtla/src/cli_p.h
@@ -305,6 +305,9 @@ static int opt_filter_cb(const struct option *opt, const char *arg, int unset)
 	"the minimum delta to be considered a noise", \
 	opt_llong_callback)
 
+#define OSNOISE_OPT_IPI OPT_BOOLEAN(0, "ipi", &params->common.ipi, \
+	"track sources of IPIs")
+
 /*
  * Callback functions for command line options for osnoise tools
  */
diff --git a/tools/tracing/rtla/src/common.h b/tools/tracing/rtla/src/common.h
index 04b287a03f6d4..045253230fcf2 100644
--- a/tools/tracing/rtla/src/common.h
+++ b/tools/tracing/rtla/src/common.h
@@ -108,6 +108,7 @@ struct common_params {
 	bool			kernel_workload;
 	bool			user_data;
 	bool			aa_only;
+	bool			ipi;
 
 	struct actions		threshold_actions;
 	struct actions		end_actions;
-- 
2.55.0

[PATCH v4 2/6] rtla/osnoise: Record IPI count in osnoise top

[Valentin Schneider]

Leverage the ipi_send_cpu and ipi_send_cpumask trace events to record the
count of IPIs sent to monitored CPUs. These interferences are already
accounted by the IRQ count, but this split gives a better overall picture.

This uses the newly added --ipi cmdline option.

Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
 tools/tracing/rtla/src/osnoise_top.c | 116 ++++++++++++++++++++++++++-
 1 file changed, 115 insertions(+), 1 deletion(-)

diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c
index 512a6299cb018..7b5ae5336cf08 100644
--- a/tools/tracing/rtla/src/osnoise_top.c
+++ b/tools/tracing/rtla/src/osnoise_top.c
@@ -8,6 +8,7 @@
 #include <string.h>
 #include <signal.h>
 #include <unistd.h>
+#include <errno.h>
 #include <stdio.h>
 #include <time.h>
 
@@ -25,6 +26,7 @@ struct osnoise_top_cpu {
 	unsigned long long	irq_count;
 	unsigned long long	softirq_count;
 	unsigned long long	thread_count;
+	unsigned long long	ipi_count;
 
 	int			sum_cycles;
 };
@@ -164,6 +166,8 @@ static void osnoise_top_header(struct osnoise_tool *top)
 		goto eol;
 
 	trace_seq_printf(s, "          IRQ      Softirq       Thread");
+	if (params->common.ipi)
+		trace_seq_printf(s, "          IPI");
 
 eol:
 	if (pretty)
@@ -218,7 +222,10 @@ static void osnoise_top_print(struct osnoise_tool *tool, int cpu)
 
 	trace_seq_printf(s, "%12llu ", cpu_data->irq_count);
 	trace_seq_printf(s, "%12llu ", cpu_data->softirq_count);
-	trace_seq_printf(s, "%12llu\n", cpu_data->thread_count);
+	trace_seq_printf(s, "%12llu", cpu_data->thread_count);
+	if (params->common.ipi)
+		trace_seq_printf(s, " %12llu", cpu_data->ipi_count);
+	trace_seq_printf(s, "\n");
 }
 
 /*
@@ -275,12 +282,93 @@ osnoise_top_apply_config(struct osnoise_tool *tool)
 	return -1;
 }
 
+static void account_ipi(struct osnoise_tool *tool, unsigned long long dst_cpu)
+{
+	struct osnoise_top_cpu *cpu_data;
+	struct osnoise_top_data *data;
+	unsigned long long inc = 1;
+
+	data = tool->data;
+	cpu_data = &data->cpu_data[dst_cpu];
+
+	update_sum(&cpu_data->ipi_count, &inc);
+}
+
+/*
+ * osnoise_ipi_cpu_handler - this is the handler for single CPU IPI events.
+ */
+static int
+osnoise_ipi_cpu_handler(struct trace_seq *s, struct tep_record *record,
+		     struct tep_event *event, void *context)
+{
+	struct osnoise_tool *tool;
+	struct osnoise_params *params;
+	unsigned long long dst_cpu;
+	struct trace_instance *trace = context;
+
+	tool = container_of(trace, struct osnoise_tool, trace);
+	params = to_osnoise_params(tool->params);
+
+	tep_get_field_val(s, event, "cpu", record, &dst_cpu, 1);
+
+	if (CPU_ISSET(dst_cpu, &params->common.monitored_cpus))
+		account_ipi(tool, dst_cpu);
+
+	return 0;
+}
+
+static cpu_set_t cpumask_tmp_cpus;
+
+/*
+ * osnoise_ipi_cpumask_handler - this is the handler for broadcasted IPI events.
+ */
+static int
+osnoise_ipi_cpumask_handler(struct trace_seq *s, struct tep_record *record,
+			 struct tep_event *event, void *context)
+{
+	struct trace_instance *trace = context;
+	struct osnoise_tool *tool;
+	struct osnoise_params *params;
+	struct tep_format_field *field;
+	cpu_set_t *event_cpus;
+	int len;
+
+	tool = container_of(trace, struct osnoise_tool, trace);
+	params = to_osnoise_params(tool->params);
+
+	field = tep_find_field(event, "cpumask");
+	if (!field)
+		return 0;
+
+	event_cpus = tep_get_field_raw(s, event, "cpumask", record, &len, 1);
+	if (!event_cpus) {
+		err_msg("Failed to get cpumask field\n");
+		return 0;
+	}
+
+	CPU_AND(&cpumask_tmp_cpus, event_cpus, &params->common.monitored_cpus);
+
+	/*
+	 * Computing the mask weight is overkill but there is no leaner option
+	 * provided by glibc, e.g cpumask_first() or somesuch.
+	 */
+	if (CPU_COUNT(&cpumask_tmp_cpus)) {
+		for (int cpu = 0; cpu < nr_cpus; cpu++) {
+			if (CPU_ISSET(cpu, &cpumask_tmp_cpus))
+				account_ipi(tool, cpu);
+		}
+	}
+
+	return 0;
+}
+
 /*
  * osnoise_init_top - initialize a osnoise top tool with parameters
  */
 struct osnoise_tool *osnoise_init_top(struct common_params *params)
 {
 	struct osnoise_tool *tool;
+	int retval;
 
 	tool = osnoise_init_tool("osnoise_top");
 	if (!tool)
@@ -295,7 +383,33 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params)
 	tep_register_event_handler(tool->trace.tep, -1, "ftrace", "osnoise",
 				   osnoise_top_handler, NULL);
 
+	if (!params->ipi)
+		goto out;
+
+	retval = tracefs_event_enable(tool->trace.inst, "ipi", "ipi_send_cpu");
+	if (retval < 0 && !errno) {
+		err_msg("Could not find ipi_send_cpu event\n");
+		goto out_err;
+	}
+
+	retval = tracefs_event_enable(tool->trace.inst, "ipi", "ipi_send_cpumask");
+	if (retval < 0 && !errno) {
+		err_msg("Could not find ipi_send_cpumask event\n");
+		goto out_err;
+	}
+
+	tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpu",
+				   osnoise_ipi_cpu_handler, NULL);
+
+	tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpumask",
+				   osnoise_ipi_cpumask_handler, NULL);
+
+out:
 	return tool;
+out_err:
+	osnoise_free_top_tool(tool);
+	osnoise_destroy_tool(tool);
+	return NULL;
 }
 
 struct tool_ops osnoise_top_ops = {
-- 
2.55.0

[PATCH v4 3/6] rtla/osnoise: Leverage IPI event filters when tracing a subset of CPUs

[Valentin Schneider]

Leverage the kernel event filtering infrastructure to only emit IPI events
if they target CPUs that are being traced, as specified by the -c cmdline
option.

Note that some post-processing is still required for the ipi_send_cpumask
event, as the event being emitted means *some* CPUs targeted by that event
are monitored, but not all of them - userspace has to recompute that
intersection.

Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
 tools/tracing/rtla/src/osnoise_top.c | 38 ++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c
index 7b5ae5336cf08..353b435fefcf8 100644
--- a/tools/tracing/rtla/src/osnoise_top.c
+++ b/tools/tracing/rtla/src/osnoise_top.c
@@ -302,17 +302,13 @@ osnoise_ipi_cpu_handler(struct trace_seq *s, struct tep_record *record,
 		     struct tep_event *event, void *context)
 {
 	struct osnoise_tool *tool;
-	struct osnoise_params *params;
 	unsigned long long dst_cpu;
 	struct trace_instance *trace = context;
 
 	tool = container_of(trace, struct osnoise_tool, trace);
-	params = to_osnoise_params(tool->params);
-
 	tep_get_field_val(s, event, "cpu", record, &dst_cpu, 1);
 
-	if (CPU_ISSET(dst_cpu, &params->common.monitored_cpus))
-		account_ipi(tool, dst_cpu);
+	account_ipi(tool, dst_cpu);
 
 	return 0;
 }
@@ -346,6 +342,11 @@ osnoise_ipi_cpumask_handler(struct trace_seq *s, struct tep_record *record,
 		return 0;
 	}
 
+	/*
+	 * Despite already filtering for such an intersection, we need to compute
+	 * the intersection here as the @cpumask field may contain non-monitored
+	 * CPUs.
+	 */
 	CPU_AND(&cpumask_tmp_cpus, event_cpus, &params->common.monitored_cpus);
 
 	/*
@@ -398,6 +399,33 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params)
 		goto out_err;
 	}
 
+	/*
+	 * If tracing on a subset of possible CPUs, leverage the kernel filtering
+	 * infrastructure to only generate events on traced CPUs.
+	 */
+	if (params->cpus) {
+		char filter[MAX_PATH];
+
+		snprintf(filter, ARRAY_SIZE(filter), "cpu & CPUS{%s}\n", params->cpus);
+		retval = tracefs_event_file_write(tool->trace.inst,
+						  "ipi", "ipi_send_cpu", "filter",
+						  filter);
+		if (retval < 0) {
+			err_msg("Could not set ipi_send_cpu CPU filter\n");
+			goto out_err;
+		}
+
+
+		snprintf(filter, ARRAY_SIZE(filter), "cpumask & CPUS{%s}\n", params->cpus);
+		retval = tracefs_event_file_write(tool->trace.inst,
+						  "ipi", "ipi_send_cpumask", "filter",
+						  filter);
+		if (retval < 0) {
+			err_msg("Could not set ipi_send_cpumask CPU filter\n");
+			goto out_err;
+		}
+	}
+
 	tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpu",
 				   osnoise_ipi_cpu_handler, NULL);
 
-- 
2.55.0

[PATCH v4 4/6] rtla/osnoise: Allow IPI filters to gracefully fail

[Valentin Schneider]

Kernels pre v6.6 won't have:

  39f7c41c908b ("tracing/filters: Enable filtering a cpumask field by another cpumask")

and thus won't be able to filter events using a user-provided cpumask, but
will still be capable of recording IPI events.

Make failing to set a filter for IPI events an acceptable error and fall
back to event handlers that do the filtering job themselves.

Suggested-by: Tomas Glozar <tglozar@redhat.com>
Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
 tools/tracing/rtla/src/osnoise_top.c | 49 +++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 4 deletions(-)

diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c
index 353b435fefcf8..afab2f341a1e9 100644
--- a/tools/tracing/rtla/src/osnoise_top.c
+++ b/tools/tracing/rtla/src/osnoise_top.c
@@ -313,6 +313,30 @@ osnoise_ipi_cpu_handler(struct trace_seq *s, struct tep_record *record,
 	return 0;
 }
 
+/*
+ * osnoise_ipi_cpu_unfiltered_handler - this is the handler for single CPU IPI
+ *                                      events. Slightly less optimized than
+ *                                      the filtered variant.
+ */
+static int
+osnoise_ipi_cpu_unfiltered_handler(struct trace_seq *s, struct tep_record *record,
+		     struct tep_event *event, void *context)
+{
+	struct osnoise_tool *tool;
+	unsigned long long dst_cpu;
+	struct osnoise_params *params;
+	struct trace_instance *trace = context;
+
+	tool = container_of(trace, struct osnoise_tool, trace);
+	params = to_osnoise_params(tool->params);
+	tep_get_field_val(s, event, "cpu", record, &dst_cpu, 1);
+
+	if (CPU_ISSET(dst_cpu, &params->common.monitored_cpus))
+		account_ipi(tool, dst_cpu);
+
+	return 0;
+}
+
 static cpu_set_t cpumask_tmp_cpus;
 
 /*
@@ -343,7 +367,7 @@ osnoise_ipi_cpumask_handler(struct trace_seq *s, struct tep_record *record,
 	}
 
 	/*
-	 * Despite already filtering for such an intersection, we need to compute
+	 * Even if already filtering for such an intersection, we need to compute
 	 * the intersection here as the @cpumask field may contain non-monitored
 	 * CPUs.
 	 */
@@ -368,6 +392,7 @@ osnoise_ipi_cpumask_handler(struct trace_seq *s, struct tep_record *record,
  */
 struct osnoise_tool *osnoise_init_top(struct common_params *params)
 {
+	bool ipi_filters_enabled = false;
 	struct osnoise_tool *tool;
 	int retval;
 
@@ -402,6 +427,8 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params)
 	/*
 	 * If tracing on a subset of possible CPUs, leverage the kernel filtering
 	 * infrastructure to only generate events on traced CPUs.
+	 * Older kernels (pre v6.6) may have the IPI events but not the ability
+	 * to filter them, so allow that to fail gracefully.
 	 */
 	if (params->cpus) {
 		char filter[MAX_PATH];
@@ -411,8 +438,8 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params)
 						  "ipi", "ipi_send_cpu", "filter",
 						  filter);
 		if (retval < 0) {
-			err_msg("Could not set ipi_send_cpu CPU filter\n");
-			goto out_err;
+			debug_msg("Could not set ipi_send_cpu CPU filter\n");
+			goto no_filter;
 		}
 
 
@@ -421,13 +448,27 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params)
 						  "ipi", "ipi_send_cpumask", "filter",
 						  filter);
 		if (retval < 0) {
+			/*
+			 * If we managed to set up the previous filter but not
+			 * this one, something's really wrong
+			 */
 			err_msg("Could not set ipi_send_cpumask CPU filter\n");
 			goto out_err;
 		}
+
+		ipi_filters_enabled = true;
 	}
+no_filter:
 
+	/*
+	 * If no filtering is available and we're tracing all CPUs, we can still
+	 * use the filtered callback since stats are collected for all CPUs.
+	 */
 	tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpu",
-				   osnoise_ipi_cpu_handler, NULL);
+				   (!params->cpus || ipi_filters_enabled) ?
+				   osnoise_ipi_cpu_handler :
+				   osnoise_ipi_cpu_unfiltered_handler,
+				   NULL);
 
 	tep_register_event_handler(tool->trace.tep, -1, "ipi", "ipi_send_cpumask",
 				   osnoise_ipi_cpumask_handler, NULL);
-- 
2.55.0

[PATCH v4 5/6] rtla: Unconditionally clean any pre-existing filters for user-provided events

[Valentin Schneider]

A later commit will apply a filter to events recorded to the trace
output. To prevent any user confusion, remove pre-existing filters when
enabling an event provided via the '-e' command line argument.

Suggested-by: Tomas Glozar <tglozar@redhat.com>
Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
 tools/tracing/rtla/src/trace.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/tracing/rtla/src/trace.c b/tools/tracing/rtla/src/trace.c
index e407447773d04..f67f02cc92cdd 100644
--- a/tools/tracing/rtla/src/trace.c
+++ b/tools/tracing/rtla/src/trace.c
@@ -473,6 +473,10 @@ static int trace_event_enable_filter(struct trace_instance *instance,
 	char filter[MAX_PATH];
 	int retval;
 
+	/* Unconditionally clean any pre-existing filters */
+	tracefs_event_file_write(instance->inst, tevent->system,
+				 tevent->event, "filter", "0");
+
 	if (!tevent->filter)
 		return 0;
 
-- 
2.55.0

Re: [PATCH v4 5/6] rtla: Unconditionally clean any pre-existing filters for user-provided events

[Tomas Glozar]

út 4. 8. 2026 v 19:43 odesílatel Valentin Schneider
<vschneid@redhat.com> napsal:
>
> A later commit will apply a filter to events recorded to the trace
> output. To prevent any user confusion, remove pre-existing filters when
> enabling an event provided via the '-e' command line argument.
>
> Suggested-by: Tomas Glozar <tglozar@redhat.com>
> Signed-off-by: Valentin Schneider <vschneid@redhat.com>
> ---

I found that I missed one case when suggesting this: the user might
supply an event twice. With this change, it will now clear the filter
the second time the event is applied:

$ rtla timerlat hist --no-aa --on-threshold trace \
    --on-threshold shell,command="grep sched_switch timerlat_trace.txt
| grep -Fv '[000]' | head -n1" \
    -i 1 -e sched:sched_switch -e sched:sched_switch --filter "cpu == 0"

Without this commit:

```
 Saving trace to timerlat_trace.txt
# RTLA timerlat histogram
...
```

With this commit:

```
 Saving trace to timerlat_trace.txt
         <idle>-0       [002] d..2. 423185.347008: sched_switch: ...
# RTLA timerlat histogram
...
```

(Note that events are processed in opposite order to the command line.)

This is unexpected and might break scripts that for some reason enable
an event twice. So I'm not sure if my suggestion was the best
solution.

> [truncated]

Tomas

Re: [PATCH v4 5/6] rtla: Unconditionally clean any pre-existing filters for user-provided events

[Valentin Schneider]

On 10/08/26 13:24, Tomas Glozar wrote:
> út 4. 8. 2026 v 19:43 odesílatel Valentin Schneider
> <vschneid@redhat.com> napsal:
>>
>> A later commit will apply a filter to events recorded to the trace
>> output. To prevent any user confusion, remove pre-existing filters when
>> enabling an event provided via the '-e' command line argument.
>>
>> Suggested-by: Tomas Glozar <tglozar@redhat.com>
>> Signed-off-by: Valentin Schneider <vschneid@redhat.com>
>> ---
>
> I found that I missed one case when suggesting this: the user might
> supply an event twice. With this change, it will now clear the filter
> the second time the event is applied:
>
> $ rtla timerlat hist --no-aa --on-threshold trace \
>     --on-threshold shell,command="grep sched_switch timerlat_trace.txt
> | grep -Fv '[000]' | head -n1" \
>     -i 1 -e sched:sched_switch -e sched:sched_switch --filter "cpu == 0"
>
> Without this commit:
>
> ```
>  Saving trace to timerlat_trace.txt
> # RTLA timerlat histogram
> ...
> ```
>
> With this commit:
>
> ```
>  Saving trace to timerlat_trace.txt
>          <idle>-0       [002] d..2. 423185.347008: sched_switch: ...
> # RTLA timerlat histogram
> ...
> ```
>
> (Note that events are processed in opposite order to the command line.)
>
> This is unexpected and might break scripts that for some reason enable
> an event twice. So I'm not sure if my suggestion was the best
> solution.
>

Hm, didn't think of that.

I would say having the last defined event+filter override any previous
filter would make the most sense.

trace-cmd does this partially:

bash-5.3# trace-cmd record -e sched_switch -f 'CPU==0' -e sched_switch -- bash -c 'ls &>/dev/null'
CPU0 data recorded at offset=0x180000
    109 bytes in size (8192 uncompressed)
CPU1 data recorded at offset=0x181000
    0 bytes in size (0 uncompressed)
CPU2 data recorded at offset=0x181000
    0 bytes in size (0 uncompressed)
CPU3 data recorded at offset=0x181000
    0 bytes in size (0 uncompressed)

bash-5.3# trace-cmd record -e sched_switch -f 'CPU==0' -e sched_switch -f 'CPU==1' -- bash -c 'ls &>/dev/null'
CPU0 data recorded at offset=0x180000
    0 bytes in size (0 uncompressed)
CPU1 data recorded at offset=0x180000
    1428 bytes in size (237568 uncompressed)
CPU2 data recorded at offset=0x181000
    0 bytes in size (0 uncompressed)
CPU3 data recorded at offset=0x181000
    0 bytes in size (0 uncompressed)

Although I didn't realize that events were handled in reverse cmdline input
order until you pointed it out.

AIUI filers and triggers rely on the LIFO ordering to grab the
last-provided event; making the events list double-linked (but not
circular) would let us process them in FIFO order; something like the
barely tested:

---
diff --git a/tools/tracing/rtla/src/cli_p.h b/tools/tracing/rtla/src/cli_p.h
index 3c939de9abf02..4638cc317ea26 100644
--- a/tools/tracing/rtla/src/cli_p.h
+++ b/tools/tracing/rtla/src/cli_p.h
@@ -221,8 +221,10 @@ static int opt_event_cb(const struct option *opt, const char *arg, int unset)
 	if (!tevent)
 		fatal("Error alloc trace event");
 
-	if (*events)
+	if (*events) {
 		tevent->next = *events;
+		(*events)->prev = tevent;
+	}
 	*events = tevent;
 
 	return 0;
diff --git a/tools/tracing/rtla/src/trace.c b/tools/tracing/rtla/src/trace.c
index e407447773d04..35601a2e8d0c7 100644
--- a/tools/tracing/rtla/src/trace.c
+++ b/tools/tracing/rtla/src/trace.c
@@ -440,6 +440,18 @@ static void trace_event_disable_trigger(struct trace_instance *instance,
 			tevent->event ? : "*", tevent->trigger);
 }
 
+static inline struct trace_events *trace_events_tail(struct trace_events *tevent)
+{
+	while (tevent && tevent->next)
+		tevent = tevent->next;
+
+	return tevent;
+}
+
+/* Events are stashed in LIFO order; flip that to FIFO processing */
+#define for_each_trace_event(tevent) \
+	for (tevent = trace_events_tail(tevent); tevent; tevent = tevent->prev)
+
 /*
  * trace_events_disable - disable all trace events
  */
@@ -451,7 +463,7 @@ void trace_events_disable(struct trace_instance *instance,
 	if (!events)
 		return;
 
-	while (tevent) {
+	for_each_trace_event(tevent) {
 		debug_msg("Disabling event %s:%s\n", tevent->system, tevent->event ? : "*");
 		if (tevent->enabled) {
 			trace_event_disable_filter(instance, tevent);
@@ -460,7 +472,6 @@ void trace_events_disable(struct trace_instance *instance,
 		}
 
 		tevent->enabled = 0;
-		tevent = tevent->next;
 	}
 }
 
@@ -544,7 +555,10 @@ int trace_events_enable(struct trace_instance *instance,
 	struct trace_events *tevent = events;
 	int retval;
 
-	while (tevent) {
+	if (!events)
+		return 0;
+
+	for_each_trace_event(tevent) {
 		debug_msg("Enabling event %s:%s\n", tevent->system, tevent->event ? : "*");
 		retval = tracefs_event_enable(instance->inst, tevent->system, tevent->event);
 		if (retval < 0) {
@@ -562,7 +576,6 @@ int trace_events_enable(struct trace_instance *instance,
 			return 1;
 
 		tevent->enabled = 1;
-		tevent = tevent->next;
 	}
 
 	return 0;
diff --git a/tools/tracing/rtla/src/trace.h b/tools/tracing/rtla/src/trace.h
index 95b911a2228b2..eacafc0c96b31 100644
--- a/tools/tracing/rtla/src/trace.h
+++ b/tools/tracing/rtla/src/trace.h
@@ -4,6 +4,7 @@
 
 struct trace_events {
 	struct trace_events *next;
+	struct trace_events *prev;
 	char *system;
 	char *event;
 	char *filter;

[PATCH v4 6/6] rtla/osnoise: Trace IPI events when recording a trace file

[Valentin Schneider]

IPIs can now be monitored and accounted by osnoise top. When that is
the case, also record them when saving a trace file.

To match what is being recorded by the tool for its own analysis, event
filters are applied to the events recorded to the trace output.

Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
 tools/tracing/rtla/src/common.c      |  2 +-
 tools/tracing/rtla/src/common.h      |  2 +-
 tools/tracing/rtla/src/osnoise.c     | 72 +++++++++++++++++++++++++++-
 tools/tracing/rtla/src/osnoise.h     |  4 ++
 tools/tracing/rtla/src/osnoise_top.c | 36 ++------------
 5 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/tools/tracing/rtla/src/common.c b/tools/tracing/rtla/src/common.c
index 8c7f5e75b2ec8..2f6cf83475550 100644
--- a/tools/tracing/rtla/src/common.c
+++ b/tools/tracing/rtla/src/common.c
@@ -205,7 +205,7 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[])
 
 	if (params->threshold_actions.present[ACTION_TRACE_OUTPUT] ||
 	    params->end_actions.present[ACTION_TRACE_OUTPUT]) {
-		tool->record = osnoise_init_trace_tool(ops->tracer);
+		tool->record = osnoise_init_trace_tool(params, ops->tracer);
 		if (!tool->record) {
 			err_msg("Failed to enable the trace instance\n");
 			goto out_free;
diff --git a/tools/tracing/rtla/src/common.h b/tools/tracing/rtla/src/common.h
index 045253230fcf2..421e06e10f3f1 100644
--- a/tools/tracing/rtla/src/common.h
+++ b/tools/tracing/rtla/src/common.h
@@ -178,7 +178,7 @@ int osnoise_set_workload(struct osnoise_context *context, bool onoff);
 
 void osnoise_destroy_tool(struct osnoise_tool *top);
 struct osnoise_tool *osnoise_init_tool(char *tool_name);
-struct osnoise_tool *osnoise_init_trace_tool(const char *tracer);
+struct osnoise_tool *osnoise_init_trace_tool(struct common_params *params, const char *tracer);
 bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record);
 int osnoise_set_stop_us(struct osnoise_context *context, long long stop_us);
 int osnoise_set_stop_total_us(struct osnoise_context *context,
diff --git a/tools/tracing/rtla/src/osnoise.c b/tools/tracing/rtla/src/osnoise.c
index 4ff5dad013b10..ae6e5f03e828f 100644
--- a/tools/tracing/rtla/src/osnoise.c
+++ b/tools/tracing/rtla/src/osnoise.c
@@ -1178,10 +1178,56 @@ struct osnoise_tool *osnoise_init_tool(char *tool_name)
 	return top;
 }
 
+/*
+ * osnoise_init_ipi_filters - Initialize event filtering for IPI events
+ */
+int osnoise_init_ipi_filters(struct osnoise_tool *tool,
+				    struct common_params *params,
+				    bool *filters_enabled)
+{
+	char filter[MAX_PATH];
+	int retval;
+	/*
+	 * If tracing on a subset of possible CPUs, leverage the kernel filtering
+	 * infrastructure to only generate events on traced CPUs.
+	 * Older kernels (pre v6.6) may have the IPI events but not the ability
+	 * to filter them, so allow that to fail gracefully.
+	 */
+
+	snprintf(filter, ARRAY_SIZE(filter), "cpu & CPUS{%s}\n", params->cpus);
+	retval = tracefs_event_file_write(tool->trace.inst,
+					  "ipi", "ipi_send_cpu", "filter",
+					  filter);
+	if (retval < 0) {
+		debug_msg("Could not set ipi_send_cpu CPU filter\n");
+		*filters_enabled = false;
+		return 0;
+	}
+
+
+	snprintf(filter, ARRAY_SIZE(filter), "cpumask & CPUS{%s}\n", params->cpus);
+	retval = tracefs_event_file_write(tool->trace.inst,
+					  "ipi", "ipi_send_cpumask", "filter",
+					  filter);
+	if (retval < 0) {
+		/*
+		 * If we managed to set up the previous filter but not
+		 * this one, something's really wrong
+		 */
+		err_msg("Could not set ipi_send_cpumask CPU filter\n");
+		*filters_enabled = false;
+		return -1;
+	}
+
+	*filters_enabled = true;
+	return 0;
+}
+
 /*
  * osnoise_init_trace_tool - init a tracer instance to trace osnoise events
  */
-struct osnoise_tool *osnoise_init_trace_tool(const char *tracer)
+struct osnoise_tool *osnoise_init_trace_tool(struct common_params *params,
+					     const char *tracer)
 {
 	struct osnoise_tool *trace;
 	int retval;
@@ -1196,6 +1242,30 @@ struct osnoise_tool *osnoise_init_trace_tool(const char *tracer)
 		goto out_err;
 	}
 
+	if (!params->ipi)
+		goto done;
+
+	retval = tracefs_event_enable(trace->trace.inst, "ipi", "ipi_send_cpu");
+	if (retval < 0 && !errno) {
+		err_msg("Could not find ipi_send_cpu event\n");
+		goto out_err;
+	}
+
+	retval = tracefs_event_enable(trace->trace.inst, "ipi", "ipi_send_cpumask");
+	if (retval < 0 && !errno) {
+		err_msg("Could not find ipi_send_cpumask event\n");
+		goto out_err;
+	}
+
+	if (params->cpus) {
+		bool unused;
+
+		retval = osnoise_init_ipi_filters(trace, params, &unused);
+		if (retval < 0)
+			goto out_err;
+	}
+
+done:
 	retval = enable_tracer_by_name(trace->trace.inst, tracer);
 	if (retval) {
 		err_msg("Could not enable %s tracer for tracing\n", tracer);
diff --git a/tools/tracing/rtla/src/osnoise.h b/tools/tracing/rtla/src/osnoise.h
index 340ff5a64e6e4..81a704c361ec0 100644
--- a/tools/tracing/rtla/src/osnoise.h
+++ b/tools/tracing/rtla/src/osnoise.h
@@ -63,6 +63,10 @@ int osnoise_enable(struct osnoise_tool *tool);
 int osnoise_main(int argc, char **argv);
 int hwnoise_main(int argc, char **argv);
 
+int osnoise_init_ipi_filters(struct osnoise_tool *tool,
+			     struct common_params *params,
+			     bool *filters_enabled);
+
 extern struct tool_ops timerlat_top_ops, timerlat_hist_ops;
 extern struct tool_ops osnoise_top_ops, osnoise_hist_ops;
 
diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c
index afab2f341a1e9..87d28865515b5 100644
--- a/tools/tracing/rtla/src/osnoise_top.c
+++ b/tools/tracing/rtla/src/osnoise_top.c
@@ -392,7 +392,7 @@ osnoise_ipi_cpumask_handler(struct trace_seq *s, struct tep_record *record,
  */
 struct osnoise_tool *osnoise_init_top(struct common_params *params)
 {
-	bool ipi_filters_enabled = false;
+	bool ipi_filters_enabled;
 	struct osnoise_tool *tool;
 	int retval;
 
@@ -424,41 +424,11 @@ struct osnoise_tool *osnoise_init_top(struct common_params *params)
 		goto out_err;
 	}
 
-	/*
-	 * If tracing on a subset of possible CPUs, leverage the kernel filtering
-	 * infrastructure to only generate events on traced CPUs.
-	 * Older kernels (pre v6.6) may have the IPI events but not the ability
-	 * to filter them, so allow that to fail gracefully.
-	 */
 	if (params->cpus) {
-		char filter[MAX_PATH];
-
-		snprintf(filter, ARRAY_SIZE(filter), "cpu & CPUS{%s}\n", params->cpus);
-		retval = tracefs_event_file_write(tool->trace.inst,
-						  "ipi", "ipi_send_cpu", "filter",
-						  filter);
-		if (retval < 0) {
-			debug_msg("Could not set ipi_send_cpu CPU filter\n");
-			goto no_filter;
-		}
-
-
-		snprintf(filter, ARRAY_SIZE(filter), "cpumask & CPUS{%s}\n", params->cpus);
-		retval = tracefs_event_file_write(tool->trace.inst,
-						  "ipi", "ipi_send_cpumask", "filter",
-						  filter);
-		if (retval < 0) {
-			/*
-			 * If we managed to set up the previous filter but not
-			 * this one, something's really wrong
-			 */
-			err_msg("Could not set ipi_send_cpumask CPU filter\n");
+		retval = osnoise_init_ipi_filters(tool, params, &ipi_filters_enabled);
+		if (retval < 0)
 			goto out_err;
-		}
-
-		ipi_filters_enabled = true;
 	}
-no_filter:
 
 	/*
 	 * If no filtering is available and we're tracing all CPUs, we can still
-- 
2.55.0