From: Steven Rostedt <rostedt@goodmis.org>
To: linux-kernel@vger.kernel.org
Cc: Masami Hiramatsu <mhiramat@kernel.org>,
Mark Rutland <mark.rutland@arm.com>,
Mathieu Desnoyers <mathieu.desnoyers@efficios.com>,
Andrew Morton <akpm@linux-foundation.org>
Subject: [for-next][PATCH 23/31] tracing: Convert the per CPU "disabled" counter to local from atomic
Date: Fri, 09 May 2025 09:13:12 -0400 [thread overview]
Message-ID: <20250509131317.337063144@goodmis.org> (raw)
In-Reply-To: 20250509131249.340302366@goodmis.org
From: Steven Rostedt <rostedt@goodmis.org>
The per CPU "disabled" counter is used for the latency tracers and stack
tracers to make sure that their accounting isn't messed up by an NMI or
interrupt coming in and affecting the same CPU data. But the counter is an
atomic_t type. As it only needs to synchronize against the current CPU,
switch it over to local_t type.
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Link: https://lore.kernel.org/20250505212236.394925376@goodmis.org
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
kernel/trace/trace.h | 2 +-
kernel/trace/trace_functions.c | 8 ++++----
kernel/trace/trace_irqsoff.c | 22 +++++++++++-----------
kernel/trace/trace_sched_wakeup.c | 18 +++++++++---------
4 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 69c1ecfb2290..188032d4ab69 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -183,7 +183,7 @@ struct trace_array;
* the trace, etc.)
*/
struct trace_array_cpu {
- atomic_t disabled;
+ local_t disabled;
void *buffer_page; /* ring buffer spare */
unsigned long entries;
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index bd153219a712..99a90f182485 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -291,7 +291,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
parent_ip = function_get_true_parent_ip(parent_ip, fregs);
cpu = raw_smp_processor_id();
data = per_cpu_ptr(tr->array_buffer.data, cpu);
- disabled = atomic_inc_return(&data->disabled);
+ disabled = local_inc_return(&data->disabled);
if (likely(disabled == 1)) {
trace_ctx = tracing_gen_ctx_flags(flags);
@@ -303,7 +303,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
__trace_stack(tr, trace_ctx, skip);
}
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
local_irq_restore(flags);
}
@@ -402,7 +402,7 @@ function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
parent_ip = function_get_true_parent_ip(parent_ip, fregs);
cpu = raw_smp_processor_id();
data = per_cpu_ptr(tr->array_buffer.data, cpu);
- disabled = atomic_inc_return(&data->disabled);
+ disabled = local_inc_return(&data->disabled);
if (likely(disabled == 1)) {
last_info = per_cpu_ptr(tr->last_func_repeats, cpu);
@@ -417,7 +417,7 @@ function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
}
out:
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
local_irq_restore(flags);
}
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 40c39e946940..0b6d932a931e 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -123,12 +123,12 @@ static int func_prolog_dec(struct trace_array *tr,
return 0;
*data = per_cpu_ptr(tr->array_buffer.data, cpu);
- disabled = atomic_inc_return(&(*data)->disabled);
+ disabled = local_inc_return(&(*data)->disabled);
if (likely(disabled == 1))
return 1;
- atomic_dec(&(*data)->disabled);
+ local_dec(&(*data)->disabled);
return 0;
}
@@ -152,7 +152,7 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip,
trace_function(tr, ip, parent_ip, trace_ctx, fregs);
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
}
#endif /* CONFIG_FUNCTION_TRACER */
@@ -209,7 +209,7 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace,
trace_ctx = tracing_gen_ctx_flags(flags);
ret = __trace_graph_entry(tr, trace, trace_ctx);
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
return ret;
}
@@ -238,7 +238,7 @@ static void irqsoff_graph_return(struct ftrace_graph_ret *trace,
trace_ctx = tracing_gen_ctx_flags(flags);
__trace_graph_return(tr, trace, trace_ctx, *calltime, rettime);
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
}
static struct fgraph_ops fgraph_ops = {
@@ -408,10 +408,10 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip)
data = per_cpu_ptr(tr->array_buffer.data, cpu);
- if (unlikely(!data) || atomic_read(&data->disabled))
+ if (unlikely(!data) || local_read(&data->disabled))
return;
- atomic_inc(&data->disabled);
+ local_inc(&data->disabled);
data->critical_sequence = max_sequence;
data->preempt_timestamp = ftrace_now(cpu);
@@ -421,7 +421,7 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip)
per_cpu(tracing_cpu, cpu) = 1;
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
}
static nokprobe_inline void
@@ -445,16 +445,16 @@ stop_critical_timing(unsigned long ip, unsigned long parent_ip)
data = per_cpu_ptr(tr->array_buffer.data, cpu);
if (unlikely(!data) ||
- !data->critical_start || atomic_read(&data->disabled))
+ !data->critical_start || local_read(&data->disabled))
return;
- atomic_inc(&data->disabled);
+ local_inc(&data->disabled);
trace_ctx = tracing_gen_ctx();
__trace_function(tr, ip, parent_ip, trace_ctx);
check_critical_timing(tr, data, parent_ip ? : ip, cpu);
data->critical_start = 0;
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
}
/* start and stop critical timings used to for stoppage (in idle) */
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index a0db3404f7f7..bf1cb80742ae 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -83,14 +83,14 @@ func_prolog_preempt_disable(struct trace_array *tr,
goto out_enable;
*data = per_cpu_ptr(tr->array_buffer.data, cpu);
- disabled = atomic_inc_return(&(*data)->disabled);
+ disabled = local_inc_return(&(*data)->disabled);
if (unlikely(disabled != 1))
goto out;
return 1;
out:
- atomic_dec(&(*data)->disabled);
+ local_dec(&(*data)->disabled);
out_enable:
preempt_enable_notrace();
@@ -144,7 +144,7 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace,
*calltime = trace_clock_local();
ret = __trace_graph_entry(tr, trace, trace_ctx);
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
preempt_enable_notrace();
return ret;
@@ -173,7 +173,7 @@ static void wakeup_graph_return(struct ftrace_graph_ret *trace,
return;
__trace_graph_return(tr, trace, trace_ctx, *calltime, rettime);
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
preempt_enable_notrace();
return;
@@ -243,7 +243,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
trace_function(tr, ip, parent_ip, trace_ctx, fregs);
local_irq_restore(flags);
- atomic_dec(&data->disabled);
+ local_dec(&data->disabled);
preempt_enable_notrace();
}
@@ -471,7 +471,7 @@ probe_wakeup_sched_switch(void *ignore, bool preempt,
/* disable local data, not wakeup_cpu data */
cpu = raw_smp_processor_id();
- disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
+ disabled = local_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
if (likely(disabled != 1))
goto out;
@@ -508,7 +508,7 @@ probe_wakeup_sched_switch(void *ignore, bool preempt,
arch_spin_unlock(&wakeup_lock);
local_irq_restore(flags);
out:
- atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
+ local_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
}
static void __wakeup_reset(struct trace_array *tr)
@@ -563,7 +563,7 @@ probe_wakeup(void *ignore, struct task_struct *p)
(!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
return;
- disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
+ disabled = local_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
if (unlikely(disabled != 1))
goto out;
@@ -610,7 +610,7 @@ probe_wakeup(void *ignore, struct task_struct *p)
out_locked:
arch_spin_unlock(&wakeup_lock);
out:
- atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
+ local_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
}
static void start_wakeup_tracer(struct trace_array *tr)
--
2.47.2
next prev parent reply other threads:[~2025-05-09 13:13 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-05-09 13:12 [for-next][PATCH 00/31] tracing: Updates for v6.16 Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 01/31] tracing: Update function trace addresses with module addresses Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 02/31] tracing: Show function names when possible when listing fields Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 03/31] tracing: Only return an adjusted address if it matches the kernel address Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 04/31] tracing: Adjust addresses for printing out fields Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 05/31] tracing: Show preempt and irq events callsites from the offsets in field print Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 06/31] tracing: Always use memcpy() in histogram add_to_key() Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 07/31] tracing: Move histogram trigger variables from stack to per CPU structure Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 08/31] tracing: Add common_comm to histograms Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 09/31] ftrace: Show subops in enabled_functions Steven Rostedt
2025-05-09 13:12 ` [for-next][PATCH 10/31] ftrace: Expose call graph depth as unsigned int Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 11/31] ftrace: Comment that ftrace_func_mapper is freed with free_ftrace_hash() Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 12/31] tracing/osnoise: Allow arbitrarily long CPU string Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 13/31] tracing/mmiotrace: Remove reference to unused per CPU data pointer Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 14/31] ftrace: Do not bother checking per CPU "disabled" flag Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 15/31] tracing: Just use this_cpu_read() to access ignore_pid Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 16/31] tracing: Add tracer_tracing_disable/enable() functions Steven Rostedt
2025-05-09 15:49 ` Doug Anderson
2025-05-09 13:13 ` [for-next][PATCH 17/31] tracing: Use tracer_tracing_disable() instead of "disabled" field for ftrace_dump_one() Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 18/31] tracing: kdb: Use tracer_tracing_on/off() instead of setting per CPU disabled Steven Rostedt
2025-05-09 15:49 ` Doug Anderson
2025-05-09 15:57 ` Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 19/31] ftrace: Do not disabled function graph based on "disabled" field Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 20/31] tracing: Do not use per CPU array_buffer.data->disabled for cpumask Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 21/31] ring-buffer: Add ring_buffer_record_is_on_cpu() Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 22/31] tracing: branch: Use trace_tracing_is_on_cpu() instead of "disabled" field Steven Rostedt
2025-05-09 13:13 ` Steven Rostedt [this message]
2025-05-09 13:13 ` [for-next][PATCH 24/31] tracing: Use atomic_inc_return() for updating "disabled" counter in irqsoff tracer Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 25/31] tracing: Remove unused buffer_page field from trace_array_cpu structure Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 26/31] tracing: Replace deprecated strncpy() with strscpy() for stack_trace_filter_buf Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 27/31] tracing: Rename event_trigger_alloc() to trigger_data_alloc() Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 28/31] tracing: Fix error handling in event_trigger_parse() Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 29/31] tracing: Remove unnecessary "goto out" that simply returns ret is trigger code Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 30/31] tracing: Add a helper function to handle the dereference arg in verifier Steven Rostedt
2025-05-09 13:13 ` [for-next][PATCH 31/31] tracing: Allow the top level trace_marker to write into another instances Steven Rostedt
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250509131317.337063144@goodmis.org \
--to=rostedt@goodmis.org \
--cc=akpm@linux-foundation.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mathieu.desnoyers@efficios.com \
--cc=mhiramat@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.