From: Peter Zijlstra <a.p.zijlstra@chello.nl>
To: Ingo Molnar <mingo@elte.hu>, Paul Mackerras <paulus@samba.org>
Cc: linux-kernel@vger.kernel.org,
Peter Zijlstra <a.p.zijlstra@chello.nl>,
Frederic Weisbecker <fweisbec@gmail.com>
Subject: [PATCH 7/8] perf: undo some recursion damage
Date: Mon, 23 Nov 2009 11:37:29 +0100 [thread overview]
Message-ID: <20091123103819.993226816@chello.nl> (raw)
In-Reply-To: 20091123103722.140831906@chello.nl
[-- Attachment #1: perf-foo-7.patch --]
[-- Type: text/plain, Size: 10647 bytes --]
Make perf_swevent_get_recursion_context return a context number and
disable preemption.
This could be used to remove the IRQ disable from the trace bit and
index the per-cpu buffer with.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
---
include/linux/perf_event.h | 8 ++--
include/trace/ftrace.h | 17 +++++-----
kernel/perf_event.c | 71 ++++++++++++++++++------------------------
kernel/trace/trace_kprobe.c | 14 ++++----
kernel/trace/trace_syscalls.c | 14 ++++----
5 files changed, 61 insertions(+), 63 deletions(-)
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -874,8 +874,8 @@ extern int perf_output_begin(struct perf
extern void perf_output_end(struct perf_output_handle *handle);
extern void perf_output_copy(struct perf_output_handle *handle,
const void *buf, unsigned int len);
-extern int perf_swevent_get_recursion_context(int **recursion);
-extern void perf_swevent_put_recursion_context(int *recursion);
+extern int perf_swevent_get_recursion_context(void);
+extern void perf_swevent_put_recursion_context(int rctx);
#else
static inline void
perf_event_task_sched_in(struct task_struct *task, int cpu) { }
@@ -904,8 +904,8 @@ static inline void perf_event_mmap(struc
static inline void perf_event_comm(struct task_struct *tsk) { }
static inline void perf_event_fork(struct task_struct *tsk) { }
static inline void perf_event_init(void) { }
-static int perf_swevent_get_recursion_context(int **recursion) { return -1; }
-static void perf_swevent_put_recursion_context(int *recursion) { }
+static inline int perf_swevent_get_recursion_context(void) { return -1; }
+static inline void perf_swevent_put_recursion_context(int rctx) { }
#endif
Index: linux-2.6/include/trace/ftrace.h
===================================================================
--- linux-2.6.orig/include/trace/ftrace.h
+++ linux-2.6/include/trace/ftrace.h
@@ -724,8 +724,8 @@ __attribute__((section("_ftrace_events")
static void ftrace_profile_##call(proto) \
{ \
struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
- extern int perf_swevent_get_recursion_context(int **recursion); \
- extern void perf_swevent_put_recursion_context(int *recursion); \
+ extern int perf_swevent_get_recursion_context(void); \
+ extern void perf_swevent_put_recursion_context(int rctx); \
struct ftrace_event_call *event_call = &event_##call; \
extern void perf_tp_event(int, u64, u64, void *, int); \
struct ftrace_raw_##call *entry; \
@@ -736,8 +736,8 @@ static void ftrace_profile_##call(proto)
int __data_size; \
char *trace_buf; \
char *raw_data; \
- int *recursion; \
int __cpu; \
+ int rctx; \
int pc; \
\
pc = preempt_count(); \
@@ -753,8 +753,9 @@ static void ftrace_profile_##call(proto)
\
local_irq_save(irq_flags); \
\
- if (perf_swevent_get_recursion_context(&recursion)) \
- goto end_recursion; \
+ rctx = perf_swevent_get_recursion_context(); \
+ if (rctx < 0) \
+ goto end_recursion; \
\
__cpu = smp_processor_id(); \
\
@@ -781,9 +782,9 @@ static void ftrace_profile_##call(proto)
perf_tp_event(event_call->id, __addr, __count, entry, \
__entry_size); \
\
-end: \
- perf_swevent_put_recursion_context(recursion); \
-end_recursion: \
+end: \
+ perf_swevent_put_recursion_context(rctx); \
+end_recursion: \
local_irq_restore(irq_flags); \
\
}
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -3869,45 +3869,50 @@ static void perf_swevent_ctx_event(struc
}
}
-/*
- * Must be called with preemption disabled
- */
-int perf_swevent_get_recursion_context(int **recursion)
+int perf_swevent_get_recursion_context(void)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
+ int rctx;
if (in_nmi())
- *recursion = &cpuctx->recursion[3];
+ rctx = 3;
else if (in_irq())
- *recursion = &cpuctx->recursion[2];
+ rctx = 2;
else if (in_softirq())
- *recursion = &cpuctx->recursion[1];
+ rctx = 1;
else
- *recursion = &cpuctx->recursion[0];
+ rctx = 0;
- if (**recursion)
+ if (cpuctx->recursion[rctx]) {
+ put_cpu_var(perf_cpu_context);
return -1;
+ }
- (**recursion)++;
+ cpuctx->recursion[rctx]++;
+ barrier();
- return 0;
+ return rctx;
}
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
-void perf_swevent_put_recursion_context(int *recursion)
+void perf_swevent_put_recursion_context(int rctx)
{
- (*recursion)--;
+ struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ barrier();
+ cpuctx->recursion[rctx]++;
+ put_cpu_var(perf_cpu_context);
}
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
-static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
- u64 nr, int nmi,
- struct perf_sample_data *data,
- struct pt_regs *regs)
+static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
+ u64 nr, int nmi,
+ struct perf_sample_data *data,
+ struct pt_regs *regs)
{
+ struct perf_cpu_context *cpuctx;
struct perf_event_context *ctx;
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ cpuctx = &__get_cpu_var(perf_cpu_context);
rcu_read_lock();
perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
nr, nmi, data, regs);
@@ -3921,35 +3926,23 @@ static void __do_perf_sw_event(enum perf
rcu_read_unlock();
}
-static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
- u64 nr, int nmi,
- struct perf_sample_data *data,
- struct pt_regs *regs)
-{
- int *recursion;
-
- preempt_disable();
-
- if (perf_swevent_get_recursion_context(&recursion))
- goto out;
-
- __do_perf_sw_event(type, event_id, nr, nmi, data, regs);
-
- perf_swevent_put_recursion_context(recursion);
-out:
- preempt_enable();
-}
-
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
struct pt_regs *regs, u64 addr)
{
struct perf_sample_data data;
+ int rctx;
+
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
+ return;
data.addr = addr;
data.raw = NULL;
do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi,
&data, regs);
+
+ perf_swevent_put_recursion_context(rctx);
}
static void perf_swevent_read(struct perf_event *event)
@@ -4173,7 +4166,7 @@ void perf_tp_event(int event_id, u64 add
regs = task_pt_regs(current);
/* Trace events already protected against recursion */
- __do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
+ do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
&data, regs);
}
EXPORT_SYMBOL_GPL(perf_tp_event);
Index: linux-2.6/kernel/trace/trace_kprobe.c
===================================================================
--- linux-2.6.orig/kernel/trace/trace_kprobe.c
+++ linux-2.6/kernel/trace/trace_kprobe.c
@@ -1213,7 +1213,7 @@ static __kprobes int kprobe_profile_func
unsigned long irq_flags;
char *trace_buf;
char *raw_data;
- int *recursion;
+ int rctx;
pc = preempt_count();
__size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
@@ -1229,7 +1229,8 @@ static __kprobes int kprobe_profile_func
*/
local_irq_save(irq_flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
__cpu = smp_processor_id();
@@ -1258,7 +1259,7 @@ static __kprobes int kprobe_profile_func
perf_tp_event(call->id, entry->ip, 1, entry, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(irq_flags);
@@ -1276,8 +1277,8 @@ static __kprobes int kretprobe_profile_f
int size, __size, i, pc, __cpu;
unsigned long irq_flags;
char *trace_buf;
- int *recursion;
char *raw_data;
+ int rctx;
pc = preempt_count();
__size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
@@ -1293,7 +1294,8 @@ static __kprobes int kretprobe_profile_f
*/
local_irq_save(irq_flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
__cpu = smp_processor_id();
@@ -1323,7 +1325,7 @@ static __kprobes int kretprobe_profile_f
perf_tp_event(call->id, entry->ret_ip, 1, entry, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(irq_flags);
Index: linux-2.6/kernel/trace/trace_syscalls.c
===================================================================
--- linux-2.6.orig/kernel/trace/trace_syscalls.c
+++ linux-2.6/kernel/trace/trace_syscalls.c
@@ -481,8 +481,8 @@ static void prof_syscall_enter(struct pt
unsigned long flags;
char *trace_buf;
char *raw_data;
- int *recursion;
int syscall_nr;
+ int rctx;
int size;
int cpu;
@@ -506,7 +506,8 @@ static void prof_syscall_enter(struct pt
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
cpu = smp_processor_id();
@@ -530,7 +531,7 @@ static void prof_syscall_enter(struct pt
perf_tp_event(sys_data->enter_id, 0, 1, rec, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(flags);
}
@@ -582,7 +583,7 @@ static void prof_syscall_exit(struct pt_
int syscall_nr;
char *trace_buf;
char *raw_data;
- int *recursion;
+ int rctx;
int size;
int cpu;
@@ -609,7 +610,8 @@ static void prof_syscall_exit(struct pt_
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(flags);
- if (perf_swevent_get_recursion_context(&recursion))
+ rctx = perf_swevent_get_recursion_context();
+ if (rctx < 0)
goto end_recursion;
cpu = smp_processor_id();
@@ -634,7 +636,7 @@ static void prof_syscall_exit(struct pt_
perf_tp_event(sys_data->exit_id, 0, 1, rec, size);
end:
- perf_swevent_put_recursion_context(recursion);
+ perf_swevent_put_recursion_context(rctx);
end_recursion:
local_irq_restore(flags);
}
--
next prev parent reply other threads:[~2009-11-23 10:40 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-11-23 10:37 [PATCH 0/8] More perf patches Peter Zijlstra
2009-11-23 10:37 ` [PATCH 1/8] perf: undo copy/paste damage Peter Zijlstra
2009-11-23 11:53 ` [tip:perf/core] perf_events: Undo " tip-bot for Peter Zijlstra
2009-11-23 10:37 ` [PATCH 2/8] perf: style nits Peter Zijlstra
2009-11-23 11:54 ` [tip:perf/core] perf_events: Fix " tip-bot for Peter Zijlstra
2009-11-23 10:37 ` [PATCH 3/8] perf: disable events when we detach them Peter Zijlstra
2009-11-23 11:54 ` [tip:perf/core] perf_events: Disable " tip-bot for Peter Zijlstra
2009-11-23 10:37 ` [PATCH 4/8] perf: update the context time on exit Peter Zijlstra
2009-11-23 11:54 ` [tip:perf/core] perf_events: Update " tip-bot for Peter Zijlstra
2009-11-23 10:37 ` [PATCH 5/8] perf: fix __perf_event_exit_task update_event_times locking Peter Zijlstra
2009-11-23 11:54 ` [tip:perf/core] perf_events: Fix __perf_event_exit_task() vs. update_event_times() locking tip-bot for Peter Zijlstra
2009-11-23 10:37 ` [PATCH 6/8] perf: optimize __perf_sw_event() Peter Zijlstra
2009-11-23 10:37 ` Peter Zijlstra [this message]
2009-11-23 11:55 ` [tip:perf/core] perf_events: Undo some recursion damage tip-bot for Peter Zijlstra
2009-11-23 12:39 ` Frederic Weisbecker
2009-11-23 12:50 ` Peter Zijlstra
2009-11-23 10:37 ` [PATCH 8/8][RFC] perf: be paranoid about child times? Peter Zijlstra
2009-11-23 10:51 ` Ingo Molnar
2009-11-23 14:00 ` [PATCH 9/8] perf_events: Restore sanity to scaling land Peter Zijlstra
2009-11-23 17:42 ` [tip:perf/core] " tip-bot for Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20091123103819.993226816@chello.nl \
--to=a.p.zijlstra@chello.nl \
--cc=fweisbec@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@elte.hu \
--cc=paulus@samba.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.