[RFC PATCH 12/30 v3] separate out the percpu date into a percpu struct

Steven Rostedt
2008-01-15T20:59:24Z

[-- Attachment #1: mcount-tracer-percpu-struct.patch --]
[-- Type: text/plain, Size: 5263 bytes --]

For better cacheline performance, this patch creates a separate
struct for each CPU with the percpu data grouped together.

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
---
lib/tracing/tracer.c | 42 +++++++++++++++++++++++-------------------
lib/tracing/tracer.h | 12 ++++++++----
2 files changed, 31 insertions(+), 23 deletions(-)

Index: linux-compile.git/lib/tracing/tracer.c
===================================================================
--- linux-compile.git.orig/lib/tracing/tracer.c 2008-01-14 13:14:13.000000000 -0500
+++ linux-compile.git/lib/tracing/tracer.c 2008-01-14 14:57:43.000000000 -0500
@@ -16,6 +16,7 @@
#include <linux/module.h>
#include <linux/linkage.h>
#include <linux/seq_file.h>
+#include <linux/percpu.h>
#include <linux/debugfs.h>
#include <linux/kallsyms.h>
#include <linux/uaccess.h>
@@ -25,6 +26,7 @@
#include "tracer_interface.h"

static struct mctracer_trace mctracer_trace;
+static DEFINE_PER_CPU(struct mctracer_trace_cpu, mctracer_trace_cpu);
static int trace_enabled __read_mostly;

static inline notrace void
@@ -36,21 +38,22 @@ mctracer_add_trace_entry(struct mctracer
unsigned long idx, idx_next;
struct mctracer_entry *entry;
struct task_struct *tsk = current;
+ struct mctracer_trace_cpu *data = tr->data[cpu];

- idx = tr->trace_idx[cpu];
+ idx = data->trace_idx;
idx_next = idx + 1;

if (unlikely(idx_next >= tr->entries)) {
- atomic_inc(&tr->underrun[cpu]);
+ atomic_inc(&data->underrun);
idx_next = 0;
}

- tr->trace_idx[cpu] = idx_next;
+ data->trace_idx = idx_next;

- if (unlikely(idx_next != 0 && atomic_read(&tr->underrun[cpu])))
- atomic_inc(&tr->underrun[cpu]);
+ if (unlikely(idx_next != 0 && atomic_read(&data->underrun)))
+ atomic_inc(&data->underrun);

- entry = tr->trace[cpu] + idx * MCTRACER_ENTRY_SIZE;
+ entry = data->trace + idx * MCTRACER_ENTRY_SIZE;
entry->idx = atomic_inc_return(&tr->cnt);
entry->ip = ip;
entry->parent_ip = parent_ip;
@@ -73,11 +76,11 @@ static notrace void trace_function(const

cpu = raw_smp_processor_id();

- atomic_inc(&tr->disabled[cpu]);
- if (likely(atomic_read(&tr->disabled[cpu]) == 1))
+ atomic_inc(&tr->data[cpu]->disabled);
+ if (likely(atomic_read(&tr->data[cpu]->disabled) == 1))
mctracer_add_trace_entry(tr, cpu, ip, parent_ip);

- atomic_dec(&tr->disabled[cpu]);
+ atomic_dec(&tr->data[cpu]->disabled);

out:
raw_local_irq_restore(flags);
@@ -93,8 +96,8 @@ static notrace void mctracer_reset(struc
int cpu;

for_each_online_cpu(cpu) {
- tr->trace_idx[cpu] = 0;
- atomic_set(&tr->underrun[cpu], 0);
+ tr->data[cpu]->trace_idx = 0;
+ atomic_set(&tr->data[cpu]->underrun, 0);
}
}

@@ -115,16 +118,16 @@ static struct mctracer_entry *mctracer_e
unsigned long idx,
int cpu)
{
- struct mctracer_entry *array = tr->trace[cpu];
+ struct mctracer_entry *array = tr->data[cpu]->trace;
unsigned long underrun;

if (idx >= tr->entries)
return NULL;

- underrun = atomic_read(&tr->underrun[cpu]);
+ underrun = atomic_read(&tr->data[cpu]->underrun);
if (underrun)
idx = ((underrun - 1) + idx) % tr->entries;
- else if (idx >= tr->trace_idx[cpu])
+ else if (idx >= tr->data[cpu]->trace_idx)
return NULL;

return &array[idx];
@@ -139,7 +142,7 @@ static void *find_next_entry(struct mctr
int i;

for_each_possible_cpu(i) {
- if (!tr->trace[i])
+ if (!tr->data[i]->trace)
continue;
ent = mctracer_entry_idx(tr, iter->next_idx[i], i);
if (ent && (!next || next->idx > ent->idx)) {
@@ -461,6 +464,7 @@ static notrace int mctracer_alloc_buffer
int i;

for_each_possible_cpu(i) {
+ mctracer_trace.data[i] = &per_cpu(mctracer_trace_cpu, i);
array = (struct mctracer_entry *)
__get_free_pages(GFP_KERNEL, order);
if (array == NULL) {
@@ -468,7 +472,7 @@ static notrace int mctracer_alloc_buffer
" %ld bytes for trace buffer!\n", size);
goto free_buffers;
}
- mctracer_trace.trace[i] = array;
+ mctracer_trace.data[i]->trace = array;
}

/*
@@ -489,10 +493,10 @@ static notrace int mctracer_alloc_buffer

free_buffers:
for (i-- ; i >= 0; i--) {
- if (mctracer_trace.trace[i]) {
- free_pages((unsigned long)mctracer_trace.trace[i],
+ if (mctracer_trace.data[i] && mctracer_trace.data[i]->trace) {
+ free_pages((unsigned long)mctracer_trace.data[i]->trace,
order);
- mctracer_trace.trace[i] = NULL;
+ mctracer_trace.data[i]->trace = NULL;
}
}
return -ENOMEM;
Index: linux-compile.git/lib/tracing/tracer.h
===================================================================
--- linux-compile.git.orig/lib/tracing/tracer.h 2008-01-14 13:14:13.000000000 -0500
+++ linux-compile.git/lib/tracing/tracer.h 2008-01-14 14:57:43.000000000 -0500
@@ -12,15 +12,19 @@ struct mctracer_entry {
pid_t pid;
};

+struct mctracer_trace_cpu {
+ void *trace;
+ unsigned long trace_idx;
+ atomic_t disabled;
+ atomic_t underrun;
+};
+
struct mctracer_trace {
- void *trace[NR_CPUS];
- unsigned long trace_idx[NR_CPUS];
unsigned long entries;
long ctrl;
unsigned long iter_flags;
atomic_t cnt;
- atomic_t disabled[NR_CPUS];
- atomic_t underrun[NR_CPUS];
+ struct mctracer_trace_cpu *data[NR_CPUS];
};

#endif /* _LINUX_MCOUNT_TRACER_H */

--