Re: Mainline kernel OLTP performance update

Gregory Haskins
2009-01-16T18:11:26Z

[-- Attachment #1.1: Type: text/plain, Size: 1964 bytes --]

Ma, Chinang wrote:
> Gregory.
> I will test the resched-ipi instrumentation patch with our OLTP if you can post the patch and some instructions.
> Thanks,
> -Chinang
>

Hi Chinang,
Please find a patch attached which applies to linus.git as of today.
You will also want to enable CONFIG_FUNCTION_TRACER as well as the trace
components. Here is my system:

ghaskins@dev:~/sandbox/git/linux-2.6-rt> grep TRACE .config
CONFIG_STACKTRACE_SUPPORT=y
CONFIG_TRACEPOINTS=y
CONFIG_HAVE_ARCH_TRACEHOOK=y
CONFIG_BLK_DEV_IO_TRACE=y
# CONFIG_TREE_RCU_TRACE is not set
# CONFIG_PREEMPT_RCU_TRACE is not set
CONFIG_X86_PTRACE_BTS=y
# CONFIG_ACPI_DEBUG_FUNC_TRACE is not set
CONFIG_NETFILTER_XT_TARGET_TRACE=m
CONFIG_SOUND_TRACEINIT=y
CONFIG_TRACE_IRQFLAGS_SUPPORT=y
CONFIG_TRACE_IRQFLAGS=y
CONFIG_STACKTRACE=y
# CONFIG_BACKTRACE_SELF_TEST is not set
CONFIG_USER_STACKTRACE_SUPPORT=y
CONFIG_NOP_TRACER=y
CONFIG_HAVE_FUNCTION_TRACER=y
CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
CONFIG_HAVE_DYNAMIC_FTRACE=y
CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
CONFIG_HAVE_HW_BRANCH_TRACER=y
CONFIG_TRACER_MAX_TRACE=y
CONFIG_FUNCTION_TRACER=y
CONFIG_FUNCTION_GRAPH_TRACER=y
CONFIG_IRQSOFF_TRACER=y
CONFIG_SYSPROF_TRACER=y
CONFIG_SCHED_TRACER=y
CONFIG_CONTEXT_SWITCH_TRACER=y
# CONFIG_BOOT_TRACER is not set
# CONFIG_TRACE_BRANCH_PROFILING is not set
CONFIG_POWER_TRACER=y
CONFIG_STACK_TRACER=y
CONFIG_HW_BRANCH_TRACER=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FTRACE_MCOUNT_RECORD=y
# CONFIG_FTRACE_STARTUP_TEST is not set
# CONFIG_MMIOTRACE is not set
# CONFIG_KVM_TRACE is not set

Then on your booted system, do:

echo sched_switch > /sys/kernel/debug/tracing/current_tracer
echo 1 > /sys/kernel/debug/tracing/tracing_enabled
$run_oltp && echo 0 > /sys/kernel/debug/tracing/tracing_enabled

(where $run_oltp is your suite)

Then, email the contents of /sys/kernel/debug/tracing/trace to me

-Greg

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1.2: instrumentation.patch --]
[-- Type: text/x-patch; name="instrumentation.patch", Size: 3263 bytes --]

ftrace instrumentation for RT tasks

From: Gregory Haskins <ghaskins@novell.com>

Signed-off-by: Gregory Haskins <ghaskins@novell.com>
---

arch/x86/kernel/smp.c | 2 ++
include/linux/sched.h | 6 ++++++
kernel/sched.c | 3 +++
kernel/sched_rt.c | 10 ++++++++++
4 files changed, 21 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index e6faa33..468abeb 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -118,6 +118,7 @@ static void native_smp_send_reschedule(int cpu)
WARN_ON(1);
return;
}
+ ftrace_printk("cpu %d\n", cpu);
send_IPI_mask(cpumask_of(cpu), RESCHEDULE_VECTOR);
}

@@ -171,6 +172,7 @@ static void native_smp_send_stop(void)
*/
void smp_reschedule_interrupt(struct pt_regs *regs)
{
+ ftrace_printk("NEEDS_RESCHED\n");
ack_APIC_irq();
inc_irq_stat(irq_resched_count);
}
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 4cae9b8..a320692 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2094,8 +2094,14 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
return test_ti_thread_flag(task_thread_info(tsk), flag);
}

+# define ftrace_printk(fmt...) __ftrace_printk(_THIS_IP_, fmt)
+extern int
+__ftrace_printk(unsigned long ip, const char *fmt, ...)
+ __attribute__ ((format (printf, 2, 3)));
+
static inline void set_tsk_need_resched(struct task_struct *tsk)
{
+ ftrace_printk("%s/%d\n", tsk->comm, tsk->pid);
set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
}

diff --git a/kernel/sched.c b/kernel/sched.c
index 52bbf1c..d55fcf1 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -1874,6 +1874,9 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
*new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
u64 clock_offset;

+ ftrace_printk("migrate %s/%d [%d] -> [%d]\n",
+ p->comm, p->pid, task_cpu(p), new_cpu);
+
clock_offset = old_rq->clock - new_rq->clock;

trace_sched_migrate_task(p, task_cpu(p), new_cpu);
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 954e1a8..59cf64b 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -1102,6 +1102,8 @@ static int push_rt_task(struct rq *rq)
if (!next_task)
return 0;

+ ftrace_printk("attempting push\n");
+
retry:
if (unlikely(next_task == rq->curr)) {
WARN_ON(1);
@@ -1139,6 +1141,8 @@ static int push_rt_task(struct rq *rq)
goto out;
}

+ ftrace_printk("%s/%d\n", next_task->comm, next_task->pid);
+
deactivate_task(rq, next_task, 0);
set_task_cpu(next_task, lowest_rq->cpu);
activate_task(lowest_rq, next_task, 0);
@@ -1180,6 +1184,8 @@ static int pull_rt_task(struct rq *this_rq)
if (likely(!rt_overloaded(this_rq)))
return 0;

+ ftrace_printk("attempting pull\n");
+
next = pick_next_task_rt(this_rq);

for_each_cpu(cpu, this_rq->rd->rto_mask) {
@@ -1234,6 +1240,10 @@ static int pull_rt_task(struct rq *this_rq)

ret = 1;

+ ftrace_printk("pull %s/%d [%d] -> [%d]\n",
+ p->comm, p->pid,
+ src_rq->cpu, this_rq->cpu);
+
deactivate_task(src_rq, p, 0);
set_task_cpu(p, this_cpu);
activate_task(this_rq, p, 0);

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 257 bytes --]