[tip:sched/urgent] sched, x86: Fix cpufreq + sched_clock() TSC scaling

tip-bot for Peter Zijlstra
2009-06-16T19:55:52Z

Commit-ID: 702dd8c6624b353eb48dd20a0d36405cfe4f4f99
Gitweb: http://git.kernel.org/tip/702dd8c6624b353eb48dd20a0d36405cfe4f4f99
Author: Peter Zijlstra <peterz@infradead.org>
AuthorDate: Tue, 16 Jun 2009 12:34:17 -0700
Committer: Ingo Molnar <mingo@elte.hu>
CommitDate: Tue, 16 Jun 2009 21:48:01 +0200

sched, x86: Fix cpufreq + sched_clock() TSC scaling

For freqency dependent TSCs we only scale the cycles, we do not account
for the discrepancy in absolute value. Add a constant factor so that:

c1 + f1*cyc == c2 + f2*cyc

[ Impact: fix/reduce sched_clock() jumps across frequency changing events ]

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Chucked-on-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

---
arch/x86/include/asm/timer.h | 6 +++++-
arch/x86/kernel/tsc.c | 8 ++++++--
2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index bd37ed4..20ca9c4 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -45,12 +45,16 @@ extern int no_timer_check;
*/

DECLARE_PER_CPU(unsigned long, cyc2ns);
+DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);

#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */

static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
{
- return cyc * per_cpu(cyc2ns, smp_processor_id()) >> CYC2NS_SCALE_FACTOR;
+ int cpu = smp_processor_id();
+ unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
+ ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
+ return ns;
}

static inline unsigned long long cycles_2_ns(unsigned long long cyc)
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 3e1c057..ef4dac5 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -589,22 +589,26 @@ EXPORT_SYMBOL(recalibrate_cpu_khz);
*/

DEFINE_PER_CPU(unsigned long, cyc2ns);
+DEFINE_PER_CPU(unsigned long long, cyc2ns_offset);

static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
{
- unsigned long long tsc_now, ns_now;
+ unsigned long long tsc_now, ns_now, *offset;
unsigned long flags, *scale;

local_irq_save(flags);
sched_clock_idle_sleep_event();

scale = &per_cpu(cyc2ns, cpu);
+ offset = &per_cpu(cyc2ns_offset, cpu);

rdtscll(tsc_now);
ns_now = __cycles_2_ns(tsc_now);

- if (cpu_khz)
+ if (cpu_khz) {
*scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
+ *offset = ns_now - (tsc_now * *scale >> CYC2NS_SCALE_FACTOR);
+ }

sched_clock_idle_wakeup_event(0);
local_irq_restore(flags);