* [Patch 0/2] TSC: fix incorrectly marked unstable TSC clock @ 2009-04-07 14:31 Luis Claudio R. Goncalves 2009-04-07 14:33 ` [Patch 1/2] backport of x86: add rdtsc barrier to TSC sync check Luis Claudio R. Goncalves 0 siblings, 1 reply; 3+ messages in thread From: Luis Claudio R. Goncalves @ 2009-04-07 14:31 UTC (permalink / raw) To: linux-rt-users; +Cc: Clark Williams, Thomas Gleixner Thomas, I have backported to 2.6.24.7-rt27 a set of changes from upstream that fixes several cases where TSC was erroneously marked unstable, mainly in newer systems. These changes are simple and pretty straightforward. I have been running kernels with these changes for more than a month and in several different boxes. Please check if these patches are of interest to carry in the PREEMPT_RT patchset. Regards, Luis -- [ Luis Claudio R. Goncalves Bass - Gospel - RT ] [ Fingerprint: 4FDD B8C4 3C59 34BD 8BE9 2696 7203 D980 A448 C8F8 ] ^ permalink raw reply [flat|nested] 3+ messages in thread
* [Patch 1/2] backport of x86: add rdtsc barrier to TSC sync check 2009-04-07 14:31 [Patch 0/2] TSC: fix incorrectly marked unstable TSC clock Luis Claudio R. Goncalves @ 2009-04-07 14:33 ` Luis Claudio R. Goncalves 2009-04-07 14:36 ` [Patch 2/2] TSC: several enhancements on callers of mark_tsc_unstable() Luis Claudio R. Goncalves 0 siblings, 1 reply; 3+ messages in thread From: Luis Claudio R. Goncalves @ 2009-04-07 14:33 UTC (permalink / raw) To: linux-rt-users; +Cc: Clark Williams, Thomas Gleixner This changeset is mainly a backport of upstream commit 93ce99e849433ede4ce8b410b749dc0cad1100b2 and its required infrastructure. | commit 93ce99e849433ede4ce8b410b749dc0cad1100b2 | Author: Venki Pallipadi <venkatesh.pallipadi@intel.com> | Date: Mon Nov 17 14:43:58 2008 -0800 | | x86: add rdtsc barrier to TSC sync check | | Impact: fix incorrectly marked unstable TSC clock | | Patch (commit 0d12cdd "sched: improve sched_clock() performance") has | a regression on one of the test systems here. | | With the patch, I see: | | checking TSC synchronization [CPU#0 -> CPU#1]: | Measured 28 cycles TSC warp between CPUs, turning off TSC clock. | Marking TSC unstable due to check_tsc_sync_source failed | | Whereas, without the patch syncs pass fine on all CPUs: | | checking TSC synchronization [CPU#0 -> CPU#1]: passed. | | Due to this, TSC is marked unstable, when it is not actually unstable. | This is because syncs in check_tsc_wrap() goes away due to this commit. | | As per the discussion on this thread, correct way to fix this is to add | explicit syncs as below? | | Signed-off-by: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> | Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Luis Claudio R. Goncalves <lgoncalv@redhat.com> --- diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index 4f08c7b..f5fd23c 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -46,7 +46,9 @@ static __cpuinit void check_tsc_warp(void) cycles_t start, now, prev, end; int i; - start = get_cycles_sync(); + rdtsc_barrier(); + start = get_cycles(); + rdtsc_barrier(); /* * The measurement runs for 20 msecs: */ @@ -61,18 +63,20 @@ static __cpuinit void check_tsc_warp(void) */ __raw_spin_lock(&sync_lock); prev = last_tsc; - now = get_cycles_sync(); + rdtsc_barrier(); + now = get_cycles(); + rdtsc_barrier(); last_tsc = now; __raw_spin_unlock(&sync_lock); /* * Be nice every now and then (and also check whether - * measurement is done [we also insert a 100 million + * measurement is done [we also insert a 10 million * loops safety exit, so we dont lock up in case the * TSC readout is totally broken]): */ if (unlikely(!(i & 7))) { - if (now > end || i > 100000000) + if (now > end || i > 10000000) break; cpu_relax(); touch_nmi_watchdog(); @@ -87,8 +91,10 @@ static __cpuinit void check_tsc_warp(void) nr_warps++; __raw_spin_unlock(&sync_lock); } - } + if (!(now-start)) + printk(KERN_WARNING "Warning: zero tsc calibration delta:" + " %lld [max: %lld]\n", now-start, end-start); } /* @@ -97,7 +103,6 @@ static __cpuinit void check_tsc_warp(void) */ void __cpuinit check_tsc_sync_source(int cpu) { - unsigned long flags; int cpus = 2; /* @@ -118,11 +123,8 @@ void __cpuinit check_tsc_sync_source(int cpu) /* * Wait for the target to arrive: */ - local_save_flags(flags); - local_irq_enable(); while (atomic_read(&start_count) != cpus-1) cpu_relax(); - local_irq_restore(flags); /* * Trigger the target to continue into the measurement too: */ @@ -133,24 +135,24 @@ void __cpuinit check_tsc_sync_source(int cpu) while (atomic_read(&stop_count) != cpus-1) cpu_relax(); - /* - * Reset it - just in case we boot another CPU later: - */ - atomic_set(&start_count, 0); - if (nr_warps) { printk("\n"); printk(KERN_WARNING "Measured %Ld cycles TSC warp between CPUs," " turning off TSC clock.\n", max_warp); mark_tsc_unstable("check_tsc_sync_source failed"); - nr_warps = 0; - max_warp = 0; - last_tsc = 0; } else { printk(" passed.\n"); } /* + * Reset it - just in case we boot another CPU later: + */ + atomic_set(&start_count, 0); + nr_warps = 0; + max_warp = 0; + last_tsc = 0; + + /* * Let the target continue with the bootup: */ atomic_inc(&stop_count); diff --git a/include/asm-x86/cpufeature_32.h b/include/asm-x86/cpufeature_32.h index f17e688..49f14b4 100644 --- a/include/asm-x86/cpufeature_32.h +++ b/include/asm-x86/cpufeature_32.h @@ -82,6 +82,8 @@ /* 14 free */ #define X86_FEATURE_SYNC_RDTSC (3*32+15) /* RDTSC synchronizes the CPU */ #define X86_FEATURE_REP_GOOD (3*32+16) /* rep microcode works well on this CPU */ +#define X86_FEATURE_MFENCE_RDTSC (3*32+17) /* "" Mfence synchronizes RDTSC */ +#define X86_FEATURE_LFENCE_RDTSC (3*32+18) /* "" Lfence synchronizes RDTSC */ /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ #define X86_FEATURE_XMM3 (4*32+ 0) /* Streaming SIMD Extensions-3 */ diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h index 46e1c04..e1dc8fc 100644 --- a/include/asm-x86/processor.h +++ b/include/asm-x86/processor.h @@ -1,5 +1,23 @@ +#ifndef __ASM_PROCESSOR_H +#define __ASM_PROCESSOR_H + #ifdef CONFIG_X86_32 # include "processor_32.h" #else # include "processor_64.h" #endif + +/* + * Stop RDTSC speculation. This is needed when you need to use RDTSC + * (or get_cycles or vread that possibly accesses the TSC) in a defined + * code region. + * + * (Could use an alternative three way for this if there was one.) + */ +static inline void rdtsc_barrier(void) +{ + alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC); + alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC); +} + +#endif /* __ASM_PROCESSOR_H */ -- [ Luis Claudio R. Goncalves Red Hat - Realtime Team ] [ Fingerprint: 4FDD B8C4 3C59 34BD 8BE9 2696 7203 D980 A448 C8F8 ] ^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [Patch 2/2] TSC: several enhancements on callers of mark_tsc_unstable() 2009-04-07 14:33 ` [Patch 1/2] backport of x86: add rdtsc barrier to TSC sync check Luis Claudio R. Goncalves @ 2009-04-07 14:36 ` Luis Claudio R. Goncalves 0 siblings, 0 replies; 3+ messages in thread From: Luis Claudio R. Goncalves @ 2009-04-07 14:36 UTC (permalink / raw) To: linux-rt-users; +Cc: Clark Williams, Thomas Gleixner This patch is a backport from Linus' linux-2.6 git tree of various enhancements related to the functions that could eventually call mark_tsc_unstable(). Kernel 2.6.24.7-rt27 has been marking TSC as unstable in several machines where it is, in fact, stable and constant. Signed-off-by: Luis Claudio R. Goncalves <lclaudio@redhat.com> --- diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 1ff88c7..184cb50 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -259,8 +259,10 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c) if (cpuid_eax(0x80000000) >= 0x80000007) { c->x86_power = cpuid_edx(0x80000007); - if (c->x86_power & (1<<8)) + if (c->x86_power & (1<<8)) { set_bit(X86_FEATURE_CONSTANT_TSC, c->x86_capability); + set_bit(X86_FEATURE_NONSTOP_TSC, c->x86_capability); + } } #ifdef CONFIG_X86_HT diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index cc8c501..450f7f1 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -211,6 +211,15 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) (c->x86 == 0x6 && c->x86_model >= 0x0e)) set_bit(X86_FEATURE_CONSTANT_TSC, c->x86_capability); + /* + * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate + * with P/T states and does not stop in deep C-states + */ + if (c->x86_power & (1 << 8)) { + set_bit(X86_FEATURE_NONSTOP_TSC, c->x86_capability); + set_bit(X86_FEATURE_CONSTANT_TSC, c->x86_capability); + } + if (cpu_has_ds) { unsigned int l1; rdmsr(MSR_IA32_MISC_ENABLE, l1, l2); diff --git a/arch/x86/kernel/setup_64.c b/arch/x86/kernel/setup_64.c index 30d94d1..167d62b 100644 --- a/arch/x86/kernel/setup_64.c +++ b/arch/x86/kernel/setup_64.c @@ -663,8 +663,10 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c) display_cacheinfo(c); /* c->x86_power is 8000_0007 edx. Bit 8 is constant TSC */ - if (c->x86_power & (1<<8)) + if (c->x86_power & (1<<8)) { set_bit(X86_FEATURE_CONSTANT_TSC, &c->x86_capability); + set_bit(X86_FEATURE_NONSTOP_TSC, &c->x86_capability); + } /* Multi core CPU? */ if (c->extended_cpuid_level >= 0x80000008) @@ -810,8 +812,10 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) if (c->x86 == 15) c->x86_cache_alignment = c->x86_clflush_size * 2; if ((c->x86 == 0xf && c->x86_model >= 0x03) || - (c->x86 == 0x6 && c->x86_model >= 0x0e)) + (c->x86 == 0x6 && c->x86_model >= 0x0e)) { set_bit(X86_FEATURE_CONSTANT_TSC, &c->x86_capability); + set_bit(X86_FEATURE_NONSTOP_TSC, &c->x86_capability); + } if (c->x86 == 6) set_bit(X86_FEATURE_REP_GOOD, &c->x86_capability); if (c->x86 == 15) diff --git a/arch/x86/kernel/tsc_32.c b/arch/x86/kernel/tsc_32.c index 1feecd4..0aa43ed 100644 --- a/arch/x86/kernel/tsc_32.c +++ b/arch/x86/kernel/tsc_32.c @@ -214,6 +214,9 @@ time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data) { struct cpufreq_freqs *freq = data; + if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC)) + return 0; + if (!ref_freq) { if (!freq->old){ ref_freq = freq->new; diff --git a/arch/x86/kernel/tsc_64.c b/arch/x86/kernel/tsc_64.c index 5d24959..72a389e 100644 --- a/arch/x86/kernel/tsc_64.c +++ b/arch/x86/kernel/tsc_64.c @@ -215,27 +215,26 @@ void __init tsc_calibrate(void) */ __cpuinit int unsynchronized_tsc(void) { - if (tsc_unstable) + if (!cpu_has_tsc || tsc_unstable) return 1; -#ifdef CONFIG_SMP +#ifdef CONFIG_X86_SMP if (apic_is_clustered_box()) return 1; #endif - /* Most intel systems have synchronized TSCs except for - multi node systems */ - if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { -#ifdef CONFIG_ACPI - /* But TSC doesn't tick in C3 so don't use it there */ - if (acpi_gbl_FADT.header.length > 0 && - acpi_gbl_FADT.C3latency < 1000) - return 1; -#endif + if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) return 0; + /* + * Intel systems are normally all synchronized. + * Exceptions must mark TSC as unstable: + */ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { + /* assume multi socket systems are not synchronized: */ + if (num_possible_cpus() > 1) + tsc_unstable = 1; } - /* Assume multi socket systems are not synchronized */ - return num_present_cpus() > 1; + return tsc_unstable; } int __init notsc_setup(char *s) diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index e5d21e4..a91246a 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -516,7 +516,8 @@ static void acpi_processor_idle(void) #if defined (CONFIG_GENERIC_TIME) && defined (CONFIG_X86_TSC) /* TSC halts in C2, so notify users */ - mark_tsc_unstable("possible TSC halt in C2"); + if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) + mark_tsc_unstable("possible TSC halt in C2"); #endif /* Compute time (ticks) that we were actually asleep */ sleep_ticks = ticks_elapsed(t1, t2); @@ -579,7 +580,8 @@ static void acpi_processor_idle(void) #if defined (CONFIG_GENERIC_TIME) && defined (CONFIG_X86_TSC) /* TSC halts in C3, so notify users */ - mark_tsc_unstable("TSC halts in C3"); + if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) + mark_tsc_unstable("TSC halts in C3"); #endif /* Compute time (ticks) that we were actually asleep */ sleep_ticks = ticks_elapsed(t1, t2); @@ -1456,7 +1458,8 @@ static int acpi_idle_enter_simple(struct cpuidle_device *dev, #if defined (CONFIG_GENERIC_TIME) && defined (CONFIG_X86_TSC) /* TSC could halt in idle, so notify users */ - mark_tsc_unstable("TSC halts in idle");; + if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) + mark_tsc_unstable("TSC halts in idle");; #endif sleep_ticks = ticks_elapsed(t1, t2); @@ -1567,7 +1570,8 @@ static int acpi_idle_enter_bm(struct cpuidle_device *dev, #if defined (CONFIG_GENERIC_TIME) && defined (CONFIG_X86_TSC) /* TSC could halt in idle, so notify users */ - mark_tsc_unstable("TSC halts in idle"); + if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) + mark_tsc_unstable("TSC halts in idle"); #endif sleep_ticks = ticks_elapsed(t1, t2); /* Tell the scheduler how much we idled: */ diff --git a/include/asm-x86/cpufeature_32.h b/include/asm-x86/cpufeature_32.h index 49f14b4..bdc6e27 100644 --- a/include/asm-x86/cpufeature_32.h +++ b/include/asm-x86/cpufeature_32.h @@ -84,6 +84,8 @@ #define X86_FEATURE_REP_GOOD (3*32+16) /* rep microcode works well on this CPU */ #define X86_FEATURE_MFENCE_RDTSC (3*32+17) /* "" Mfence synchronizes RDTSC */ #define X86_FEATURE_LFENCE_RDTSC (3*32+18) /* "" Lfence synchronizes RDTSC */ +#define X86_FEATURE_TSC_RELIABLE (3*32+23) /* TSC is known to be reliable */ +#define X86_FEATURE_NONSTOP_TSC (3*32+24) /* TSC does not stop in C states */ /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ #define X86_FEATURE_XMM3 (4*32+ 0) /* Streaming SIMD Extensions-3 */ --- -- [ Luis Claudio R. Goncalves Red Hat - Realtime Team ] [ Fingerprint: 4FDD B8C4 3C59 34BD 8BE9 2696 7203 D980 A448 C8F8 ] ^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2009-04-07 14:36 UTC | newest] Thread overview: 3+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2009-04-07 14:31 [Patch 0/2] TSC: fix incorrectly marked unstable TSC clock Luis Claudio R. Goncalves 2009-04-07 14:33 ` [Patch 1/2] backport of x86: add rdtsc barrier to TSC sync check Luis Claudio R. Goncalves 2009-04-07 14:36 ` [Patch 2/2] TSC: several enhancements on callers of mark_tsc_unstable() Luis Claudio R. Goncalves
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).