[PATCH 13/39] NLKD/x86-64 - time adjustment

["Jan Beulich" <JBeulich@novell.com>]

[-- Attachment #1: Type: text/plain, Size: 394 bytes --]

Since x86-64 time handling is not overflow-safe, these are the
adjustments needed for allowing debuggers to update time after
having halted the system for perhaps extended periods of time.

Note that this depends on the HPET definitions adjustments, which
aren't in 2.6.14, but have supposedly been merged already into 2.6.15.

From: Jan Beulich <jbeulich@novell.com>

(actual patch attached)


[-- Attachment #2: linux-2.6.14-nlkd-time-x86_64.patch --]
[-- Type: application/octet-stream, Size: 14844 bytes --]

Since x86-64 time handling is not overflow-safe, these are the
adjustments needed for allowing debuggers to update time after
having halted the system for perhaps extended periods of time.

Note that this depends on the HPET definitions adjustments, which
aren't in 2.6.14, but have supposedly been merged already into 2.6.15.

From: Jan Beulich <jbeulich@novell.com>

Index: 2.6.14-nlkd/arch/x86_64/kernel/pmtimer.c
===================================================================
--- 2.6.14-nlkd.orig/arch/x86_64/kernel/pmtimer.c	2005-11-09 10:40:17.000000000 +0100
+++ 2.6.14-nlkd/arch/x86_64/kernel/pmtimer.c	2005-11-04 16:19:33.000000000 +0100
@@ -52,22 +52,53 @@ int pmtimer_mark_offset(void)
 {
 	static int first_run = 1;
 	unsigned long tsc;
-	u32 lost;
-
+	u32 lost, delta;
 	u32 tick = inl(pmtmr_ioport);
-	u32 delta;
+#ifdef debugger_jiffies
+# define delta64 delta
+#else
+	u64 delta64;
+#endif
+
+	rdtscll(tsc);
 
-	delta = cyc2us((tick - last_pmtmr_tick) & ACPI_PM_MASK);
+	if (likely(!debugger_jiffies)) {
+		delta = cyc2us((tick - last_pmtmr_tick) & ACPI_PM_MASK);
+#ifndef delta64
+		delta64 = delta;
+	}
+	else {
+# ifdef CONFIG_SMP
+		/* When the TSC gets reset during AP startup, the code below would
+		   incorrectly think we lost a huge amount of ticks. */
+		if (unlikely((long long)(tsc - vxtime.last_tsc) < 0))
+			delta64 = tsc;
+		else
+# endif
+			delta64 = tsc - vxtime.last_tsc;
+		__asm__("mulq %1\n\t"
+		        "shrdq $32, %%rdx, %0"
+		        : "+a" (delta64)
+		        : "rm" (vxtime.tsc_quot)
+		        : "rdx");
+		delta = (u32)delta64 ? (u32)delta64 - 1 : (offset_delay = 0);
+#endif
+	}
 
 	last_pmtmr_tick = tick;
 	monotonic_base += delta * NSEC_PER_USEC;
 
-	delta += offset_delay;
-
-	lost = delta / (USEC_PER_SEC / HZ);
-	offset_delay = delta % (USEC_PER_SEC / HZ);
+	if (likely(delta == delta64)) {
+		delta += offset_delay;
+		lost = delta / (USEC_PER_SEC / HZ);
+		offset_delay = delta % (USEC_PER_SEC / HZ);
+	}
+	else {
+		lost = (u32)(delta64 / (USEC_PER_SEC / HZ));
+		offset_delay = 0;
+	}
+#undef delta64
 
-	rdtscll(tsc);
 	vxtime.last_tsc = tsc - offset_delay * cpu_khz;
 
 	/* don't calculate delay for first run,
Index: 2.6.14-nlkd/arch/x86_64/kernel/time.c
===================================================================
--- 2.6.14-nlkd.orig/arch/x86_64/kernel/time.c	2005-11-09 10:40:17.000000000 +0100
+++ 2.6.14-nlkd/arch/x86_64/kernel/time.c	2005-11-04 16:19:33.000000000 +0100
@@ -58,6 +58,12 @@ DEFINE_SPINLOCK(i8253_lock);
 static int nohpet __initdata = 0;
 static int notsc __initdata = 0;
 
+#ifndef CONFIG_DEBUG_KERNEL
+# define hpet64 0
+#else
+static int hpet64 = -1;
+#endif
+
 #undef HPET_HACK_ENABLE_DANGEROUS
 
 unsigned int cpu_khz;					/* TSC clocks / usec, not used here */
@@ -65,7 +71,7 @@ static unsigned long hpet_period;			/* f
 unsigned long hpet_tick;				/* HPET clocks / interrupt */
 static int hpet_use_timer;
 unsigned long vxtime_hz = PIT_TICK_RATE;
-int report_lost_ticks;				/* command line option */
+static int report_lost_ticks;				/* command line option */
 unsigned long long monotonic_base;
 
 struct vxtime_data __vxtime __section_vxtime;	/* for vsyscalls */
@@ -75,6 +81,19 @@ unsigned long __wall_jiffies __section_w
 struct timespec __xtime __section_xtime;
 struct timezone __sys_tz __section_sys_tz;
 
+static unsigned int cyc2ns_scale;
+#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
+
+static inline void set_cyc2ns_scale(unsigned long cpu_mhz)
+{
+	cyc2ns_scale = (1000 << CYC2NS_SCALE_FACTOR)/cpu_mhz;
+}
+
+static inline unsigned long long cycles_2_ns(unsigned long long cyc)
+{
+	return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
+}
+
 static inline void rdtscll_sync(unsigned long *tsc)
 {
 #ifdef CONFIG_SMP
@@ -365,7 +384,14 @@ static irqreturn_t timer_interrupt(int i
 {
 	static unsigned long rtc_update = 0;
 	unsigned long tsc;
-	int delay, offset = 0, lost = 0;
+	unsigned int delay;
+#ifdef hpet64
+	int
+#else
+	long
+# define last last64
+#endif
+		offset = 0, lost = 0;
 
 /*
  * Here we are in the timer irq handler. We have irqs locally disabled (so we
@@ -384,8 +410,14 @@ static irqreturn_t timer_interrupt(int i
 		 * we can more accurately know the counter value
 		 * when the timer interrupt occured.
 		 */
+		unsigned int hi1 = hpet64 > 0 ? hpet_readl(HPET_COUNTER+4) : 0;
+
 		offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
 		delay = hpet_readl(HPET_COUNTER) - offset;
+		if (hpet64 > 0)
+			offset += (long)(offset >= 0 ? hi1 : hpet_readl(HPET_COUNTER+4)) << 32;
+		else
+			offset = (unsigned int)offset;
 	} else {
 		spin_lock(&i8253_lock);
 		outb_p(0x00, 0x43);
@@ -398,21 +430,66 @@ static irqreturn_t timer_interrupt(int i
 	rdtscll_sync(&tsc);
 
 	if (vxtime.mode == VXTIME_HPET) {
-		if (offset - vxtime.last > hpet_tick) {
-			lost = (offset - vxtime.last) / hpet_tick - 1;
+		if (hpet64 > 0) {
+			unsigned long delta = offset - vxtime.last;
+
+			if (delta > hpet_tick)
+				lost = delta / hpet_tick - 1;
+			monotonic_base += delta * (NSEC_PER_SEC/HZ) / hpet_tick;
 		}
+		else if (likely(!debugger_jiffies)) {
+			unsigned int delta = offset - vxtime.last;
+
+			if (delta > hpet_tick)
+				lost = delta / hpet_tick - 1;
+			monotonic_base += delta * (NSEC_PER_SEC/HZ) / hpet_tick;
+		}
+		else {
+			unsigned long delta;
+
+#ifdef CONFIG_SMP
+			/* When the TSC gets reset during AP startup, the code below would
+			   incorrectly think we lost a huge amount of ticks. */
+			if(unlikely((long)(tsc - vxtime.last_tsc) < 0))
+				delta = tsc;
+			else
+#endif
+				delta = tsc - vxtime.last_tsc;
+			delta = cycles_2_ns(delta);
 
-		monotonic_base += 
-			(offset - vxtime.last)*(NSEC_PER_SEC/HZ) / hpet_tick;
+			if (delta > NSEC_PER_SEC/HZ)
+				lost = delta / (NSEC_PER_SEC/HZ) - 1;
+			monotonic_base += delta;
+		}
 
 		vxtime.last = offset;
+#ifndef debugger_jiffies
+		vxtime.last_tsc = tsc;
+#endif
 #ifdef CONFIG_X86_PM_TIMER
 	} else if (vxtime.mode == VXTIME_PMTMR) {
 		lost = pmtimer_mark_offset();
 #endif
 	} else {
-		offset = (((tsc - vxtime.last_tsc) *
-			   vxtime.tsc_quot) >> 32) - (USEC_PER_SEC / HZ);
+		long delta;
+
+		if (likely(!debugger_jiffies))
+			delta = ((tsc - vxtime.last_tsc) * vxtime.tsc_quot) >> 32;
+		else {
+			delta = tsc - vxtime.last_tsc;
+#ifdef CONFIG_SMP
+			/* When the TSC gets reset during AP startup, the code below would
+			   incorrectly think we lost a huge amount of ticks. */
+			if (delta < 0)
+				delta = tsc;
+#endif
+			__asm__("mulq %1\n\t"
+			        "shrdq $32, %%rdx, %0"
+			        : "+a" (delta)
+			        : "rm" (vxtime.tsc_quot)
+			        : "rdx");
+		}
+		offset = delta - (USEC_PER_SEC / HZ);
 
 		if (offset < 0)
 			offset = 0;
@@ -426,16 +503,31 @@ static irqreturn_t timer_interrupt(int i
 
 		vxtime.last_tsc = tsc - vxtime.quot * delay / vxtime.tsc_quot;
 
-		if ((((tsc - vxtime.last_tsc) *
-		      vxtime.tsc_quot) >> 32) < offset)
-			vxtime.last_tsc = tsc -
-				(((long) offset << 32) / vxtime.tsc_quot) - 1;
+		if (delta < offset) {
+			if (likely(!debugger_jiffies))
+				vxtime.last_tsc = tsc -
+					(((long) offset << 32) / vxtime.tsc_quot) - 1;
+			else {
+				unsigned long temp;
+
+				/* This division cannot overflow because the dividend was the
+				   result of a multiplication with the divisor. */
+				__asm__("divq %3"
+					: "=a" (delta),
+					  "=d" (temp)
+					: "A" ((__uint128_t)(unsigned long)offset << 32),
+					  "rm" (vxtime.tsc_quot));
+				vxtime.last_tsc = tsc - delta - 1;
+			}
+		}
 	}
 
 	if (lost > 0) {
-		handle_lost_ticks(lost, regs);
+		if (!debugger_jiffies)
+			handle_lost_ticks(lost, regs);
 		jiffies += lost;
 	}
+#undef last
 
 /*
  * Do the timer stuff.
@@ -478,19 +570,6 @@ static irqreturn_t timer_interrupt(int i
 	return IRQ_HANDLED;
 }
 
-static unsigned int cyc2ns_scale;
-#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
-
-static inline void set_cyc2ns_scale(unsigned long cpu_mhz)
-{
-	cyc2ns_scale = (1000 << CYC2NS_SCALE_FACTOR)/cpu_mhz;
-}
-
-static inline unsigned long long cycles_2_ns(unsigned long long cyc)
-{
-	return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
-}
-
 unsigned long long sched_clock(void)
 {
 	unsigned long a = 0;
@@ -791,7 +870,7 @@ static __init int late_hpet_init(void)
 fs_initcall(late_hpet_init);
 #endif
 
-static int hpet_timer_stop_set_go(unsigned long tick)
+static int hpet_timer_stop_set_go(unsigned long tick, unsigned n)
 {
 	unsigned int cfg;
 
@@ -802,8 +881,47 @@ static int hpet_timer_stop_set_go(unsign
 	cfg = hpet_readl(HPET_CFG);
 	cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
 	hpet_writel(cfg, HPET_CFG);
-	hpet_writel(0, HPET_COUNTER);
-	hpet_writel(0, HPET_COUNTER + 4);
+
+#ifndef hpet64
+	if (hpet64 && n > 0) {
+/*
+ * Unfortunately at least the ICH6 HPET runs its main counter in 32-bit mode
+ * when timer 0 is not run in 64-bit mode. Thus we need to check here whether
+ * the main counter remains in 64-bit mode when all 64-bit capable counters
+ * have been set to 32-bit mode. An alternative would be to run timer 0 in
+ * 64-bit mode while reading only its low 32 bits, but this requires that
+ * there be no 32-bit writers of the comparator or increment registers, which
+ * (given the global visibility of hpet_{read,write}{l,q}) cannot be
+ * guaranteed.
+ */
+		unsigned i;
+
+		/* Disable counters and set them to 32-bit mode. */
+		for (i = 0; i <= n; ++i)
+			hpet_writel(HPET_Tn_CFG(i), HPET_TN_32BIT);
+
+		/* Set main counter so it will immediately overflow and enable it */
+		hpet_writeq(0xffffffff, HPET_COUNTER);
+		hpet_writel(cfg | HPET_CFG_ENABLE, HPET_CFG);
+
+		/* Wait for it to overflow 32 bits. */
+		while (!(hpet_readl(HPET_COUNTER) + 1))
+			;
+
+		/* Disable main counter again. */
+		hpet_writel(cfg, HPET_CFG);
+
+		/* Check whether the high part changed. */
+		if (!hpet_readl(HPET_COUNTER + 4))
+			hpet64 = 0;
+
+		/* Perhaps this should generally be done? */
+		for (i = 0; i <= n; ++i)
+			hpet_writel(HPET_Tn_CFG(i), 0);
+	}
+#endif
+
+	hpet_writeq(0, HPET_COUNTER);
 
 /*
  * Set up timer 0, as periodic with first interrupt to happen at hpet_tick,
@@ -812,8 +930,8 @@ static int hpet_timer_stop_set_go(unsign
 	if (hpet_use_timer) {
 		hpet_writel(HPET_TN_ENABLE | HPET_TN_PERIODIC | HPET_TN_SETVAL |
 		    HPET_TN_32BIT, HPET_T0_CFG);
-		hpet_writel(hpet_tick, HPET_T0_CMP);
-		hpet_writel(hpet_tick, HPET_T0_CMP); /* AK: why twice? */
+		hpet_writeq(hpet_tick, HPET_T0_CMP); /* set comparator */
+		hpet_writeq(hpet_tick, HPET_T0_CMP); /* set increment */
 		cfg |= HPET_CFG_LEGACY;
 	}
 /*
@@ -841,7 +959,9 @@ static int hpet_init(void)
 
 	id = hpet_readl(HPET_ID);
 
-	if (!(id & HPET_ID_VENDOR) || !(id & HPET_ID_NUMBER))
+	if (!(id + 1)
+	    || !(id & HPET_ID_VENDOR)
+	    || !(id & HPET_ID_NUMBER))
 		return -1;
 
 	hpet_period = hpet_readl(HPET_PERIOD);
@@ -853,12 +973,17 @@ static int hpet_init(void)
 
 	hpet_use_timer = (id & HPET_ID_LEGSUP);
 
-	return hpet_timer_stop_set_go(hpet_tick);
+#ifndef hpet64
+	if (hpet64 && (!hpet_use_timer || !(id & HPET_ID_64BIT)))
+		hpet64 = 0;
+#endif
+
+	return hpet_timer_stop_set_go(hpet_tick, (id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT);
 }
 
 static int hpet_reenable(void)
 {
-	return hpet_timer_stop_set_go(hpet_tick);
+	return hpet_timer_stop_set_go(hpet_tick, 0);
 }
 
 void __init pit_init(void)
@@ -889,16 +1014,42 @@ void __init time_init(void)
 	char *timename;
 
 #ifdef HPET_HACK_ENABLE_DANGEROUS
-        if (!vxtime.hpet_address) {
+	if (!vxtime.hpet_address && !nohpet) {
 		printk(KERN_WARNING "time.c: WARNING: Enabling HPET base "
 		       "manually!\n");
-                outl(0x800038a0, 0xcf8);
-                outl(0xff000001, 0xcfc);
-                outl(0x800038a0, 0xcf8);
-                vxtime.hpet_address = inl(0xcfc) & 0xfffffffe;
-		printk(KERN_WARNING "time.c: WARNING: Enabled HPET "
-		       "at %#lx.\n", vxtime.hpet_address);
-        }
+		outl(0x80003800, 0xcf8);
+		if (inw(0xcfc) == 0x1022) {
+			outl(0x800038a0, 0xcf8);
+			outl(inl(0xcfc) | 0x00000001, 0xcfc);
+			vxtime.hpet_address = inl(0xcfc) & 0xfffffc00;
+		}
+		outl(0x8000f800, 0xcf8);
+		if (inw(0xcfc) == 0x8086) {
+			switch(inl(0xcfc) >> 16) {
+			case 0x24d0: // ICH5
+				outl(0x8000f8d0, 0xcf8);
+				outl(inl(0xcfc) | 0x00020000, 0xcfc);
+				vxtime.hpet_address = 0xfed00000 | ((inl(0xcfc) >> 3) & 0x00003000);
+				break;
+			case 0x2640: // ICH6/ICH6R
+			case 0x2641: // ICH6-M
+			case 0x26B8: // ICH7/ICH7R
+				outl(0x8000f8f0, 0xcf8);
+				outl(inl(0xcfc) | 0x00000001, 0xcfc);
+				vxtime.hpet_address = inl(0xcfc) & 0xffffc000;
+				if (vxtime.hpet_address) {
+					set_fixmap_nocache(FIX_HPET_BASE, vxtime.hpet_address + 0x3000);
+					hpet_writel(hpet_readl(0x404) | 0x80, 0x404);
+					vxtime.hpet_address = 0xfed00000 | ((hpet_readl(0x404) & 0x00000003) << 12);
+					__set_fixmap(FIX_HPET_BASE, 0, __pgprot(0));
+				}
+				break;
+			}
+		}
+		if (vxtime.hpet_address)
+			printk(KERN_WARNING "time.c: WARNING: Enabled HPET "
+			       "at %#lx.\n", vxtime.hpet_address);
+	}
 #endif
 	if (nohpet)
 		vxtime.hpet_address = 0;
@@ -974,8 +1125,16 @@ void __init time_init_gtod(void)
 	if (unsynchronized_tsc())
 		notsc = 1;
 	if (vxtime.hpet_address && notsc) {
+		unsigned int hi1 = hpet64 ? hpet_readl(HPET_COUNTER+4) : 0;
+
 		timetype = hpet_use_timer ? "HPET" : "PIT/HPET";
 		vxtime.last = hpet_readl(HPET_T0_CMP) - hpet_tick;
+#ifndef hpet64
+		if (hpet64) {
+			vxtime.last64 += (long)(vxtime.last >= 0 ? hi1 : hpet_readl(HPET_COUNTER+4)) << 32;
+			hpet64 = 1;
+		}
+#endif
 		vxtime.mode = VXTIME_HPET;
 		do_gettimeoffset = do_gettimeoffset_hpet;
 #ifdef CONFIG_X86_PM_TIMER
@@ -1295,6 +1454,15 @@ static int __init nohpet_setup(char *s) 
 
 __setup("nohpet", nohpet_setup);
 
+#ifndef hpet64
+static int __init nohpet64_setup(char *s)
+{
+	hpet64 = 0;
+	return 0;
+}
+
+__setup("nohpet64", nohpet64_setup);
+#endif
 
 static int __init notsc_setup(char *s)
 {
Index: 2.6.14-nlkd/include/asm-x86_64/vsyscall.h
===================================================================
--- 2.6.14-nlkd.orig/include/asm-x86_64/vsyscall.h	2005-11-09 10:40:17.000000000 +0100
+++ 2.6.14-nlkd/include/asm-x86_64/vsyscall.h	2005-11-07 10:39:05.000000000 +0100
@@ -29,7 +29,10 @@ enum vsyscall_num {
 
 struct vxtime_data {
 	long hpet_address;	/* HPET base address */
-	int last;
+	union {
+		int last;
+		long last64;
+	};
 	unsigned long last_tsc;
 	long quot;
 	long tsc_quot;
@@ -37,7 +40,9 @@ struct vxtime_data {
 };
 
 #define hpet_readl(a)           readl((void *)fix_to_virt(FIX_HPET_BASE) + a)
+#define hpet_readq(a)           readq((void *)fix_to_virt(FIX_HPET_BASE) + a)
 #define hpet_writel(d,a)        writel(d, (void *)fix_to_virt(FIX_HPET_BASE) + a)
+#define hpet_writeq(d,a)        writeq(d, (void *)fix_to_virt(FIX_HPET_BASE) + a)
 
 /* vsyscall space (readonly) */
 extern struct vxtime_data __vxtime;