[Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Jes Sorensen]

Hi,

On systems with a drifting ITC, such as the SGI SN2, the current
implementation of timer_interrupt() and gettimeoffset() wont work as
they rely heavily on ar.itc.

On SN2 this has been solved by using the synchronized RTC in the system,
which results a different implementation of the two above mentioned
functions.

To get around this I suggest we change gettimeoffset() and
timer_intererupt() to be machvec's.

Patch against a not too old Bjorn-2.4 kernel attached. Tested on DIG.

Cheers,
Jes

diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/kernel/time.c linux-2.4.21-pre5-030312/arch/ia64/kernel/time.c
--- linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/kernel/time.c	Thu Nov 28 18:53:09 2002
+++ linux-2.4.21-pre5-030312/arch/ia64/kernel/time.c	Fri Apr  4 18:44:21 2003
@@ -33,7 +33,7 @@
 
 #endif
 
-static void
+void
 do_profile (unsigned long ip)
 {
 	extern unsigned long prof_cpu_mask;
@@ -61,8 +61,8 @@
  * Return the number of micro-seconds that elapsed since the last update to jiffy.  The
  * xtime_lock must be at least read-locked when calling this routine.
  */
-static inline unsigned long
-gettimeoffset (void)
+unsigned long
+__ia64_gettimeoffset (void)
 {
 	unsigned long elapsed_cycles, lost = jiffies - wall_jiffies;
 	unsigned long now, last_tick;
@@ -146,8 +146,8 @@
 	tv->tv_usec = usec;
 }
 
-static void
-timer_interrupt(int irq, void *dev_id, struct pt_regs *regs)
+void
+__ia64_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs)
 {
 	unsigned long new_itm;
 
@@ -284,7 +284,7 @@
 }
 
 static struct irqaction timer_irqaction = {
-	handler:	timer_interrupt,
+	handler:	NULL,
 	flags:		SA_INTERRUPT,
 	name:		"timer"
 };
@@ -292,6 +294,7 @@
 void __init
 time_init (void)
 {
+	timer_irqaction.handler = platform_timer_interrupt;
 	register_percpu_irq(IA64_TIMER_VECTOR, &timer_irqaction);
 	efi_gettimeofday((struct timeval *) &xtime);
 	ia64_init_itm();
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/sn/kernel/setup.c linux-2.4.21-pre5-030312/arch/ia64/sn/kernel/setup.c
--- linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/sn/kernel/setup.c	Wed Mar 12 17:24:55 2003
+++ linux-2.4.21-pre5-030312/arch/ia64/sn/kernel/setup.c	Sun Apr  6 17:52:15 2003
@@ -76,6 +76,7 @@
 
 extern void bte_init_node (nodepda_t *, cnodeid_t);
 extern void bte_init_cpu (void);
+extern void sn_timer_init (void);
 
 unsigned long sn_rtc_cycles_per_second;   
 unsigned long sn_rtc_usec_per_cyc;
@@ -313,6 +314,8 @@
 	 * Turn off "floating-point assist fault" warnings by default.
 	 */
 	current->thread.flags |= IA64_THREAD_FPEMU_NOPRINT;
+
+	sn_timer_init();
 }
 
 /**
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/sn/kernel/sn2/Makefile linux-2.4.21-pre5-030312/arch/ia64/sn/kernel/sn2/Makefile
--- linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/sn/kernel/sn2/Makefile	Wed Mar 12 17:24:55 2003
+++ linux-2.4.21-pre5-030312/arch/ia64/sn/kernel/sn2/Makefile	Fri Apr  4 17:25:34 2003
@@ -42,6 +42,6 @@
 
 O_TARGET        = sn2.o
 
-obj-y          = cache.o iomv.o ptc_deadlock.o sn2_smp.o sn_proc_fs.o
+obj-y          = cache.o iomv.o ptc_deadlock.o sn2_smp.o sn_proc_fs.o timer.o
 
 include $(TOPDIR)/Rules.make
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/sn/kernel/sn2/timer.c linux-2.4.21-pre5-030312/arch/ia64/sn/kernel/sn2/timer.c
--- linux-2.4.21-pre5-030312-pretimeofday/arch/ia64/sn/kernel/sn2/timer.c	Wed Dec 31 19:00:00 1969
+++ linux-2.4.21-pre5-030312/arch/ia64/sn/kernel/sn2/timer.c	Sun Apr  6 19:07:50 2003
@@ -0,0 +1,176 @@
+/*
+ * linux/arch/ia64/sn/kernel/sn2/timer.c
+ *
+ * Copyright (C) 2003 Silicon Graphics, Inc.
+ * Copyright (C) 1998-2001 Hewlett-Packard Co
+ * Copyright (C) 1998-2000 Stephane Eranian <eranian@hpl.hp.com>
+ * Copyright (C) 1999-2001 David Mosberger <davidm@hpl.hp.com>
+ * Copyright (C) 1999 Don Dugger <don.dugger@intel.com>
+ * Copyright (C) 1999-2000 VA Linux Systems
+ * Copyright (C) 1999-2000 Walt Drummond <drummond@valinux.com>
+ */
+
+#include <linux/config.h>
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/interrupt.h>
+#include <linux/efi.h>
+
+#include <asm/delay.h>
+#include <asm/hw_irq.h>
+#include <asm/ptrace.h>
+#include <asm/sal.h>
+#include <asm/system.h>
+
+#include <asm/sn/leds.h>
+#include <asm/sn/clksupport.h>
+
+extern rwlock_t xtime_lock;
+extern void do_profile (unsigned long ip);
+
+extern unsigned long sn_rtc_cycles_per_second;
+extern unsigned long sn_rtc_usec_per_cyc;
+static unsigned long sn_rtc_per_itc;
+static unsigned long sn_rtc_delta;
+static volatile unsigned long last_rtc_val;
+
+/**
+ * gettimeoffset - number of usecs elapsed since &xtime was last updated
+ *
+ * This function is used by do_gettimeofday() to determine the number
+ * of usecs that have elapsed since the last update to &xtime.  On SN
+ * this is accomplished using the RTC built in to each Hub chip; each
+ * is guaranteed to be synchronized by the PROM, so a local read will
+ * suffice (get_cycles() does this for us).  A snapshot of the RTC value
+ * is taken on every timer interrupt and this function more or less
+ * subtracts that snapshot value from the current value.
+ *
+ * Note that if a lot of processing was done during the last timer
+ * interrupt then &xtime may be some number of jiffies out of date.
+ * This function must account for that.
+ */
+extern unsigned long wall_jiffies; /* from kernel/timer.c */
+
+unsigned long
+sn2_gettimeoffset(void)
+{
+	unsigned long elapsed_rtc_cycles, lost = jiffies - wall_jiffies;
+	unsigned long local_last_rtc_val, current_rtc_val, last_tick;
+
+	local_last_rtc_val = last_rtc_val;
+	last_tick = local_last_rtc_val -
+		(((lost + 1)* sn_rtc_delta) >> SN_RTC_PER_ITC_SHIFT);
+	current_rtc_val = get_cycles();
+	if ((long)(current_rtc_val - last_tick) < 0) {
+		/* This happens when a timer interrupt occurred on cpu 0
+		 * and our local jiffies/wall_jiffies values are out of
+		 * sync.  The frlock around this code will detect the
+		 * inconsistency and try again */
+		return 0; 
+	}
+
+	elapsed_rtc_cycles = current_rtc_val - last_tick;
+	return (elapsed_rtc_cycles * (long)sn_rtc_usec_per_cyc) >>
+		IA64_USEC_PER_CYC_SHIFT; 
+}
+
+void
+sn2_timer_interrupt(int irq, void *dev_id, struct pt_regs *regs)
+{
+	unsigned long new_itm;
+
+	if (!pda.hb_count--) {
+		pda.hb_count = HZ/2;
+		pda.hb_state ^= LED_CPU_HEARTBEAT;
+		set_led_bits(pda.hb_state, LED_CPU_HEARTBEAT);
+	}
+
+	if (pda.pio_shub_war_cam_addr)
+		*pda.pio_shub_war_cam_addr = 0x8000000000000010UL;
+
+#if 1	/* XXX - what is this? - do we really need it? - Jes */
+#define SN_LB_INT_WAR_INTERVAL 100
+	if (pda.sn_lb_int_war_ticks = 0) {
+		sn_lb_int_war_check();
+	}
+	pda.sn_lb_int_war_ticks++;
+	if (pda.sn_lb_int_war_ticks >= SN_LB_INT_WAR_INTERVAL)
+		pda.sn_lb_int_war_ticks = 0;
+#endif
+
+	new_itm = local_cpu_data->itm_next;
+
+	if (!time_after(ia64_get_itc(), new_itm))
+		printk("Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
+		       ia64_get_itc(), new_itm);
+
+	while (1) {
+		/*
+		 * Do kernel PC profiling here. We multiply the instruction
+		 * number by four so that we can use a prof_shift of 2
+		 * to get instruction-level instead of just
+		 * bundle-level accuracy.
+		 */
+		if (!user_mode(regs))
+			do_profile(regs->cr_iip + 4*ia64_psr(regs)->ri);
+
+#ifdef CONFIG_SMP
+		smp_do_timer(regs);
+#endif
+		new_itm += local_cpu_data->itm_delta;
+
+		if (smp_processor_id() = 0) {
+			long last_rtc_del;
+			/*
+			 * Here we are in the timer irq handler. We have 
+			 * irqs locally disabled, but we don't know if the
+			 * timer_bh is running on another CPU. We need to
+			 * avoid to SMP race by acquiring the xtime_lock.
+			 */
+			write_lock(&xtime_lock);
+
+	  		/* Used by gettimeoffset on 'drifty' platforms */
+			last_rtc_del = ((((long)new_itm -
+					  (long)ia64_get_itc()) *
+					 (long)sn_rtc_per_itc) >>
+					SN_RTC_PER_ITC_SHIFT);
+			last_rtc_val = (unsigned long)last_rtc_del +
+				GET_RTC_COUNTER();
+
+			do_timer(regs);
+			local_cpu_data->itm_next = new_itm;
+			write_unlock(&xtime_lock);
+		} else
+			local_cpu_data->itm_next = new_itm;
+
+		if (time_after(new_itm, ia64_get_itc()))
+			break;
+	}
+
+	do {
+		/*
+		 * If we're too close to the next clock tick for comfort, we
+		 * increase the saftey margin by intentionally dropping the
+		 * next tick(s).  We do NOT update itm.next because that 
+		 * would force us to call do_timer() which in turn would let 
+		 * our clock run too fast (with the potentially devastating 
+		 * effect of losing monotony of time).
+		 */
+		while (!time_after(new_itm, ia64_get_itc() +
+				   local_cpu_data->itm_delta/2))
+			new_itm += local_cpu_data->itm_delta;
+		ia64_set_itm(new_itm);
+		/* double check, in case we got hit by a (slow) PMI: */
+	} while (time_after_eq(ia64_get_itc(), new_itm));
+}
+
+void __init
+sn_timer_init (void)
+{
+	sn_rtc_per_itc = (sn_rtc_cycles_per_second << SN_RTC_PER_ITC_SHIFT) /
+		local_cpu_data->itc_freq;
+	sn_rtc_delta = local_cpu_data->itm_delta * sn_rtc_per_itc;
+}
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/machvec.h linux-2.4.21-pre5-030312/include/asm-ia64/machvec.h
--- linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/machvec.h	Sun Mar 16 10:55:27 2003
+++ linux-2.4.21-pre5-030312/include/asm-ia64/machvec.h	Fri Apr  4 19:23:31 2003
@@ -64,6 +64,8 @@
 typedef void ia64_mv_outb_t (unsigned char, unsigned long);
 typedef void ia64_mv_outw_t (unsigned short, unsigned long);
 typedef void ia64_mv_outl_t (unsigned int, unsigned long);
+typedef unsigned long ia64_mv_gettimeoffset_t (void);
+typedef void ia64_mv_timer_interrupt_t(int, void *, struct pt_regs *);
 
 extern void machvec_noop (void);
 
@@ -115,6 +117,8 @@
 #  define platform_outb		ia64_mv.outb
 #  define platform_outw		ia64_mv.outw
 #  define platform_outl		ia64_mv.outl
+#  define platform_gettimeoffset	ia64_mv.gettimeoffset
+#  define platform_timer_interrupt	ia64_mv.timer_interrupt
 # endif
 
 struct ia64_machine_vector {
@@ -151,6 +155,8 @@
 	ia64_mv_outb_t *outb;
 	ia64_mv_outw_t *outw;
 	ia64_mv_outl_t *outl;
+	ia64_mv_gettimeoffset_t *gettimeoffset;
+	ia64_mv_timer_interrupt_t *timer_interrupt;
 };
 
 #define MACHVEC_INIT(name)			\
@@ -187,7 +193,9 @@
 	platform_inl,				\
 	platform_outb,				\
 	platform_outw,				\
-	platform_outl				\
+	platform_outl,				\
+	platform_gettimeoffset,			\
+	platform_timer_interrupt		\
 }
 
 extern struct ia64_machine_vector ia64_mv;
@@ -211,6 +219,8 @@
 extern ia64_mv_pci_dma_sync_single swiotlb_sync_single;
 extern ia64_mv_pci_dma_sync_sg swiotlb_sync_sg;
 extern ia64_mv_pci_dma_supported swiotlb_pci_dma_supported;
+extern ia64_mv_gettimeoffset_t __ia64_gettimeoffset;
+extern void __ia64_timer_interrupt(int, void *, struct pt_regs *);
 
 /*
  * Define default versions so we can extend machvec for new platforms without having
@@ -309,5 +319,11 @@
 #ifndef platform_outl
 # define platform_outl		__ia64_outl
 #endif
+#ifndef platform_gettimeoffset
+# define platform_gettimeoffset	__ia64_gettimeoffset
+#endif
+#ifndef platform_timer_interrupt
+# define platform_timer_interrupt	__ia64_timer_interrupt
+#endif
 
 #endif /* _ASM_IA64_MACHVEC_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/machvec_sn2.h linux-2.4.21-pre5-030312/include/asm-ia64/machvec_sn2.h
--- linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/machvec_sn2.h	Sun Mar 16 10:55:27 2003
+++ linux-2.4.21-pre5-030312/include/asm-ia64/machvec_sn2.h	Fri Apr  4 19:23:57 2003
@@ -59,6 +59,8 @@
 extern ia64_mv_pci_dma_sync_single	sn_pci_dma_sync_single;
 extern ia64_mv_pci_dma_sync_sg		sn_pci_dma_sync_sg;
 extern ia64_mv_pci_dma_supported	sn_pci_dma_supported;
+extern ia64_mv_gettimeoffset_t		sn_gettimeoffset;
+extern ia64_mv_timer_interrupt_t	sn_timer_interrupt;
 
 /*
  * This stuff has dual use!
@@ -95,5 +97,7 @@
 #define platform_pci_dma_sync_single	sn_pci_dma_sync_single
 #define platform_pci_dma_sync_sg	sn_pci_dma_sync_sg
 #define platform_pci_dma_supported	sn_pci_dma_supported
+#define platform_gettimeoffset		sn_gettimeoffset
+#define platform_timer_interrupt	sn_timer_interrupt
 
 #endif /* _ASM_IA64_MACHVEC_SN2_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/sn/clksupport.h linux-2.4.21-pre5-030312/include/asm-ia64/sn/clksupport.h
--- linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/sn/clksupport.h	Wed Mar 12 17:24:59 2003
+++ linux-2.4.21-pre5-030312/include/asm-ia64/sn/clksupport.h	Sun Apr  6 18:59:05 2003
@@ -38,7 +38,7 @@
 
 extern nasid_t master_nasid;
 
-#define RTC_MASK		(0x007fffffffffffff)
+#define RTC_MASK		0x007fffffffffffffUL
 /* clocks are not synchronized yet on SN1  - used node 0 (problem if no NASID 0) */
 #define RTC_COUNTER_ADDR	((clkreg_t*)REMOTE_HUB_ADDR(master_nasid, PI_RT_COUNTER))
 #define RTC_COMPARE_A_ADDR      ((clkreg_t*)REMOTE_HUB_ADDR(master_nasid, PI_RT_COMPARE_A))
@@ -52,7 +52,7 @@
 #include <asm/sn/sn2/addrs.h>
 #include <asm/sn/sn2/shubio.h>
 #include <asm/sn/sn2/shub_mmr.h>
-#define RTC_MASK		(SH_RTC_MASK)
+#define RTC_MASK		SH_RTC_MASK
 #define RTC_COUNTER_ADDR	((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #define RTC_COMPARE_A_ADDR      ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #define RTC_COMPARE_B_ADDR      ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
@@ -62,7 +62,7 @@
 #define RTC_INT_ENABLED_B_ADDR  ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #endif	/* CONFIG_IA64_SGI_SN1 */
 
-
+#define SN_RTC_PER_ITC_SHIFT	34
 #define GET_RTC_COUNTER()	(*RTC_COUNTER_ADDR)
 #define rtc_time()		GET_RTC_COUNTER()
 
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/timex.h linux-2.4.21-pre5-030312/include/asm-ia64/timex.h
--- linux-2.4.21-pre5-030312-pretimeofday/include/asm-ia64/timex.h	Thu Nov 28 18:53:15 2002
+++ linux-2.4.21-pre5-030312/include/asm-ia64/timex.h	Fri Apr  4 15:07:17 2003
@@ -24,4 +24,6 @@
 #define vxtime_lock()		do {} while (0)
 #define vxtime_unlock()		do {} while (0)
 
+#define gettimeoffset()		platform_gettimeoffset()
+
 #endif /* _ASM_IA64_TIMEX_H */

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[David Mosberger]

>>>>> On Tue, 8 Apr 2003 16:35:56 -0400, Jes Sorensen <jes@wildopensource.com> said:

  Jes> Hi, On systems with a drifting ITC, such as the SGI SN2, the
  Jes> current implementation of timer_interrupt() and gettimeoffset()
  Jes> wont work as they rely heavily on ar.itc.

  Jes> On SN2 this has been solved by using the synchronized RTC in
  Jes> the system, which results a different implementation of the two
  Jes> above mentioned functions.

  Jes> To get around this I suggest we change gettimeoffset() and
  Jes> timer_intererupt() to be machvec's.

I suspect we're better off in the long term making gettimeoffset() a
function pointer.  I can certainly imagine platforms where multiple
interpolation sources are available and depending on parameters (e.g.,
long-term stability or overhead), one might want to choose one or
another.

The motivation for changing timer_interrupt() seems suspect to me.
The timer interupt is generated PER CPU, so there is no drift issue at
all (in fact, we intentionally _skew_ things at boot time to reduce
the likelihood of getting all timer interrupts at the same time).

	--david

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Jesse Barnes]

On Tue, Apr 08, 2003 at 02:41:52PM -0700, David Mosberger wrote:
> I suspect we're better off in the long term making gettimeoffset() a
> function pointer.  I can certainly imagine platforms where multiple
> interpolation sources are available and depending on parameters (e.g.,
> long-term stability or overhead), one might want to choose one or
> another.

I agree, since it'll probably refer to a platform specific RTC.

> The motivation for changing timer_interrupt() seems suspect to me.
> The timer interupt is generated PER CPU, so there is no drift issue at
> all (in fact, we intentionally _skew_ things at boot time to reduce
> the likelihood of getting all timer interrupts at the same time).

The problem is that we want high resolution gettimeofday, which means
that we need to know how long its been since wall time was updated
when gettimeoffset is called.  We do that right now by snapshoting the
RTC in timer_interrupt...

Jesse

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[David Mosberger]

>>>>> On Tue, 8 Apr 2003 14:49:25 -0700, Jesse Barnes <jbarnes@sgi.com> said:

  >> The motivation for changing timer_interrupt() seems suspect to me.
  >> The timer interupt is generated PER CPU, so there is no drift issue at
  >> all (in fact, we intentionally _skew_ things at boot time to reduce
  >> the likelihood of getting all timer interrupts at the same time).

  Jesse> The problem is that we want high resolution gettimeofday, which means
  Jesse> that we need to know how long its been since wall time was updated
  Jesse> when gettimeoffset is called.  We do that right now by snapshoting the
  Jesse> RTC in timer_interrupt...

Clearly you need to establish the relationship between the external
clock and time-of-day somewhere, but I'm not so sure this should be
done in the arch-specific timer interrupt handler.  I suspect you
really want to do it where the time-of-day gets updated.  Also, I
think this should be treated much more as a driver issue rather than a
platform-issue (suppose someone plugged in an adapter card providing a
low-latency, atomic accurracy & high precision lock, you'd presumably
want to be able to use that card in favor of whatever other hardware
might be there.

	--david

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Grant Grundler]

On Wed, Apr 09, 2003 at 10:53:20AM -0700, David Mosberger wrote:
> Also, I think this should be treated much more as a driver issue rather
> than a platform-issue

In general, I like the idea.  It's really three peices of HW
involved: the RTC, the interval timer, and high resolution support.

The RTC is/has a driver in most cases it seems.

The interval timer gets started very early (bogomips) by
arch code. A seperate driver could work here which "claims"
the (per CPU) interval timer, registers the timer interrupt
handler, and syncronizes the multiple timers. But someone would
need to unravel dependencies and fix the init sequence accordingly.
Sounds like a pretty ugly task.

(Occasionally I try to figure out a timer problem that shows up on some
SMP parisc-linux boxes and then give up in the morass of interactions.)

The high resolution support is typically based on a cycle counter
in the CPU - very arch specific. But that probably doesn't need
to change (much) or become a driver.

> (suppose someone plugged in an adapter card providing a
> low-latency, atomic accurracy & high precision lock, you'd presumably
> want to be able to use that card in favor of whatever other hardware
> might be there.

yup.

thanks,
grant

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Jesse Barnes]

On Wed, Apr 09, 2003 at 10:53:20AM -0700, David Mosberger wrote:
> Clearly you need to establish the relationship between the external
> clock and time-of-day somewhere, but I'm not so sure this should be
> done in the arch-specific timer interrupt handler.  I suspect you
> really want to do it where the time-of-day gets updated.  Also, I

I'd like for it to be there too.  Jes and I discussed this on irc and
I think he's going to take a stab at it.

> think this should be treated much more as a driver issue rather than a
> platform-issue (suppose someone plugged in an adapter card providing a
> low-latency, atomic accurracy & high precision lock, you'd presumably
> want to be able to use that card in favor of whatever other hardware
> might be there.

Agreed.  It would be nice to have that kind of flexibility.

Thanks,
Jesse

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Jes Sorensen]

>>>>> "David" = David Mosberger <davidm@napali.hpl.hp.com> writes:

>>>>> On Tue, 8 Apr 2003 14:49:25 -0700, Jesse Barnes <jbarnes@sgi.com> said:
Jesse> The problem is that we want high resolution gettimeofday, which
Jesse> means that we need to know how long its been since wall time
Jesse> was updated when gettimeoffset is called.  We do that right now
Jesse> by snapshoting the RTC in timer_interrupt...

David> Clearly you need to establish the relationship between the
David> external clock and time-of-day somewhere, but I'm not so sure
David> this should be done in the arch-specific timer interrupt
David> handler.  I suspect you really want to do it where the
David> time-of-day gets updated.  Also, I think this should be treated
David> much more as a driver issue rather than a platform-issue
David> (suppose someone plugged in an adapter card providing a
David> low-latency, atomic accurracy & high precision lock, you'd
David> presumably want to be able to use that card in favor of
David> whatever other hardware might be there.

Hi David,

Here is an attempt to do just that using the SN2 RTC timer by
providing a hook one can plug into where wall time is updated. This
patch is against Bjorn's tree as of a little while ago and leaves the
old concept in place for the ITC codepath - not sure if we want to try
and convert that over.

Comments? If you think this is a better solution, I'll go ahead and
forward port it to 2.5 as well.

Cheers,
Jes

diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/kernel/time.c linux-2.4.21-hack-timer/arch/ia64/kernel/time.c
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/kernel/time.c	Thu Nov 28 18:53:09 2002
+++ linux-2.4.21-hack-timer/arch/ia64/kernel/time.c	Thu Apr 17 13:06:14 2003
@@ -61,8 +61,8 @@
  * Return the number of micro-seconds that elapsed since the last update to jiffy.  The
  * xtime_lock must be at least read-locked when calling this routine.
  */
-static inline unsigned long
-gettimeoffset (void)
+unsigned long
+__ia64_gettimeoffset (void)
 {
 	unsigned long elapsed_cycles, lost = jiffies - wall_jiffies;
 	unsigned long now, last_tick;
@@ -102,6 +102,9 @@
 			tv->tv_sec--;
 		}
 
+		if (update_wall_time_plug)
+			update_wall_time_plug();
+
 		xtime = *tv;
 		time_adjust = 0;		/* stop active adjtime() */
 		time_status |= STA_UNSYNC;
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/setup.c linux-2.4.21-hack-timer/arch/ia64/sn/kernel/setup.c
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/setup.c	Sun Apr  6 17:14:59 2003
+++ linux-2.4.21-hack-timer/arch/ia64/sn/kernel/setup.c	Thu Apr 17 17:26:09 2003
@@ -76,6 +76,7 @@
 
 extern void bte_init_node (nodepda_t *, cnodeid_t);
 extern void bte_init_cpu (void);
+extern void sn_timer_init (void);
 
 unsigned long sn_rtc_cycles_per_second;   
 unsigned long sn_rtc_usec_per_cyc;
@@ -313,6 +314,8 @@
 	 * Turn off "floating-point assist fault" warnings by default.
 	 */
 	current->thread.flags |= IA64_THREAD_FPEMU_NOPRINT;
+
+	sn_timer_init();
 }
 
 /**
@@ -510,54 +513,6 @@
 	return GET_RTC_COUNTER();
 }
 
-/**
- * gettimeoffset - number of usecs elapsed since &xtime was last updated
- *
- * This function is used by do_gettimeofday() to determine the number
- * of usecs that have elapsed since the last update to &xtime.  On SN
- * this is accomplished using the RTC built in to each Hub chip; each
- * is guaranteed to be synchronized by the PROM, so a local read will
- * suffice (get_cycles() does this for us).  A snapshot of the RTC value
- * is taken on every timer interrupt and this function more or less
- * subtracts that snapshot value from the current value.
- *
- * Note that if a lot of processing was done during the last timer
- * interrupt then &xtime may be some number of jiffies out of date.
- * This function must account for that.
- */
-unsigned long
-gettimeoffset(void)
-{
-	unsigned long current_rtc_val, local_last_rtc_val;
-	unsigned long usec;
-
-	local_last_rtc_val = last_rtc_val;
-	current_rtc_val = get_cycles();
-	usec = last_itc_lost_usec;
-
-	/* If the RTC has wrapped around, compensate */
-	if (unlikely(current_rtc_val < local_last_rtc_val)) {
-		printk(KERN_NOTICE "RTC wrapped cpu:%d current:0x%lx last:0x%lx\n",
-				smp_processor_id(), current_rtc_val,
-				local_last_rtc_val);
-		current_rtc_val += RTC_MASK;
-	}
-
-	usec += ((current_rtc_val - local_last_rtc_val)*sn_rtc_usec_per_cyc) >> 
-		IA64_USEC_PER_CYC_SHIFT;
-
-	/*
-	 * usec is the number of microseconds into the current clock interval. Every
-	 * clock tick, xtime is advanced by "tick" microseconds. If "usec"
-	 * is allowed to get larger than "tick", the time value returned by gettimeofday
-	 * will go backward.
-	 */
-	if (usec >= tick)
-		usec = tick-1;
-
-	return usec;
-}
-
 #ifdef II_PRTE_TLB_WAR
 long iiprt_lock[16*64] __cacheline_aligned; /* allow for NASIDs up to 64 */
 #endif
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/Makefile linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/Makefile
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/Makefile	Sun Apr  6 17:14:59 2003
+++ linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/Makefile	Tue Apr 15 19:37:40 2003
@@ -42,6 +42,6 @@
 
 O_TARGET        = sn2.o
 
-obj-y          = cache.o iomv.o ptc_deadlock.o sn2_smp.o sn_proc_fs.o
+obj-y          = cache.o iomv.o ptc_deadlock.o sn2_smp.o sn_proc_fs.o timer.o
 
 include $(TOPDIR)/Rules.make
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/timer.c linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/timer.c
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/timer.c	Wed Dec 31 19:00:00 1969
+++ linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/timer.c	Thu Apr 17 17:30:42 2003
@@ -0,0 +1,83 @@
+/*
+ * linux/arch/ia64/sn/kernel/sn2/timer.c
+ *
+ * Copyright (C) 2003 Silicon Graphics, Inc.
+ */
+
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/interrupt.h>
+#include <linux/efi.h>
+
+#include <asm/delay.h>
+#include <asm/hw_irq.h>
+#include <asm/ptrace.h>
+#include <asm/sal.h>
+#include <asm/system.h>
+
+#include <asm/sn/leds.h>
+#include <asm/sn/clksupport.h>
+
+extern rwlock_t xtime_lock;
+
+extern unsigned long sn_rtc_cycles_per_second;
+extern unsigned long sn_rtc_usec_per_cyc;
+static unsigned long sn_rtc_per_itc;
+static unsigned long sn_rtc_delta;
+static volatile unsigned long last_rtc_val;
+
+/**
+ * gettimeoffset - number of usecs elapsed since &xtime was last updated
+ *
+ * This function is used by do_gettimeofday() to determine the number
+ * of usecs that have elapsed since the last update to &xtime.  On SN
+ * this is accomplished using the RTC built in to each Hub chip; each
+ * is guaranteed to be synchronized by the PROM, so a local read will
+ * suffice (GET_RTC_COUNTER() does this for us).  A snapshot of the RTC
+ * value is taken every time wall_jiffies is updated by the
+ * update_wall_time_plug (sn2_update_wall_time) which means we don't
+ * have to adjust for lost jiffies ticks or anything like that.
+ */
+unsigned long
+sn_gettimeoffset(void)
+{
+	unsigned long current_rtc;
+	long elapsed_rtc;
+
+	current_rtc = GET_RTC_COUNTER();
+
+	/*
+	 * Need to address wrapping here!
+	 */
+	elapsed_rtc = (long)(current_rtc - last_wall_rtc);
+
+	if (elapsed_rtc < 0) {
+		printk(KERN_INFO "sn_gettimeoffset(): time goes backwards! "
+		       "current_rtc 0x%016lx, last_wall_rtc 0x%016lx\n",
+		       current_rtc, last_wall_rtc);
+	}
+
+	return (elapsed_rtc * (long)sn_rtc_usec_per_cyc) >>
+		IA64_USEC_PER_CYC_SHIFT; 
+}
+
+
+void sn2_update_wall_time()
+{
+	last_wall_rtc = GET_RTC();
+}
+
+
+void __init
+sn_timer_init (void)
+{
+	sn_rtc_per_itc = (sn_rtc_cycles_per_second << SN_RTC_PER_ITC_SHIFT) /
+		local_cpu_data->itc_freq;
+	sn_rtc_delta = local_cpu_data->itm_delta * sn_rtc_per_itc;
+
+	last_wall_rtc = GET_RTC_COUNTER();
+	update_wall_time_plug = sn2_update_wall_time;
+}
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/machvec.h linux-2.4.21-hack-timer/include/asm-ia64/machvec.h
--- linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/machvec.h	Sun Apr  6 17:37:58 2003
+++ linux-2.4.21-hack-timer/include/asm-ia64/machvec.h	Tue Apr 15 19:37:40 2003
@@ -63,6 +63,7 @@
 typedef void ia64_mv_outb_t (unsigned char, unsigned long);
 typedef void ia64_mv_outw_t (unsigned short, unsigned long);
 typedef void ia64_mv_outl_t (unsigned int, unsigned long);
+typedef unsigned long ia64_mv_gettimeoffset_t (void);
 
 extern void machvec_noop (void);
 
@@ -113,6 +114,7 @@
 #  define platform_outb		ia64_mv.outb
 #  define platform_outw		ia64_mv.outw
 #  define platform_outl		ia64_mv.outl
+#  define platform_gettimeoffset	ia64_mv.gettimeoffset
 # endif
 
 struct ia64_machine_vector {
@@ -148,6 +150,7 @@
 	ia64_mv_outb_t *outb;
 	ia64_mv_outw_t *outw;
 	ia64_mv_outl_t *outl;
+	ia64_mv_gettimeoffset_t *gettimeoffset;
 };
 
 #define MACHVEC_INIT(name)			\
@@ -183,7 +186,8 @@
 	platform_inl,				\
 	platform_outb,				\
 	platform_outw,				\
-	platform_outl				\
+	platform_outl,				\
+	platform_gettimeoffset			\
 }
 
 extern struct ia64_machine_vector ia64_mv;
@@ -206,6 +210,7 @@
 extern ia64_mv_pci_dma_sync_single swiotlb_sync_single;
 extern ia64_mv_pci_dma_sync_sg swiotlb_sync_sg;
 extern ia64_mv_pci_dma_supported swiotlb_pci_dma_supported;
+extern ia64_mv_gettimeoffset_t __ia64_gettimeoffset;
 
 /*
  * Define default versions so we can extend machvec for new platforms without having
@@ -301,5 +306,8 @@
 #ifndef platform_outl
 # define platform_outl		__ia64_outl
 #endif
+#ifndef platform_gettimeoffset
+# define platform_gettimeoffset	__ia64_gettimeoffset
+#endif
 
 #endif /* _ASM_IA64_MACHVEC_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/machvec_sn2.h linux-2.4.21-hack-timer/include/asm-ia64/machvec_sn2.h
--- linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/machvec_sn2.h	Sun Apr  6 17:15:02 2003
+++ linux-2.4.21-hack-timer/include/asm-ia64/machvec_sn2.h	Tue Apr 15 19:37:40 2003
@@ -58,6 +58,7 @@
 extern ia64_mv_pci_dma_sync_single	sn_pci_dma_sync_single;
 extern ia64_mv_pci_dma_sync_sg		sn_pci_dma_sync_sg;
 extern ia64_mv_pci_dma_supported	sn_pci_dma_supported;
+extern ia64_mv_gettimeoffset_t		sn_gettimeoffset;
 
 /*
  * This stuff has dual use!
@@ -93,5 +94,6 @@
 #define platform_pci_dma_sync_single	sn_pci_dma_sync_single
 #define platform_pci_dma_sync_sg	sn_pci_dma_sync_sg
 #define platform_pci_dma_supported	sn_pci_dma_supported
+#define platform_gettimeoffset		sn_gettimeoffset
 
 #endif /* _ASM_IA64_MACHVEC_SN2_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/sn/clksupport.h linux-2.4.21-hack-timer/include/asm-ia64/sn/clksupport.h
--- linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/sn/clksupport.h	Sun Apr  6 17:15:02 2003
+++ linux-2.4.21-hack-timer/include/asm-ia64/sn/clksupport.h	Tue Apr 15 19:37:40 2003
@@ -38,7 +38,7 @@
 
 extern nasid_t master_nasid;
 
-#define RTC_MASK		(0x007fffffffffffff)
+#define RTC_MASK		0x007fffffffffffffUL
 /* clocks are not synchronized yet on SN1  - used node 0 (problem if no NASID 0) */
 #define RTC_COUNTER_ADDR	((clkreg_t*)REMOTE_HUB_ADDR(master_nasid, PI_RT_COUNTER))
 #define RTC_COMPARE_A_ADDR      ((clkreg_t*)REMOTE_HUB_ADDR(master_nasid, PI_RT_COMPARE_A))
@@ -52,7 +52,7 @@
 #include <asm/sn/sn2/addrs.h>
 #include <asm/sn/sn2/shubio.h>
 #include <asm/sn/sn2/shub_mmr.h>
-#define RTC_MASK		(SH_RTC_MASK)
+#define RTC_MASK		SH_RTC_MASK
 #define RTC_COUNTER_ADDR	((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #define RTC_COMPARE_A_ADDR      ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #define RTC_COMPARE_B_ADDR      ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
@@ -62,7 +62,7 @@
 #define RTC_INT_ENABLED_B_ADDR  ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #endif	/* CONFIG_IA64_SGI_SN1 */
 
-
+#define SN_RTC_PER_ITC_SHIFT	34
 #define GET_RTC_COUNTER()	(*RTC_COUNTER_ADDR)
 #define rtc_time()		GET_RTC_COUNTER()
 
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/timex.h linux-2.4.21-hack-timer/include/asm-ia64/timex.h
--- linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/timex.h	Thu Nov 28 18:53:15 2002
+++ linux-2.4.21-hack-timer/include/asm-ia64/timex.h	Tue Apr 15 19:37:40 2003
@@ -24,4 +24,6 @@
 #define vxtime_lock()		do {} while (0)
 #define vxtime_unlock()		do {} while (0)
 
+#define gettimeoffset()		platform_gettimeoffset()
+
 #endif /* _ASM_IA64_TIMEX_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/linux/timex.h linux-2.4.21-hack-timer/include/linux/timex.h
--- linux-2.4.21-pre5-cset-1.1020/include/linux/timex.h	Sun Apr  6 17:37:58 2003
+++ linux-2.4.21-hack-timer/include/linux/timex.h	Thu Apr 17 13:07:29 2003
@@ -286,6 +286,12 @@
 extern long pps_errcnt;		/* calibration errors */
 extern long pps_stbcnt;		/* stability limit exceeded */
 
+/*
+ * Call-back for high precision timer sources to snapshot every time
+ * wall_jiffies is updated.
+ */
+extern void (*update_wall_time_plug)(void);
+
 #endif /* KERNEL */
 
 #endif /* LINUX_TIMEX_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/kernel/timer.c linux-2.4.21-hack-timer/kernel/timer.c
--- linux-2.4.21-pre5-cset-1.1020/kernel/timer.c	Sun Apr  6 17:17:16 2003
+++ linux-2.4.21-hack-timer/kernel/timer.c	Tue Apr 15 19:28:54 2003
@@ -514,6 +514,14 @@
 }
 
 /*
+ * Hook for using external high precision timers for the system clock.
+ * On systems where the CPU clock isn't synchronized between CPUs,
+ * it is necessary to use an external source such as an RTC to obtain
+ * precision in gettimeofday().
+ */
+void (*update_wall_time_plug)(void) = NULL;
+
+/*
  * Using a loop looks inefficient, but "ticks" is
  * usually just one (we shouldn't be losing ticks,
  * we're doing this this way mainly for interrupt
@@ -522,6 +530,9 @@
  */
 static void update_wall_time(unsigned long ticks)
 {
+	if (update_wall_time_plug)
+		update_wall_time_plug();
+
 	do {
 		ticks--;
 		update_wall_time_one_tick();

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[David Mosberger]

>>>>> On 17 Apr 2003 19:02:26 -0400, Jes Sorensen <jes@wildopensource.com> said:

  Jes> Here is an attempt to do just that using the SN2 RTC timer by
  Jes> providing a hook one can plug into where wall time is updated. This
  Jes> patch is against Bjorn's tree as of a little while ago and leaves the
  Jes> old concept in place for the ITC codepath - not sure if we want to try
  Jes> and convert that over.

  Jes> Comments? If you think this is a better solution, I'll go ahead and
  Jes> forward port it to 2.5 as well.

I thought we agreed that gettimeoffset() should just be a function
pointer?  Also, in 2.5, there is already a "clock_was_set()" call-back
(for the POSIX timers).  Perhaps that should be considered.

	--david

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Jes Sorensen]

>>>>> "David" = David Mosberger <davidm@napali.hpl.hp.com> writes:

>>>>> On 17 Apr 2003 19:02:26 -0400, Jes Sorensen <jes@wildopensource.com> said:
Jes> Comments? If you think this is a better solution, I'll go ahead
Jes> and forward port it to 2.5 as well.

David> I thought we agreed that gettimeoffset() should just be a
David> function pointer?  Also, in 2.5, there is already a
David> "clock_was_set()" call-back (for the POSIX timers).  Perhaps
David> that should be considered.

Is there a real difference between using a plain function pointer and
a machvec? Admittedly I have just been considering machvec's as
function pointers with a fancy name, however I'll be happy to change
it to a regular function pointer instead.

I'll take a look at clock_was_set().

Thanks,
Jes

Re: [Linux-ia64] [patch] 2.4 timer_interrupt/gettimeoffset machvec

[Jes Sorensen]

>>>>> "David" = David Mosberger <davidm@napali.hpl.hp.com> writes:

>>>>> On 17 Apr 2003 19:02:26 -0400, Jes Sorensen <jes@wildopensource.com> said:
Jes> Comments? If you think this is a better solution, I'll go ahead
Jes> and forward port it to 2.5 as well.

David> I thought we agreed that gettimeoffset() should just be a
David> function pointer?  Also, in 2.5, there is already a
David> "clock_was_set()" call-back (for the POSIX timers).  Perhaps
David> that should be considered.

Hi David,

I had a look at clock_was_set() and it really serves a different
purpose, which is to wake up sleeping tasks waiting for the
clock. Being called outside the xtime_lock points means that in a
preemptive kernel this would fail for us.

Anyway here is a new version of the patch, all traces of machvec's are
gone ;-) Anything else you'd like me to change?

Cheers,
Jes

diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/kernel/time.c linux-2.4.21-hack-timer/arch/ia64/kernel/time.c
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/kernel/time.c	Thu Nov 28 18:53:09 2002
+++ linux-2.4.21-hack-timer/arch/ia64/kernel/time.c	Thu Apr 17 20:23:23 2003
@@ -26,6 +26,9 @@
 extern rwlock_t xtime_lock;
 extern unsigned long wall_jiffies;
 extern unsigned long last_time_offset;
+unsigned long __ia64_gettimeoffset (void);
+
+unsigned long (*gettimeoffset)(void) = &__ia64_gettimeoffset;
 
 #ifdef CONFIG_IA64_DEBUG_IRQ
 
@@ -61,8 +64,8 @@
  * Return the number of micro-seconds that elapsed since the last update to jiffy.  The
  * xtime_lock must be at least read-locked when calling this routine.
  */
-static inline unsigned long
-gettimeoffset (void)
+unsigned long
+__ia64_gettimeoffset (void)
 {
 	unsigned long elapsed_cycles, lost = jiffies - wall_jiffies;
 	unsigned long now, last_tick;
@@ -102,6 +105,9 @@
 			tv->tv_sec--;
 		}
 
+		if (update_wall_time_plug)
+			update_wall_time_plug();
+
 		xtime = *tv;
 		time_adjust = 0;		/* stop active adjtime() */
 		time_status |= STA_UNSYNC;
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/setup.c linux-2.4.21-hack-timer/arch/ia64/sn/kernel/setup.c
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/setup.c	Sun Apr  6 17:14:59 2003
+++ linux-2.4.21-hack-timer/arch/ia64/sn/kernel/setup.c	Thu Apr 17 17:26:09 2003
@@ -76,6 +76,7 @@
 
 extern void bte_init_node (nodepda_t *, cnodeid_t);
 extern void bte_init_cpu (void);
+extern void sn_timer_init (void);
 
 unsigned long sn_rtc_cycles_per_second;   
 unsigned long sn_rtc_usec_per_cyc;
@@ -313,6 +314,8 @@
 	 * Turn off "floating-point assist fault" warnings by default.
 	 */
 	current->thread.flags |= IA64_THREAD_FPEMU_NOPRINT;
+
+	sn_timer_init();
 }
 
 /**
@@ -510,54 +513,6 @@
 	return GET_RTC_COUNTER();
 }
 
-/**
- * gettimeoffset - number of usecs elapsed since &xtime was last updated
- *
- * This function is used by do_gettimeofday() to determine the number
- * of usecs that have elapsed since the last update to &xtime.  On SN
- * this is accomplished using the RTC built in to each Hub chip; each
- * is guaranteed to be synchronized by the PROM, so a local read will
- * suffice (get_cycles() does this for us).  A snapshot of the RTC value
- * is taken on every timer interrupt and this function more or less
- * subtracts that snapshot value from the current value.
- *
- * Note that if a lot of processing was done during the last timer
- * interrupt then &xtime may be some number of jiffies out of date.
- * This function must account for that.
- */
-unsigned long
-gettimeoffset(void)
-{
-	unsigned long current_rtc_val, local_last_rtc_val;
-	unsigned long usec;
-
-	local_last_rtc_val = last_rtc_val;
-	current_rtc_val = get_cycles();
-	usec = last_itc_lost_usec;
-
-	/* If the RTC has wrapped around, compensate */
-	if (unlikely(current_rtc_val < local_last_rtc_val)) {
-		printk(KERN_NOTICE "RTC wrapped cpu:%d current:0x%lx last:0x%lx\n",
-				smp_processor_id(), current_rtc_val,
-				local_last_rtc_val);
-		current_rtc_val += RTC_MASK;
-	}
-
-	usec += ((current_rtc_val - local_last_rtc_val)*sn_rtc_usec_per_cyc) >> 
-		IA64_USEC_PER_CYC_SHIFT;
-
-	/*
-	 * usec is the number of microseconds into the current clock interval. Every
-	 * clock tick, xtime is advanced by "tick" microseconds. If "usec"
-	 * is allowed to get larger than "tick", the time value returned by gettimeofday
-	 * will go backward.
-	 */
-	if (usec >= tick)
-		usec = tick-1;
-
-	return usec;
-}
-
 #ifdef II_PRTE_TLB_WAR
 long iiprt_lock[16*64] __cacheline_aligned; /* allow for NASIDs up to 64 */
 #endif
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/Makefile linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/Makefile
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/Makefile	Sun Apr  6 17:14:59 2003
+++ linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/Makefile	Tue Apr 15 19:37:40 2003
@@ -42,6 +42,6 @@
 
 O_TARGET        = sn2.o
 
-obj-y          = cache.o iomv.o ptc_deadlock.o sn2_smp.o sn_proc_fs.o
+obj-y          = cache.o iomv.o ptc_deadlock.o sn2_smp.o sn_proc_fs.o timer.o
 
 include $(TOPDIR)/Rules.make
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/timer.c linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/timer.c
--- linux-2.4.21-pre5-cset-1.1020/arch/ia64/sn/kernel/sn2/timer.c	Wed Dec 31 19:00:00 1969
+++ linux-2.4.21-hack-timer/arch/ia64/sn/kernel/sn2/timer.c	Thu Apr 17 20:49:15 2003
@@ -0,0 +1,85 @@
+/*
+ * linux/arch/ia64/sn/kernel/sn2/timer.c
+ *
+ * Copyright (C) 2003 Silicon Graphics, Inc.
+ */
+
+#include <linux/config.h>
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/interrupt.h>
+#include <linux/efi.h>
+
+#include <asm/delay.h>
+#include <asm/hw_irq.h>
+#include <asm/ptrace.h>
+#include <asm/sal.h>
+#include <asm/system.h>
+
+#include <asm/sn/leds.h>
+#include <asm/sn/clksupport.h>
+
+extern rwlock_t xtime_lock;
+
+extern unsigned long sn_rtc_cycles_per_second;
+extern unsigned long sn_rtc_usec_per_cyc;
+static unsigned long sn_rtc_per_itc;
+static unsigned long sn_rtc_delta;
+static volatile unsigned long last_rtc_val;
+
+/**
+ * gettimeoffset - number of usecs elapsed since &xtime was last updated
+ *
+ * This function is used by do_gettimeofday() to determine the number
+ * of usecs that have elapsed since the last update to &xtime.  On SN
+ * this is accomplished using the RTC built in to each Hub chip; each
+ * is guaranteed to be synchronized by the PROM, so a local read will
+ * suffice (GET_RTC_COUNTER() does this for us).  A snapshot of the RTC
+ * value is taken every time wall_jiffies is updated by the
+ * update_wall_time_plug (sn2_update_wall_time) which means we don't
+ * have to adjust for lost jiffies ticks or anything like that.
+ */
+unsigned long
+sn_gettimeoffset(void)
+{
+	unsigned long current_rtc;
+	long elapsed_rtc;
+
+	current_rtc = GET_RTC_COUNTER();
+
+	/*
+	 * Need to address wrapping here!
+	 */
+	elapsed_rtc = (long)(current_rtc - last_wall_rtc);
+
+	if (elapsed_rtc < 0) {
+		printk(KERN_INFO "sn_gettimeoffset(): time goes backwards! "
+		       "current_rtc 0x%016lx, last_wall_rtc 0x%016lx\n",
+		       current_rtc, last_wall_rtc);
+	}
+
+	return (elapsed_rtc * (long)sn_rtc_usec_per_cyc) >>
+		IA64_USEC_PER_CYC_SHIFT; 
+}
+
+
+void sn2_update_wall_time()
+{
+	last_wall_rtc = GET_RTC();
+}
+
+
+void __init
+sn_timer_init (void)
+{
+	sn_rtc_per_itc = (sn_rtc_cycles_per_second << SN_RTC_PER_ITC_SHIFT) /
+		local_cpu_data->itc_freq;
+	sn_rtc_delta = local_cpu_data->itm_delta * sn_rtc_per_itc;
+
+	last_wall_rtc = GET_RTC_COUNTER();
+	update_wall_time_plug = sn2_update_wall_time;
+	gettimeoffset = sn_gettimeoffset;
+}
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/sn/clksupport.h linux-2.4.21-hack-timer/include/asm-ia64/sn/clksupport.h
--- linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/sn/clksupport.h	Sun Apr  6 17:15:02 2003
+++ linux-2.4.21-hack-timer/include/asm-ia64/sn/clksupport.h	Tue Apr 15 19:37:40 2003
@@ -38,7 +38,7 @@
 
 extern nasid_t master_nasid;
 
-#define RTC_MASK		(0x007fffffffffffff)
+#define RTC_MASK		0x007fffffffffffffUL
 /* clocks are not synchronized yet on SN1  - used node 0 (problem if no NASID 0) */
 #define RTC_COUNTER_ADDR	((clkreg_t*)REMOTE_HUB_ADDR(master_nasid, PI_RT_COUNTER))
 #define RTC_COMPARE_A_ADDR      ((clkreg_t*)REMOTE_HUB_ADDR(master_nasid, PI_RT_COMPARE_A))
@@ -52,7 +52,7 @@
 #include <asm/sn/sn2/addrs.h>
 #include <asm/sn/sn2/shubio.h>
 #include <asm/sn/sn2/shub_mmr.h>
-#define RTC_MASK		(SH_RTC_MASK)
+#define RTC_MASK		SH_RTC_MASK
 #define RTC_COUNTER_ADDR	((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #define RTC_COMPARE_A_ADDR      ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #define RTC_COMPARE_B_ADDR      ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
@@ -62,7 +62,7 @@
 #define RTC_INT_ENABLED_B_ADDR  ((clkreg_t*)LOCAL_MMR_ADDR(SH_RTC))
 #endif	/* CONFIG_IA64_SGI_SN1 */
 
-
+#define SN_RTC_PER_ITC_SHIFT	34
 #define GET_RTC_COUNTER()	(*RTC_COUNTER_ADDR)
 #define rtc_time()		GET_RTC_COUNTER()
 
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/timex.h linux-2.4.21-hack-timer/include/asm-ia64/timex.h
--- linux-2.4.21-pre5-cset-1.1020/include/asm-ia64/timex.h	Thu Nov 28 18:53:15 2002
+++ linux-2.4.21-hack-timer/include/asm-ia64/timex.h	Thu Apr 17 20:37:04 2003
@@ -24,4 +24,6 @@
 #define vxtime_lock()		do {} while (0)
 #define vxtime_unlock()		do {} while (0)
 
+extern unsigned long (*gettimeoffset)(void);
+
 #endif /* _ASM_IA64_TIMEX_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/include/linux/timex.h linux-2.4.21-hack-timer/include/linux/timex.h
--- linux-2.4.21-pre5-cset-1.1020/include/linux/timex.h	Sun Apr  6 17:37:58 2003
+++ linux-2.4.21-hack-timer/include/linux/timex.h	Thu Apr 17 20:37:09 2003
@@ -286,6 +286,12 @@
 extern long pps_errcnt;		/* calibration errors */
 extern long pps_stbcnt;		/* stability limit exceeded */
 
+/*
+ * Call-back for high precision timer sources to snapshot every time
+ * wall_jiffies is updated.
+ */
+extern void (*update_wall_time_plug)(void);
+
 #endif /* KERNEL */
 
 #endif /* LINUX_TIMEX_H */
diff -X /home/jes/exclude-linux -urN linux-2.4.21-pre5-cset-1.1020/kernel/timer.c linux-2.4.21-hack-timer/kernel/timer.c
--- linux-2.4.21-pre5-cset-1.1020/kernel/timer.c	Sun Apr  6 17:17:16 2003
+++ linux-2.4.21-hack-timer/kernel/timer.c	Tue Apr 15 19:28:54 2003
@@ -514,6 +514,14 @@
 }
 
 /*
+ * Hook for using external high precision timers for the system clock.
+ * On systems where the CPU clock isn't synchronized between CPUs,
+ * it is necessary to use an external source such as an RTC to obtain
+ * precision in gettimeofday().
+ */
+void (*update_wall_time_plug)(void) = NULL;
+
+/*
  * Using a loop looks inefficient, but "ticks" is
  * usually just one (we shouldn't be losing ticks,
  * we're doing this this way mainly for interrupt
@@ -522,6 +530,9 @@
  */
 static void update_wall_time(unsigned long ticks)
 {
+	if (update_wall_time_plug)
+		update_wall_time_plug();
+
 	do {
 		ticks--;
 		update_wall_time_one_tick();