[PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[john stultz]

David,
	This patch provides access to the cyclone time source found on IBM EXA
based systems (x450 and x455). This is needed on multi-node systems
where the CPU ITCs are not synchronized, causing possible time
inconsistencies.

This release fixes one last minor think-o and ran overnight without any
time inconsistencies (when used in conjunction w/ the
time-interpolator-fix_A0 patch). 

Please consider for inclusion into your tree.

thanks
-john


diff -Nru a/arch/ia64/Kconfig b/arch/ia64/Kconfig
--- a/arch/ia64/Kconfig	Wed Feb 18 11:50:43 2004
+++ b/arch/ia64/Kconfig	Wed Feb 18 11:50:43 2004
@@ -245,6 +245,12 @@
 	  Say Y here to enable machine check support for IA-64.  If you're
 	  unsure, answer Y.
 
+config IA64_CYCLONE
+	bool "Support Cyclone(EXA) Time Source"
+	help
+		Say Y here to enable support for IBM EXA Cyclone time source.
+		If you're unsure, answer N.
+	
 config PM
 	bool "Power Management support"
 	depends on IA64_GENERIC || IA64_DIG || IA64_HP_ZX1
diff -Nru a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
--- a/arch/ia64/kernel/Makefile	Wed Feb 18 11:50:43 2004
+++ b/arch/ia64/kernel/Makefile	Wed Feb 18 11:50:43 2004
@@ -18,6 +18,7 @@
 obj-$(CONFIG_MODULES)		+= module.o
 obj-$(CONFIG_SMP)		+= smp.o smpboot.o
 obj-$(CONFIG_PERFMON)		+= perfmon_default_smpl.o
+obj-$(CONFIG_IA64_CYCLONE)	+= cyclone.o
 
 # The gate DSO image is built using a special linker script.
 targets += gate.so gate-syms.o
diff -Nru a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c
--- a/arch/ia64/kernel/acpi.c	Wed Feb 18 11:50:43 2004
+++ b/arch/ia64/kernel/acpi.c	Wed Feb 18 11:50:43 2004
@@ -49,6 +49,8 @@
 #include <asm/page.h>
 #include <asm/system.h>
 #include <asm/numa.h>
+#include <asm/sal.h>
+#include <asm/cyclone.h>
 
 
 #define PREFIX			"ACPI: "
@@ -304,6 +306,22 @@
 	return 0;
 }
 
+/* Hook from generic ACPI tables.c */
+void __init acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+{
+	if (!strncmp(oem_id, "IBM", 3) &&
+	    (!strncmp(oem_table_id, "SERMOW", 6))){
+
+		/* Unfortunatly ITC_DRIFT is not yet part of the
+		 * official SAL spec, so the ITC_DRIFT bit is not
+		 * set by the BIOS on this hardware.
+		 */
+		sal_platform_features |= IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT;
+		
+		/*Start cyclone clock*/
+		cyclone_setup(0);
+	}
+}
 
 static int __init
 acpi_parse_madt (unsigned long phys_addr, unsigned long size)
@@ -327,6 +345,10 @@
 		ipi_base_addr = (unsigned long) ioremap(acpi_madt->lapic_address, 0);
 
 	printk(KERN_INFO PREFIX "Local APIC address 0x%lx\n", ipi_base_addr);
+
+	acpi_madt_oem_check(acpi_madt->header.oem_id,
+		acpi_madt->header.oem_table_id);
+
 	return 0;
 }
 
diff -Nru a/arch/ia64/kernel/cyclone.c b/arch/ia64/kernel/cyclone.c
--- /dev/null	Wed Dec 31 16:00:00 1969
+++ b/arch/ia64/kernel/cyclone.c	Wed Feb 18 11:50:43 2004
@@ -0,0 +1,158 @@
+#include <linux/smp.h>
+#include <linux/time.h>
+#include <linux/errno.h>
+
+/* IBM Summit (EXA) Cyclone counter code*/
+#define CYCLONE_CBAR_ADDR 0xFEB00CD0
+#define CYCLONE_PMCC_OFFSET 0x51A0
+#define CYCLONE_MPMC_OFFSET 0x51D0
+#define CYCLONE_MPCS_OFFSET 0x51A8
+#define CYCLONE_TIMER_FREQ 100000000
+
+int use_cyclone;
+int __init cyclone_setup(char *str)
+{
+	use_cyclone = 1;
+	return 1;
+}
+
+static u32* volatile cyclone_timer;	/* Cyclone MPMC0 register */
+static u32 last_update_cyclone;
+
+static unsigned long offset_base;
+
+static unsigned long get_offset_cyclone(void)
+{
+	u32 now;
+	unsigned long offset;
+
+	/* Read the cyclone timer */
+	now = readl(cyclone_timer);
+	/* .. relative to previous update*/
+	offset = now - last_update_cyclone;
+
+	/* convert cyclone ticks to nanoseconds */
+	offset = (offset*NSEC_PER_SEC)/CYCLONE_TIMER_FREQ;
+
+	/* our adjusted time in nanoseconds */
+	return offset_base + offset;
+}
+
+static void update_cyclone(long delta_nsec)
+{
+	u32 now;
+	unsigned long offset;
+
+	/* Read the cyclone timer */
+	now = readl(cyclone_timer);
+	/* .. relative to previous update*/
+	offset = now - last_update_cyclone;
+
+	/* convert cyclone ticks to nanoseconds */
+	offset = (offset*NSEC_PER_SEC)/CYCLONE_TIMER_FREQ;
+
+	offset += offset_base;
+
+	/* Be careful about signed/unsigned comparisons here: */
+	if (delta_nsec < 0 || (unsigned long) delta_nsec < offset)
+		offset_base = offset - delta_nsec;
+	else
+		offset_base = 0;
+
+	last_update_cyclone = now;
+}
+
+static void reset_cyclone(void)
+{
+	offset_base = 0;
+	last_update_cyclone = readl(cyclone_timer);
+}
+
+struct time_interpolator cyclone_interpolator = {
+	.get_offset =	get_offset_cyclone,
+	.update =	update_cyclone,
+	.reset =	reset_cyclone,
+	.frequency =	CYCLONE_TIMER_FREQ,
+	.drift =	-100,
+};
+
+int __init init_cyclone_clock(void)
+{
+	u64* reg;
+	u64 base;	/* saved cyclone base address */
+	u64 offset;	/* offset from pageaddr to cyclone_timer register */
+	int i;
+
+	if (!use_cyclone)
+		return -ENODEV;
+
+	printk(KERN_INFO "Summit chipset: Starting Cyclone Counter.\n");
+
+	/* find base address */
+	offset = (CYCLONE_CBAR_ADDR);
+	reg = (u64*)ioremap_nocache(offset, sizeof(u64));
+	if(!reg){
+		printk(KERN_ERR "Summit chipset: Could not find valid CBAR register.\n");
+		use_cyclone = 0;
+		return -ENODEV;
+	}
+	base = readq(reg);
+	if(!base){
+		printk(KERN_ERR "Summit chipset: Could not find valid CBAR value.\n");
+		use_cyclone = 0;
+		return -ENODEV;
+	}
+	iounmap(reg);
+		
+	/* setup PMCC */
+	offset = (base + CYCLONE_PMCC_OFFSET);
+	reg = (u64*)ioremap_nocache(offset, sizeof(u64));
+	if(!reg){
+		printk(KERN_ERR "Summit chipset: Could not find valid PMCC register.\n");
+		use_cyclone = 0;
+		return -ENODEV;
+	}
+	writel(0x00000001,reg);
+	iounmap(reg);
+	
+	/* setup MPCS */
+	offset = (base + CYCLONE_MPCS_OFFSET);
+	reg = (u64*)ioremap_nocache(offset, sizeof(u64));
+	if(!reg){
+		printk(KERN_ERR "Summit chipset: Could not find valid MPCS register.\n");
+		use_cyclone = 0;
+		return -ENODEV;
+	}
+	writel(0x00000001,reg);
+	iounmap(reg);
+	
+	/* map in cyclone_timer */
+	offset = (base + CYCLONE_MPMC_OFFSET);
+	cyclone_timer = (u32*)ioremap_nocache(offset, sizeof(u32));
+	if(!cyclone_timer){
+		printk(KERN_ERR "Summit chipset: Could not find valid MPMC register.\n");
+		use_cyclone = 0;
+		return -ENODEV;
+	}
+
+	/*quick test to make sure its ticking*/
+	for(i=0; i<3; i++){
+		u32 old = readl(cyclone_timer);
+		int stall = 100;
+		while(stall--) barrier();
+		if(readl(cyclone_timer) = old){
+			printk(KERN_ERR "Summit chipset: Counter not counting! DISABLED\n");
+			iounmap(cyclone_timer);
+			cyclone_timer = 0;
+			use_cyclone = 0;
+			return -ENODEV;
+		}
+	}
+	/* initialize last tick */
+	last_update_cyclone = readl(cyclone_timer);
+	register_time_interpolator(&cyclone_interpolator);
+
+	return 0;
+}
+
+__initcall(init_cyclone_clock);
diff -Nru a/include/asm-ia64/cyclone.h b/include/asm-ia64/cyclone.h
--- /dev/null	Wed Dec 31 16:00:00 1969
+++ b/include/asm-ia64/cyclone.h	Wed Feb 18 11:50:43 2004
@@ -0,0 +1,15 @@
+#ifndef ASM_IA64_CYCLONE_H
+#define ASM_IA64_CYCLONE_H
+
+#ifdef	CONFIG_IA64_CYCLONE
+extern int use_cyclone;
+extern int __init cyclone_setup(char*);
+#else	/* CONFIG_IA64_CYCLONE */
+#define use_cyclone 0
+static inline void cyclone_setup(char* s)
+{
+	printk(KERN_ERR "Cyclone Counter: System not configured"
+					" w/ CONFIG_IA64_CYCLONE.\n");
+}
+#endif	/* CONFIG_IA64_CYCLONE */
+#endif	/* !ASM_IA64_CYCLONE_H */

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[David Mosberger]

>>>>> On Wed, 18 Feb 2004 12:33:14 -0800, john stultz <johnstul@us.ibm.com> said:

  john> This patch provides access to the cyclone time source found on IBM EXA
  john> based systems (x450 and x455). This is needed on multi-node systems
  john> where the CPU ITCs are not synchronized, causing possible time
  john> inconsistencies.

  john> This release fixes one last minor think-o and ran overnight without any
  john> time inconsistencies (when used in conjunction w/ the
  john> time-interpolator-fix_A0 patch).

  john> Please consider for inclusion into your tree.

The current patch looks mostly fine to me (apart from the trailing
whitespace again).  I'm wondering though whether cyclone.c should go
into arch/ia64/ibm/SOMEWHERE, like we do for the SGI and HP
platform-specific code.  Or, if Cyclone sharable with your x86-based
platforms, perhaps it should even go somewhere in drivers?

	--david

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[David Mosberger]

Out of curiosity: what's the latency behavior for Cyclone?  Do all
nodes access a single (shared) counter or is the counter replicated on
each node and driven by a common clock?  My understanding that for
SGI's platform it's the latter.

	--david

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[john stultz]

On Wed, 2004-02-18 at 13:36, David Mosberger wrote:
> >>>>> On Wed, 18 Feb 2004 12:33:14 -0800, john stultz <johnstul@us.ibm.com> said:
> 
>   john> This patch provides access to the cyclone time source found on IBM EXA
>   john> based systems (x450 and x455). This is needed on multi-node systems
>   john> where the CPU ITCs are not synchronized, causing possible time
>   john> inconsistencies.
> 
>   john> This release fixes one last minor think-o and ran overnight without any
>   john> time inconsistencies (when used in conjunction w/ the
>   john> time-interpolator-fix_A0 patch).
> 
>   john> Please consider for inclusion into your tree.
> 
> The current patch looks mostly fine to me (apart from the trailing
> whitespace again).  I'm wondering though whether cyclone.c should go
> into arch/ia64/ibm/SOMEWHERE, like we do for the SGI and HP
> platform-specific code.  Or, if Cyclone sharable with your x86-based
> platforms, perhaps it should even go somewhere in drivers?

Gah! Where is this trailing whitespace? I went through this patch
specifically eliminating any I could find. Perhaps my mail client is
trying to irritate me.

The code, while similar, isn't quite shareable w/ ia32. However, I'd be
up for moving it to a separate IBM directory if you'd prefer, although
for just a single file it seems like overkill.

thanks
-john

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[john stultz]

On Wed, 2004-02-18 at 13:41, David Mosberger wrote:
> Out of curiosity: what's the latency behavior for Cyclone?  Do all
> nodes access a single (shared) counter or is the counter replicated on
> each node and driven by a common clock?  My understanding that for
> SGI's platform it's the latter.

Unfortunately there is no replicated clock on this hardware, so all
nodes access the first counter on the first node. Not great, but correct
and slow is better then fast and wrong. 

thanks
-john

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[David Mosberger]

>>>>> On Wed, 18 Feb 2004 13:52:03 -0800, john stultz <johnstul@us.ibm.com> said:

  John> Unfortunately there is no replicated clock on this hardware,
  John> so all nodes access the first counter on the first node. Not
  John> great, but correct and slow is better then fast and wrong.

OK, thanks for the info.  I'll try to keep this in mind should there
be timestamping questions again (as there was recently in the
scheduler).

>>>>> On Wed, 18 Feb 2004 13:49:44 -0800, john stultz <johnstul@us.ibm.com> said:

  John> Gah! Where is this trailing whitespace?

I already deleted the whitespace (my editor is showing it in bright
red, so it's kind a hard to ignore ;-).

  John> I went through this patch specifically eliminating any I could
  John> find. Perhaps my mail client is trying to irritate me.

The whitespace wasn't systematic, so i doubt it was the mailer.
Any chance the mail contained an old patch?

  John> The code, while similar, isn't quite shareable w/
  John> ia32. However, I'd be up for moving it to a separate IBM
  John> directory if you'd prefer, although for just a single file it
  John> seems like overkill.

I don't feel very strongly about it.  If you expect there to be other
IBM-platform specific files in the future, it would make sense to
create an ibm subdirectory.  It's really your call.

	--david

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[john stultz]

On Wed, 2004-02-18 at 14:03, David Mosberger wrote:
> >>>>> On Wed, 18 Feb 2004 13:49:44 -0800, john stultz <johnstul@us.ibm.com> said:
> 
>   John> Gah! Where is this trailing whitespace?
> 
> I already deleted the whitespace (my editor is showing it in bright
> red, so it's kind a hard to ignore ;-).
> 
>   John> I went through this patch specifically eliminating any I could
>   John> find. Perhaps my mail client is trying to irritate me.
> 
> The whitespace wasn't systematic, so i doubt it was the mailer.
> Any chance the mail contained an old patch?

No, I just checked what I sent you and its the right patch. Using vi
"set list", I still don't see any within the context of the patch,
although I notice my message had trailing whitespace.


>   John> The code, while similar, isn't quite shareable w/
>   John> ia32. However, I'd be up for moving it to a separate IBM
>   John> directory if you'd prefer, although for just a single file it
>   John> seems like overkill.
> 
> I don't feel very strongly about it.  If you expect there to be other
> IBM-platform specific files in the future, it would make sense to
> create an ibm subdirectory.  It's really your call.

I don't believe there are any current plans that would call for new IBM
platform specific files. So I prefer the patch as it was sent (modulo,
of course, the cursed trailing whitespace).

thanks
-john

Re: [PATCH] linux-2.6.3_ia64-cyclone_A2.patch

[David Mosberger]

>>>>> On Wed, 18 Feb 2004 14:24:48 -0800, john stultz <johnstul@us.ibm.com> said:

  John> I don't believe there are any current plans that would call for new IBM
  John> platform specific files. So I prefer the patch as it was sent (modulo,
  John> of course, the cursed trailing whitespace).

OK, in that case can you just resend me the patch (in private mail) so
I'm sure I'm starting with the right patch.  Perhaps I was looking at
an old mail by accident.

	--david