Re: [PATCH] ia64: avoid broken SAL_CACHE_FLUSH implementations

[Bjorn Helgaas <bjorn.helgaas@hp.com>]

On Monday 30 January 2006 16:12, Keith Owens wrote:
> Bjorn Helgaas (on Mon, 30 Jan 2006 15:11:57 -0700) wrote:
> >If SAL_CACHE_FLUSH drops interrupts, complain about it and fall back to
> >using PAL_CACHE_FLUSH instead.
> >+	while (!ia64_get_irr(IA64_TIMER_VECTOR))
> >+		;
> 
> cpu_relax() instead of an empty loop?

Updated in patch below.

> FWIW, I tried the patch on SGI SN2 hardware - there was no error
> message from check_sal_cache_flush().

On the rx5670, this defect causes a hang because the BSP drops
a timer tick during the boot-time migration cost measurement.
The hang happens a bit later, when we start waiting for jiffies
to pass.  So check_sal_cache_flush() should only find a problem
on boxes that would run into bigger problems later on.

Thanks for trying this out.


ia64: avoid broken SAL_CACHE_FLUSH implementations

If SAL_CACHE_FLUSH drops interrupts, complain about it and fall back to
using PAL_CACHE_FLUSH instead.

This is to work around a defect in HP rx5670 firmware: when an interrupt
occurs during SAL_CACHE_FLUSH, SAL drops the interrupt but leaves it marked
"in-service", which leaves the interrupt (and others of equal or lower
priority) masked.

HP internal defect reports: F1859, F2775, F3031.

Signed-off-by: Bjorn Helgaas <bjorn.helgaas@hp.com>

Index: work9/arch/ia64/kernel/sal.c
=================================--- work9.orig/arch/ia64/kernel/sal.c	2006-01-30 09:50:16.000000000 -0700
+++ work9/arch/ia64/kernel/sal.c	2006-01-30 16:16:39.000000000 -0700
@@ -14,6 +14,7 @@
 #include <linux/spinlock.h>
 #include <linux/string.h>
 
+#include <asm/delay.h>
 #include <asm/page.h>
 #include <asm/sal.h>
 #include <asm/pal.h>
@@ -214,6 +215,78 @@
 static void __init sal_desc_ap_wakeup(void *p) { }
 #endif
 
+/*
+ * HP rx5670 firmware polls for interrupts during SAL_CACHE_FLUSH by reading
+ * cr.ivr, but it never writes cr.eoi.  This leaves any interrupt marked as
+ * "in-service" and masks other interrupts of equal or lower priority.
+ *
+ * HP internal defect reports: F1859, F2775, F3031.
+ */
+static int sal_cache_flush_drops_interrupts;
+
+static void __init
+check_sal_cache_flush (void)
+{
+	unsigned long flags, itv;
+	int cpu;
+	u64 vector;
+
+	cpu = get_cpu();
+	local_irq_save(flags);
+
+	/*
+	 * Schedule a timer interrupt, wait until it's reported, and see if
+	 * SAL_CACHE_FLUSH drops it.
+	 */
+	itv = ia64_get_itv();
+	BUG_ON((itv & (1 << 16)) = 0);
+
+	ia64_set_itv(IA64_TIMER_VECTOR);
+	ia64_set_itm(ia64_get_itc() + 1000);
+
+	while (!ia64_get_irr(IA64_TIMER_VECTOR))
+		cpu_relax();
+
+	ia64_sal_cache_flush(3);
+
+	if (ia64_get_irr(IA64_TIMER_VECTOR)) {
+		vector = ia64_get_ivr();
+		ia64_eoi();
+		WARN_ON(vector != IA64_TIMER_VECTOR);
+	} else {
+		sal_cache_flush_drops_interrupts = 1;
+		printk(KERN_ERR "SAL: SAL_CACHE_FLUSH drops interrupts; "
+			"PAL_CACHE_FLUSH will be used instead\n");
+		ia64_eoi();
+	}
+
+	ia64_set_itv(itv);
+	local_irq_restore(flags);
+	put_cpu();
+}
+
+s64
+ia64_sal_cache_flush (u64 cache_type)
+{
+	struct ia64_sal_retval isrv;
+
+	if (sal_cache_flush_drops_interrupts) {
+		unsigned long flags;
+		u64 progress;
+		s64 rc;
+
+		progress = 0;
+		local_irq_save(flags);
+		rc = ia64_pal_cache_flush(cache_type,
+			PAL_CACHE_FLUSH_INVALIDATE, &progress, NULL);
+		local_irq_restore(flags);
+		return rc;
+	}
+
+	SAL_CALL(isrv, SAL_CACHE_FLUSH, cache_type, 0, 0, 0, 0, 0, 0);
+	return isrv.status;
+}
+
 void __init
 ia64_sal_init (struct ia64_sal_systab *systab)
 {
@@ -262,6 +335,8 @@
 		}
 		p += SAL_DESC_SIZE(*p);
 	}
+
+	check_sal_cache_flush();
 }
 
 int
Index: work9/include/asm-ia64/sal.h
=================================--- work9.orig/include/asm-ia64/sal.h	2006-01-30 09:50:26.000000000 -0700
+++ work9/include/asm-ia64/sal.h	2006-01-30 14:36:00.000000000 -0700
@@ -658,15 +658,7 @@
 	return isrv.status;
 }
 
-/* Flush all the processor and platform level instruction and/or data caches */
-static inline s64
-ia64_sal_cache_flush (u64 cache_type)
-{
-	struct ia64_sal_retval isrv;
-	SAL_CALL(isrv, SAL_CACHE_FLUSH, cache_type, 0, 0, 0, 0, 0, 0);
-	return isrv.status;
-}
-
+extern s64 ia64_sal_cache_flush (u64 cache_type);
 
 /* Initialize all the processor and platform level instruction and data caches */
 static inline s64
Index: work9/include/asm-ia64/processor.h
=================================--- work9.orig/include/asm-ia64/processor.h	2006-01-19 16:59:35.000000000 -0700
+++ work9/include/asm-ia64/processor.h	2006-01-30 10:03:15.000000000 -0700
@@ -559,6 +559,23 @@
 
 #define cpu_relax()	ia64_hint(ia64_hint_pause)
 
+static inline int
+ia64_get_irr(unsigned int vector)
+{
+	unsigned int reg = vector / 64;
+	unsigned int bit = vector % 64;
+	u64 irr;
+
+	switch (reg) {
+	case 0: irr = ia64_getreg(_IA64_REG_CR_IRR0); break;
+	case 1: irr = ia64_getreg(_IA64_REG_CR_IRR1); break;
+	case 2: irr = ia64_getreg(_IA64_REG_CR_IRR2); break;
+	case 3: irr = ia64_getreg(_IA64_REG_CR_IRR3); break;
+	}
+
+	return test_bit(bit, &irr);
+}
+
 static inline void
 ia64_set_lrr0 (unsigned long val)
 {