[Patch 00/14] GRU Driver Patches

[Patch 00/14] GRU Driver Patches

[steiner]

This patchset is a collection of patches to the GRU driver.
All changes are internal to the driver and have no effect
on the core kernel or interfaces to/from the core kernel.

The patches are based on linux-next (1/22)

[Patch 01/14] Add definitions of x86_64 GRU MMRs.

[steiner]

[-- Attachment #1: uv_mmrs_gru_x86_64 --]
[-- Type: text/plain, Size: 7845 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add definitions for x86_64 GRU MMRs.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 arch/x86/include/asm/uv/uv_mmrs.h |  153 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 153 insertions(+)

Index: linux/arch/x86/include/asm/uv/uv_mmrs.h
===================================================================
--- linux.orig/arch/x86/include/asm/uv/uv_mmrs.h	2009-01-11 07:28:11.000000000 -0600
+++ linux/arch/x86/include/asm/uv/uv_mmrs.h	2009-01-11 07:28:31.000000000 -0600
@@ -1,3 +1,4 @@
+
 /*
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
@@ -243,6 +244,158 @@ union uvh_event_occurred0_u {
 #define UVH_EVENT_OCCURRED0_ALIAS_32 0x005f0
 
 /* ========================================================================= */
+/*                         UVH_GR0_TLB_INT0_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR0_TLB_INT0_CONFIG 0x61b00UL
+
+#define UVH_GR0_TLB_INT0_CONFIG_VECTOR_SHFT 0
+#define UVH_GR0_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR0_TLB_INT0_CONFIG_DM_SHFT 8
+#define UVH_GR0_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR0_TLB_INT0_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR0_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR0_TLB_INT0_CONFIG_STATUS_SHFT 12
+#define UVH_GR0_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR0_TLB_INT0_CONFIG_P_SHFT 13
+#define UVH_GR0_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR0_TLB_INT0_CONFIG_T_SHFT 15
+#define UVH_GR0_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR0_TLB_INT0_CONFIG_M_SHFT 16
+#define UVH_GR0_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR0_TLB_INT0_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR0_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr0_tlb_int0_config_u {
+    unsigned long	v;
+    struct uvh_gr0_tlb_int0_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
+/*                         UVH_GR0_TLB_INT1_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR0_TLB_INT1_CONFIG 0x61b40UL
+
+#define UVH_GR0_TLB_INT1_CONFIG_VECTOR_SHFT 0
+#define UVH_GR0_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR0_TLB_INT1_CONFIG_DM_SHFT 8
+#define UVH_GR0_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR0_TLB_INT1_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR0_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR0_TLB_INT1_CONFIG_STATUS_SHFT 12
+#define UVH_GR0_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR0_TLB_INT1_CONFIG_P_SHFT 13
+#define UVH_GR0_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR0_TLB_INT1_CONFIG_T_SHFT 15
+#define UVH_GR0_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR0_TLB_INT1_CONFIG_M_SHFT 16
+#define UVH_GR0_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR0_TLB_INT1_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR0_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr0_tlb_int1_config_u {
+    unsigned long	v;
+    struct uvh_gr0_tlb_int1_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
+/*                         UVH_GR1_TLB_INT0_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR1_TLB_INT0_CONFIG 0x61f00UL
+
+#define UVH_GR1_TLB_INT0_CONFIG_VECTOR_SHFT 0
+#define UVH_GR1_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR1_TLB_INT0_CONFIG_DM_SHFT 8
+#define UVH_GR1_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR1_TLB_INT0_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR1_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR1_TLB_INT0_CONFIG_STATUS_SHFT 12
+#define UVH_GR1_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR1_TLB_INT0_CONFIG_P_SHFT 13
+#define UVH_GR1_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR1_TLB_INT0_CONFIG_T_SHFT 15
+#define UVH_GR1_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR1_TLB_INT0_CONFIG_M_SHFT 16
+#define UVH_GR1_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR1_TLB_INT0_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR1_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr1_tlb_int0_config_u {
+    unsigned long	v;
+    struct uvh_gr1_tlb_int0_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
+/*                         UVH_GR1_TLB_INT1_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR1_TLB_INT1_CONFIG 0x61f40UL
+
+#define UVH_GR1_TLB_INT1_CONFIG_VECTOR_SHFT 0
+#define UVH_GR1_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR1_TLB_INT1_CONFIG_DM_SHFT 8
+#define UVH_GR1_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR1_TLB_INT1_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR1_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR1_TLB_INT1_CONFIG_STATUS_SHFT 12
+#define UVH_GR1_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR1_TLB_INT1_CONFIG_P_SHFT 13
+#define UVH_GR1_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR1_TLB_INT1_CONFIG_T_SHFT 15
+#define UVH_GR1_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR1_TLB_INT1_CONFIG_M_SHFT 16
+#define UVH_GR1_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR1_TLB_INT1_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR1_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr1_tlb_int1_config_u {
+    unsigned long	v;
+    struct uvh_gr1_tlb_int1_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
 /*                               UVH_INT_CMPB                                */
 /* ========================================================================= */
 #define UVH_INT_CMPB 0x22080UL

Re: [Patch 01/14] Add definitions of x86_64 GRU MMRs.

[Andrew Morton]

On Thu, 22 Jan 2009 11:49:18 -0600 steiner@sgi.com wrote:

> Add definitions for x86_64 GRU MMRs.

Breaks x86_32:

arch/x86/include/asm/uv/uv_mmrs.h:409: error: width of 'real_time_cmpb' exceeds its type
arch/x86/include/asm/uv/uv_mmrs.h:425: error: width of 'real_time_cmpc' exceeds its type
arch/x86/include/asm/uv/uv_mmrs.h:441: error: width of 'real_time_cmpd' exceeds its type
arch/x86/include/asm/uv/uv_mmrs.h:491: error: width of 'address' exceeds its type
<etc>

due to stuff flike

	unsigned long	real_time_cmpb : 56;  /* RW */


this:

--- a/arch/x86/kernel/io_apic.c~a
+++ a/arch/x86/kernel/io_apic.c
@@ -59,8 +59,11 @@
 #include <asm/setup.h>
 #include <asm/irq_remapping.h>
 #include <asm/hpet.h>
+
+#ifdef CONFIG_X86_64
 #include <asm/uv/uv_hub.h>
 #include <asm/uv/uv_irq.h>
+#endif
 
 #include <mach_ipi.h>
 #include <mach_apic.h>
_

seems sufficient, but perhaps it'd be better if it was done within the
header?

Re: [Patch 01/14] Add definitions of x86_64 GRU MMRs.

[Jack Steiner]

On Wed, Jan 28, 2009 at 01:35:27AM -0800, Andrew Morton wrote:
> On Thu, 22 Jan 2009 11:49:18 -0600 steiner@sgi.com wrote:
> 
> > Add definitions for x86_64 GRU MMRs.
> 
> Breaks x86_32:
> 
> arch/x86/include/asm/uv/uv_mmrs.h:409: error: width of 'real_time_cmpb' exceeds its type
> arch/x86/include/asm/uv/uv_mmrs.h:425: error: width of 'real_time_cmpc' exceeds its type
> arch/x86/include/asm/uv/uv_mmrs.h:441: error: width of 'real_time_cmpd' exceeds its type
> arch/x86/include/asm/uv/uv_mmrs.h:491: error: width of 'address' exceeds its type
> <etc>
> 
> due to stuff flike
> 
> 	unsigned long	real_time_cmpb : 56;  /* RW */
> 
> 
> this:
> 
> --- a/arch/x86/kernel/io_apic.c~a
> +++ a/arch/x86/kernel/io_apic.c
> @@ -59,8 +59,11 @@
>  #include <asm/setup.h>
>  #include <asm/irq_remapping.h>
>  #include <asm/hpet.h>
> +
> +#ifdef CONFIG_X86_64
>  #include <asm/uv/uv_hub.h>
>  #include <asm/uv/uv_irq.h>
> +#endif
>  
>  #include <mach_ipi.h>
>  #include <mach_apic.h>
> _
> 
> seems sufficient, but perhaps it'd be better if it was done within the
> header?

There is no reason that any of the UV definitions should be needed for 32-bit
platforms. I'll find the best way to avoid them....


--- jack

[PATCH] - Exclude UV definitions on 32-bit x86

[Jack Steiner]

Eliminate compile errors on 32-bit X86 caused by UV.

Signed-off-by: Jack Steiner <steiner@sgi.com>
---

Note to Andrew:  this patch should be applied BEFORE including the recent
GRU patches.



 arch/x86/include/asm/uv/uv_hub.h |    2 ++
 1 file changed, 2 insertions(+)

Index: linux/arch/x86/include/asm/uv/uv_hub.h
===================================================================
--- linux.orig/arch/x86/include/asm/uv/uv_hub.h	2009-01-28 08:27:44.000000000 -0600
+++ linux/arch/x86/include/asm/uv/uv_hub.h	2009-01-28 08:32:43.000000000 -0600
@@ -11,6 +11,7 @@
 #ifndef _ASM_X86_UV_UV_HUB_H
 #define _ASM_X86_UV_UV_HUB_H
 
+#ifdef CONFIG_X86_64
 #include <linux/numa.h>
 #include <linux/percpu.h>
 #include <linux/timer.h>
@@ -413,4 +414,5 @@ static inline void uv_hub_send_ipi(int p
 }
 
 
+#endif /* CONFIG_X86_64 */
 #endif /* _ASM_X86_UV_UV_HUB_H */

Re: [Patch 01/14] Add definitions of x86_64 GRU MMRs.

[Ingo Molnar]

* Andrew Morton <akpm@linux-foundation.org> wrote:

> On Thu, 22 Jan 2009 11:49:18 -0600 steiner@sgi.com wrote:
> 
> > Add definitions for x86_64 GRU MMRs.
> 
> Breaks x86_32:
> 
> arch/x86/include/asm/uv/uv_mmrs.h:409: error: width of 'real_time_cmpb' exceeds its type
> arch/x86/include/asm/uv/uv_mmrs.h:425: error: width of 'real_time_cmpc' exceeds its type
> arch/x86/include/asm/uv/uv_mmrs.h:441: error: width of 'real_time_cmpd' exceeds its type
> arch/x86/include/asm/uv/uv_mmrs.h:491: error: width of 'address' exceeds its type
> <etc>
> 
> due to stuff flike
> 
> 	unsigned long	real_time_cmpb : 56;  /* RW */
> 
> 
> this:
> 
> --- a/arch/x86/kernel/io_apic.c~a
> +++ a/arch/x86/kernel/io_apic.c
> @@ -59,8 +59,11 @@
>  #include <asm/setup.h>
>  #include <asm/irq_remapping.h>
>  #include <asm/hpet.h>
> +
> +#ifdef CONFIG_X86_64
>  #include <asm/uv/uv_hub.h>
>  #include <asm/uv/uv_irq.h>
> +#endif
>  
>  #include <mach_ipi.h>
>  #include <mach_apic.h>
> _
> 
> seems sufficient, but perhaps it'd be better if it was done within the
> header?

Modifications to this file should be submitted to the x86 tree in any 
case.

	Ingo

[Patch 02/14] Add definitions of ia64 GRU MMRs.

[steiner]

[-- Attachment #1: uv_mmrs_gru_ia64 --]
[-- Type: text/plain, Size: 8084 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add definitions for IA64 GRU MMRs.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 arch/ia64/include/asm/uv/uv_mmrs.h |  158 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 155 insertions(+), 3 deletions(-)

Index: linux/arch/ia64/include/asm/uv/uv_mmrs.h
===================================================================
--- linux.orig/arch/ia64/include/asm/uv/uv_mmrs.h	2009-01-11 07:28:11.000000000 -0600
+++ linux/arch/ia64/include/asm/uv/uv_mmrs.h	2009-01-11 07:28:34.000000000 -0600
@@ -8,8 +8,8 @@
  * Copyright (C) 2007-2008 Silicon Graphics, Inc. All rights reserved.
  */
 
-#ifndef __ASM_IA64_UV_MMRS__
-#define __ASM_IA64_UV_MMRS__
+#ifndef _ASM_IA64_UV_UV_MMRS_H
+#define _ASM_IA64_UV_UV_MMRS_H
 
 #define UV_MMR_ENABLE		(1UL << 63)
 
@@ -243,6 +243,158 @@ union uvh_event_occurred0_u {
 #define UVH_EVENT_OCCURRED0_ALIAS_32 0x005f0
 
 /* ========================================================================= */
+/*                         UVH_GR0_TLB_INT0_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR0_TLB_INT0_CONFIG 0x61b00UL
+
+#define UVH_GR0_TLB_INT0_CONFIG_VECTOR_SHFT 0
+#define UVH_GR0_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR0_TLB_INT0_CONFIG_DM_SHFT 8
+#define UVH_GR0_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR0_TLB_INT0_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR0_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR0_TLB_INT0_CONFIG_STATUS_SHFT 12
+#define UVH_GR0_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR0_TLB_INT0_CONFIG_P_SHFT 13
+#define UVH_GR0_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR0_TLB_INT0_CONFIG_T_SHFT 15
+#define UVH_GR0_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR0_TLB_INT0_CONFIG_M_SHFT 16
+#define UVH_GR0_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR0_TLB_INT0_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR0_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr0_tlb_int0_config_u {
+    unsigned long	v;
+    struct uvh_gr0_tlb_int0_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
+/*                         UVH_GR0_TLB_INT1_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR0_TLB_INT1_CONFIG 0x61b40UL
+
+#define UVH_GR0_TLB_INT1_CONFIG_VECTOR_SHFT 0
+#define UVH_GR0_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR0_TLB_INT1_CONFIG_DM_SHFT 8
+#define UVH_GR0_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR0_TLB_INT1_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR0_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR0_TLB_INT1_CONFIG_STATUS_SHFT 12
+#define UVH_GR0_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR0_TLB_INT1_CONFIG_P_SHFT 13
+#define UVH_GR0_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR0_TLB_INT1_CONFIG_T_SHFT 15
+#define UVH_GR0_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR0_TLB_INT1_CONFIG_M_SHFT 16
+#define UVH_GR0_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR0_TLB_INT1_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR0_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr0_tlb_int1_config_u {
+    unsigned long	v;
+    struct uvh_gr0_tlb_int1_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
+/*                         UVH_GR1_TLB_INT0_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR1_TLB_INT0_CONFIG 0x61f00UL
+
+#define UVH_GR1_TLB_INT0_CONFIG_VECTOR_SHFT 0
+#define UVH_GR1_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR1_TLB_INT0_CONFIG_DM_SHFT 8
+#define UVH_GR1_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR1_TLB_INT0_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR1_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR1_TLB_INT0_CONFIG_STATUS_SHFT 12
+#define UVH_GR1_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR1_TLB_INT0_CONFIG_P_SHFT 13
+#define UVH_GR1_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR1_TLB_INT0_CONFIG_T_SHFT 15
+#define UVH_GR1_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR1_TLB_INT0_CONFIG_M_SHFT 16
+#define UVH_GR1_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR1_TLB_INT0_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR1_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr1_tlb_int0_config_u {
+    unsigned long	v;
+    struct uvh_gr1_tlb_int0_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
+/*                         UVH_GR1_TLB_INT1_CONFIG                           */
+/* ========================================================================= */
+#define UVH_GR1_TLB_INT1_CONFIG 0x61f40UL
+
+#define UVH_GR1_TLB_INT1_CONFIG_VECTOR_SHFT 0
+#define UVH_GR1_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_GR1_TLB_INT1_CONFIG_DM_SHFT 8
+#define UVH_GR1_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_GR1_TLB_INT1_CONFIG_DESTMODE_SHFT 11
+#define UVH_GR1_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_GR1_TLB_INT1_CONFIG_STATUS_SHFT 12
+#define UVH_GR1_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_GR1_TLB_INT1_CONFIG_P_SHFT 13
+#define UVH_GR1_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_GR1_TLB_INT1_CONFIG_T_SHFT 15
+#define UVH_GR1_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_GR1_TLB_INT1_CONFIG_M_SHFT 16
+#define UVH_GR1_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_GR1_TLB_INT1_CONFIG_APIC_ID_SHFT 32
+#define UVH_GR1_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+union uvh_gr1_tlb_int1_config_u {
+    unsigned long	v;
+    struct uvh_gr1_tlb_int1_config_s {
+	unsigned long	vector_  :  8;  /* RW */
+	unsigned long	dm       :  3;  /* RW */
+	unsigned long	destmode :  1;  /* RW */
+	unsigned long	status   :  1;  /* RO */
+	unsigned long	p        :  1;  /* RO */
+	unsigned long	rsvd_14  :  1;  /*    */
+	unsigned long	t        :  1;  /* RO */
+	unsigned long	m        :  1;  /* RW */
+	unsigned long	rsvd_17_31: 15;  /*    */
+	unsigned long	apic_id  : 32;  /* RW */
+    } s;
+};
+
+/* ========================================================================= */
 /*                               UVH_INT_CMPB                                */
 /* ========================================================================= */
 #define UVH_INT_CMPB 0x22080UL
@@ -670,4 +822,4 @@ union uvh_si_alias2_overlay_config_u {
 };
 
 
-#endif /* __ASM_IA64_UV_MMRS__ */
+#endif /* _ASM_IA64_UV_UV_MMRS_H */

[Patch 03/14] Add macros for using the UV hub to send interrupts

[steiner]

[-- Attachment #1: uv_send_ipi --]
[-- Type: text/plain, Size: 2691 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add macros for using the UV hub to send interrupts. Change the IPI
code to use these macros. These macros will also be used in additional
patches that will follow.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 arch/ia64/include/asm/uv/uv_hub.h |    6 ++++++
 arch/x86/include/asm/uv/uv_hub.h  |   12 ++++++++++++
 arch/x86/kernel/genx2apic_uv_x.c  |    9 ++-------
 3 files changed, 20 insertions(+), 7 deletions(-)

Index: linux/arch/ia64/include/asm/uv/uv_hub.h
===================================================================
--- linux.orig/arch/ia64/include/asm/uv/uv_hub.h	2009-01-11 07:28:11.000000000 -0600
+++ linux/arch/ia64/include/asm/uv/uv_hub.h	2009-01-11 07:28:36.000000000 -0600
@@ -305,5 +305,11 @@ static inline int uv_num_possible_blades
 	return 1;
 }
 
+static inline void uv_hub_send_ipi(int pnode, int apicid, int vector)
+{
+	/* not currently needed on ia64 */
+}
+
+
 #endif /* __ASM_IA64_UV_HUB__ */
 
Index: linux/arch/x86/include/asm/uv/uv_hub.h
===================================================================
--- linux.orig/arch/x86/include/asm/uv/uv_hub.h	2009-01-11 07:28:11.000000000 -0600
+++ linux/arch/x86/include/asm/uv/uv_hub.h	2009-01-11 07:28:36.000000000 -0600
@@ -16,6 +16,7 @@
 #include <linux/timer.h>
 #include <asm/types.h>
 #include <asm/percpu.h>
+#include <asm/uv/uv_mmrs.h>
 
 
 /*
@@ -401,4 +402,15 @@ static inline void uv_set_cpu_scir_bits(
 	}
 }
 
+static inline void uv_hub_send_ipi(int pnode, int apicid, int vector)
+{
+	unsigned long val;
+
+	val = (1UL << UVH_IPI_INT_SEND_SHFT) |
+			((apicid & 0x3f) << UVH_IPI_INT_APIC_ID_SHFT) |
+			(vector << UVH_IPI_INT_VECTOR_SHFT);
+	uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
+}
+
+
 #endif /* _ASM_X86_UV_UV_HUB_H */
Index: linux/arch/x86/kernel/genx2apic_uv_x.c
===================================================================
--- linux.orig/arch/x86/kernel/genx2apic_uv_x.c	2009-01-11 07:28:11.000000000 -0600
+++ linux/arch/x86/kernel/genx2apic_uv_x.c	2009-01-11 07:28:36.000000000 -0600
@@ -113,17 +113,12 @@ int uv_wakeup_secondary(int phys_apicid,
 
 static void uv_send_IPI_one(int cpu, int vector)
 {
-	unsigned long val, apicid, lapicid;
+	unsigned long apicid;
 	int pnode;
 
 	apicid = per_cpu(x86_cpu_to_apicid, cpu);
-	lapicid = apicid & 0x3f;		/* ZZZ macro needed */
 	pnode = uv_apicid_to_pnode(apicid);
-	val =
-	    (1UL << UVH_IPI_INT_SEND_SHFT) | (lapicid <<
-					      UVH_IPI_INT_APIC_ID_SHFT) |
-	    (vector << UVH_IPI_INT_VECTOR_SHFT);
-	uv_write_global_mmr64(pnode, UVH_IPI_INT, val);
+	uv_hub_send_ipi(pnode, apicid, vector);
 }
 
 static void uv_send_IPI_mask(const struct cpumask *mask, int vector)

[Patch 04/14] Misc GRU cleanup

[steiner]

[-- Attachment #1: uv_gru_misc_fixes --]
[-- Type: text/plain, Size: 13569 bytes --]

From: Jack Steiner <steiner@sgi.com>

Misc trivial GRU drivers fixes:
	- fix long lines
	- eliminate extra whitespace
	- eliminate compiler warning
	- better validation of invalidate user parameters
	- bug fix for GRU TLB flush (not the cpu TLB flush)

These changes are all internal to the SGI GRU driver and have no effect
on the base kernel.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru_instructions.h |   22 ++++++++++++----------
 drivers/misc/sgi-gru/grufault.c         |   19 ++++++++++++-------
 drivers/misc/sgi-gru/grufile.c          |   25 +++++++++++++++----------
 drivers/misc/sgi-gru/gruhandles.h       |    2 +-
 drivers/misc/sgi-gru/grukservices.c     |    4 ++--
 drivers/misc/sgi-gru/grumain.c          |    6 +++---
 drivers/misc/sgi-gru/grutables.h        |   15 ++++++++-------
 drivers/misc/sgi-gru/grutlbpurge.c      |    5 ++---
 8 files changed, 55 insertions(+), 43 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru_instructions.h	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gru_instructions.h	2009-01-21 15:51:41.000000000 -0600
@@ -19,8 +19,11 @@
 #ifndef __GRU_INSTRUCTIONS_H__
 #define __GRU_INSTRUCTIONS_H__
 
-#define gru_flush_cache_hook(p)
-#define gru_emulator_wait_hook(p, w)
+extern int gru_check_status_proc(void *cb);
+extern int gru_wait_proc(void *cb);
+extern void gru_wait_abort_proc(void *cb);
+
+
 
 /*
  * Architecture dependent functions
@@ -29,16 +32,16 @@
 #if defined(CONFIG_IA64)
 #include <linux/compiler.h>
 #include <asm/intrinsics.h>
-#define __flush_cache(p)		ia64_fc(p)
+#define __flush_cache(p)		ia64_fc((unsigned long)p)
 /* Use volatile on IA64 to ensure ordering via st4.rel */
-#define gru_ordered_store_int(p,v)					\
+#define gru_ordered_store_int(p, v)					\
 		do {							\
 			barrier();					\
 			*((volatile int *)(p)) = v; /* force st.rel */	\
 		} while (0)
 #elif defined(CONFIG_X86_64)
 #define __flush_cache(p)		clflush(p)
-#define gru_ordered_store_int(p,v)					\
+#define gru_ordered_store_int(p, v)					\
 		do {							\
 			barrier();					\
 			*(int *)p = v;					\
@@ -558,20 +561,19 @@ extern int gru_get_cb_exception_detail(v
 
 #define GRU_EXC_STR_SIZE		256
 
-extern int gru_check_status_proc(void *cb);
-extern int gru_wait_proc(void *cb);
-extern void gru_wait_abort_proc(void *cb);
 
 /*
  * Control block definition for checking status
  */
 struct gru_control_block_status {
 	unsigned int	icmd		:1;
-	unsigned int	unused1		:31;
+	unsigned int	ima		:3;
+	unsigned int	reserved0	:4;
+	unsigned int	unused1		:24;
 	unsigned int	unused2		:24;
 	unsigned int	istatus		:2;
 	unsigned int	isubstatus	:4;
-	unsigned int	inused3		:2;
+	unsigned int	unused3		:2;
 };
 
 /* Get CB status */
Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufault.c	2009-01-21 15:52:29.000000000 -0600
@@ -368,6 +368,7 @@ failupm:
 
 failfmm:
 	/* FMM state on UPM call */
+	gru_flush_cache(tfh);
 	STAT(tlb_dropin_fail_fmm);
 	gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state);
 	return 0;
@@ -497,10 +498,8 @@ int gru_handle_user_call_os(unsigned lon
 	if (!gts)
 		return -EINVAL;
 
-	if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) {
-		ret = -EINVAL;
+	if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE)
 		goto exit;
-	}
 
 	/*
 	 * If force_unload is set, the UPM TLB fault is phony. The task
@@ -508,6 +507,10 @@ int gru_handle_user_call_os(unsigned lon
 	 * unload the context. The task will page fault and assign a new
 	 * context.
 	 */
+	if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 &&
+				gts->ts_blade != uv_numa_blade_id())
+		gts->ts_force_unload = 1;
+
 	ret = -EAGAIN;
 	cbrnum = thread_cbr_number(gts, ucbnum);
 	if (gts->ts_force_unload) {
@@ -541,11 +544,13 @@ int gru_get_exception_detail(unsigned lo
 	if (!gts)
 		return -EINVAL;
 
-	if (gts->ts_gru) {
-		ucbnum = get_cb_number((void *)excdet.cb);
+	ucbnum = get_cb_number((void *)excdet.cb);
+	if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) {
+		ret = -EINVAL;
+	} else if (gts->ts_gru) {
 		cbrnum = thread_cbr_number(gts, ucbnum);
 		cbe = get_cbe_by_index(gts->ts_gru, cbrnum);
-		prefetchw(cbe);		/* Harmless on hardware, required for emulator */
+		prefetchw(cbe);/* Harmless on hardware, required for emulator */
 		excdet.opc = cbe->opccpy;
 		excdet.exopc = cbe->exopccpy;
 		excdet.ecause = cbe->ecause;
@@ -609,7 +614,7 @@ int gru_user_flush_tlb(unsigned long arg
 	if (!gts)
 		return -EINVAL;
 
-	gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.vaddr + req.len);
+	gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.len);
 	gru_unlock_gts(gts);
 
 	return 0;
Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufile.c	2009-01-21 15:51:50.000000000 -0600
@@ -57,7 +57,8 @@
 #include <asm/uv/uv_mmrs.h>
 
 struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly;
-unsigned long gru_start_paddr, gru_end_paddr __read_mostly;
+unsigned long gru_start_paddr __read_mostly;
+unsigned long gru_end_paddr __read_mostly;
 struct gru_stats_s gru_stats;
 
 /* Guaranteed user available resources on each node */
@@ -113,7 +114,7 @@ static int gru_file_mmap(struct file *fi
 		return -EPERM;
 
 	if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) ||
-	    			vma->vm_end & (GRU_GSEG_PAGESIZE - 1))
+				vma->vm_end & (GRU_GSEG_PAGESIZE - 1))
 		return -EINVAL;
 
 	vma->vm_flags |=
@@ -307,7 +308,7 @@ static int gru_init_tables(unsigned long
 	for_each_online_node(nid) {
 		bid = uv_node_to_blade_id(nid);
 		pnode = uv_node_to_pnode(nid);
-		if (gru_base[bid])
+		if (bid < 0 || gru_base[bid])
 			continue;
 		page = alloc_pages_node(nid, GFP_KERNEL, order);
 		if (!page)
@@ -320,11 +321,11 @@ static int gru_init_tables(unsigned long
 		dsrbytes = 0;
 		cbrs = 0;
 		for (gru = gru_base[bid]->bs_grus, chip = 0;
-		     		chip < GRU_CHIPLETS_PER_BLADE;
+				chip < GRU_CHIPLETS_PER_BLADE;
 				chip++, gru++) {
 			paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip);
 			vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip);
-			gru_init_chiplet(gru, paddr, vaddr, bid, nid, chip);
+			gru_init_chiplet(gru, paddr, vaddr, nid, bid, chip);
 			n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
 			cbrs = max(cbrs, n);
 			n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
@@ -382,26 +383,26 @@ static int __init gru_init(void)
 	void *gru_start_vaddr;
 
 	if (!IS_UV())
-		return 0;
+		return -ENODEV;
 
 #if defined CONFIG_IA64
 	gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */
 #else
 	gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) &
 				0x7fffffffffffUL;
-
 #endif
 	gru_start_vaddr = __va(gru_start_paddr);
-	gru_end_paddr = gru_start_paddr + MAX_NUMNODES * GRU_SIZE;
+	gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE;
 	printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n",
 	       gru_start_paddr, gru_end_paddr);
 	irq = get_base_irq();
 	for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) {
 		ret = request_irq(irq + chip, gru_intr, 0, id, NULL);
-		/* TODO: fix irq handling on x86. For now ignore failures because
+		/* TODO: fix irq handling on x86. For now ignore failure because
 		 * interrupts are not required & not yet fully supported */
 		if (ret) {
-			printk("!!!WARNING: GRU ignoring request failure!!!\n");
+			printk(KERN_WARNING
+			       "!!!WARNING: GRU ignoring request failure!!!\n");
 			ret = 0;
 		}
 		if (ret) {
@@ -481,7 +482,11 @@ struct vm_operations_struct gru_vm_ops =
 	.fault		= gru_fault,
 };
 
+#ifndef MODULE
 fs_initcall(gru_init);
+#else
+module_init(gru_init);
+#endif
 module_exit(gru_exit);
 
 module_param(gru_options, ulong, 0644);
Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.h	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruhandles.h	2009-01-21 15:51:49.000000000 -0600
@@ -489,7 +489,7 @@ enum gru_cbr_state {
  * 	 64m			26	8
  * 	...
  */
-#define GRU_PAGESIZE(sh)	((((sh) > 20 ? (sh) + 2: (sh)) >> 1) - 6)
+#define GRU_PAGESIZE(sh)	((((sh) > 20 ? (sh) + 2 : (sh)) >> 1) - 6)
 #define GRU_SIZEAVAIL(sh)	(1UL << GRU_PAGESIZE(sh))
 
 /* minimum TLB purge count to ensure a full purge */
Index: linux/drivers/misc/sgi-gru/grukservices.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.c	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grukservices.c	2009-01-21 15:51:50.000000000 -0600
@@ -122,7 +122,7 @@ int gru_get_cb_exception_detail(void *cb
 	struct gru_control_block_extended *cbe;
 
 	cbe = get_cbe(GRUBASE(cb), get_cb_number(cb));
-	prefetchw(cbe);         /* Harmless on hardware, required for emulator */
+	prefetchw(cbe);	/* Harmless on hardware, required for emulator */
 	excdet->opc = cbe->opccpy;
 	excdet->exopc = cbe->exopccpy;
 	excdet->ecause = cbe->ecause;
@@ -437,7 +437,7 @@ static int send_message_failure(void *cb
 		break;
 	case CBSS_PUT_NACKED:
 		STAT(mesq_send_put_nacked);
-		m =mq + (gru_get_amo_value_head(cb) << 6);
+		m = mq + (gru_get_amo_value_head(cb) << 6);
 		gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
 		if (gru_wait(cb) == CBS_IDLE)
 			ret = MQE_OK;
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grumain.c	2009-01-21 15:51:50.000000000 -0600
@@ -432,8 +432,8 @@ static inline long gru_copy_handle(void 
 	return GRU_HANDLE_BYTES;
 }
 
-static void gru_prefetch_context(void *gseg, void *cb, void *cbe, unsigned long cbrmap,
-				unsigned long length)
+static void gru_prefetch_context(void *gseg, void *cb, void *cbe,
+				unsigned long cbrmap, unsigned long length)
 {
 	int i, scr;
 
@@ -773,8 +773,8 @@ int gru_fault(struct vm_area_struct *vma
 		return VM_FAULT_SIGBUS;
 
 again:
-	preempt_disable();
 	mutex_lock(&gts->ts_ctxlock);
+	preempt_disable();
 	if (gts->ts_gru) {
 		if (gts->ts_gru->gs_blade_id != uv_numa_blade_id()) {
 			STAT(migrated_nopfn_unload);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-21 15:51:50.000000000 -0600
@@ -278,13 +278,12 @@ struct gru_stats_s {
 /* Generate a GRU asid value from a GRU base asid & a virtual address. */
 #if defined CONFIG_IA64
 #define VADDR_HI_BIT		64
-#define GRUREGION(addr)		((addr) >> (VADDR_HI_BIT - 3) & 3)
 #elif defined CONFIG_X86_64
 #define VADDR_HI_BIT		48
-#define GRUREGION(addr)		(0)		/* ZZZ could do better */
 #else
 #error "Unsupported architecture"
 #endif
+#define GRUREGION(addr)		((addr) >> (VADDR_HI_BIT - 3) & 3)
 #define GRUASID(asid, addr)	((asid) + GRUREGION(addr))
 
 /*------------------------------------------------------------------------------
@@ -297,12 +296,12 @@ struct gru_state;
  * This structure is pointed to from the mmstruct via the notifier pointer.
  * There is one of these per address space.
  */
-struct gru_mm_tracker {
-	unsigned int		mt_asid_gen;	/* ASID wrap count */
-	int			mt_asid;	/* current base ASID for gru */
-	unsigned short		mt_ctxbitmap;	/* bitmap of contexts using
+struct gru_mm_tracker {				/* pack to reduce size */
+	unsigned int		mt_asid_gen:24;	/* ASID wrap count */
+	unsigned int		mt_asid:24;	/* current base ASID for gru */
+	unsigned short		mt_ctxbitmap:16;/* bitmap of contexts using
 						   asid */
-};
+} __attribute__ ((packed));
 
 struct gru_mm_struct {
 	struct mmu_notifier	ms_notifier;
@@ -359,6 +358,8 @@ struct gru_thread_state {
 						   required for contest */
 	unsigned char		ts_cbr_au_count;/* Number of CBR resources
 						   required for contest */
+	char			ts_blade;	/* If >= 0, migrate context if
+						   ref from diferent blade */
 	char			ts_force_unload;/* force context to be unloaded
 						   after migration */
 	char			ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each
Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2009-01-21 12:51:13.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2009-01-21 15:51:50.000000000 -0600
@@ -187,7 +187,7 @@ void gru_flush_tlb_range(struct gru_mm_s
 	"  FLUSH gruid %d, asid 0x%x, num %ld, cbmap 0x%x\n",
 				gid, asid, num, asids->mt_ctxbitmap);
 			tgh = get_lock_tgh_handle(gru);
-			tgh_invalidate(tgh, start, 0, asid, grupagesize, 0,
+			tgh_invalidate(tgh, start, ~0, asid, grupagesize, 0,
 				       num - 1, asids->mt_ctxbitmap);
 			get_unlock_tgh_handle(tgh);
 		} else {
@@ -212,9 +212,8 @@ void gru_flush_all_tlb(struct gru_state 
 
 	gru_dbg(grudev, "gru %p, gid %d\n", gru, gru->gs_gid);
 	tgh = get_lock_tgh_handle(gru);
-	tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0);
+	tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0xffff);
 	get_unlock_tgh_handle(tgh);
-	preempt_enable();
 }
 
 /*

[Patch 05/14] Improvements to GRU debug messages & statistics

[steiner]

[-- Attachment #1: uv_gru_stats_dbg --]
[-- Type: text/plain, Size: 10227 bytes --]

From: Jack Steiner <steiner@sgi.com>

Improvements to GRU debug messages & statistics.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c     |    5 +++
 drivers/misc/sgi-gru/grufile.c      |    2 -
 drivers/misc/sgi-gru/grukservices.c |    4 +--
 drivers/misc/sgi-gru/grumain.c      |   48 +++++++++++++++++++-----------------
 drivers/misc/sgi-gru/gruprocfs.c    |    2 +
 drivers/misc/sgi-gru/grutables.h    |    2 +
 drivers/misc/sgi-gru/grutlbpurge.c  |    2 -
 7 files changed, 38 insertions(+), 27 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:06.000000000 -0600
@@ -449,6 +449,7 @@ irqreturn_t gru_intr(int irq, void *dev_
 			up_read(&gts->ts_mm->mmap_sem);
 		} else {
 			tfh_user_polling_mode(tfh);
+			STAT(intr_mm_lock_failed);
 		}
 	}
 	return IRQ_HANDLED;
@@ -508,8 +509,10 @@ int gru_handle_user_call_os(unsigned lon
 	 * context.
 	 */
 	if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 &&
-				gts->ts_blade != uv_numa_blade_id())
+				gts->ts_blade != uv_numa_blade_id()) {
+		STAT(call_os_offnode_reference);
 		gts->ts_force_unload = 1;
+	}
 
 	ret = -EAGAIN;
 	cbrnum = thread_cbr_number(gts, ucbnum);
Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufile.c	2009-01-15 08:18:06.000000000 -0600
@@ -287,7 +287,7 @@ static void gru_init_chiplet(struct gru_
 	gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
 	gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
 	gru_tgh_flush_init(gru);
-	gru_dbg(grudev, "bid %d, nid %d, gru %x, vaddr %p (0x%lx)\n",
+	gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n",
 		bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr,
 		gru->gs_gru_base_paddr);
 	gru_kservices_init(gru);
Index: linux/drivers/misc/sgi-gru/grukservices.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.c	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grukservices.c	2009-01-15 08:18:06.000000000 -0600
@@ -663,12 +663,12 @@ int gru_kservices_init(struct gru_state 
 	err = cch_allocate(cch, 0, cbr_map, dsr_map);
 	if (err) {
 		gru_dbg(grudev,
-			"Unable to allocate kernel CCH: gru %d, err %d\n",
+			"Unable to allocate kernel CCH: gid %d, err %d\n",
 			gru->gs_gid, err);
 		BUG();
 	}
 	if (cch_start(cch)) {
-		gru_dbg(grudev, "Unable to start kernel CCH: gru %d, err %d\n",
+		gru_dbg(grudev, "Unable to start kernel CCH: gid %d, err %d\n",
 			gru->gs_gid, err);
 		BUG();
 	}
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grumain.c	2009-01-15 08:18:06.000000000 -0600
@@ -76,7 +76,7 @@ int gru_cpu_fault_map_id(void)
 /* Hit the asid limit. Start over */
 static int gru_wrap_asid(struct gru_state *gru)
 {
-	gru_dbg(grudev, "gru %p\n", gru);
+	gru_dbg(grudev, "gid %d\n", gru->gs_gid);
 	STAT(asid_wrap);
 	gru->gs_asid_gen++;
 	gru_flush_all_tlb(gru);
@@ -88,7 +88,7 @@ static int gru_reset_asid_limit(struct g
 {
 	int i, gid, inuse_asid, limit;
 
-	gru_dbg(grudev, "gru %p, asid 0x%x\n", gru, asid);
+	gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
 	STAT(asid_next);
 	limit = MAX_ASID;
 	if (asid >= limit)
@@ -99,8 +99,9 @@ again:
 		if (!gru->gs_gts[i])
 			continue;
 		inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid;
-		gru_dbg(grudev, "gru %p, inuse_asid 0x%x, cxtnum %d, gts %p\n",
-			gru, inuse_asid, i, gru->gs_gts[i]);
+		gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n",
+			gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms,
+			inuse_asid, i);
 		if (inuse_asid == asid) {
 			asid += ASID_INC;
 			if (asid >= limit) {
@@ -120,8 +121,8 @@ again:
 	}
 	gru->gs_asid_limit = limit;
 	gru->gs_asid = asid;
-	gru_dbg(grudev, "gru %p, new asid 0x%x, new_limit 0x%x\n", gru, asid,
-		limit);
+	gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid,
+					asid, limit);
 	return asid;
 }
 
@@ -137,7 +138,7 @@ static int gru_assign_asid(struct gru_st
 		asid = gru_reset_asid_limit(gru, asid);
 	spin_unlock(&gru->gs_asid_lock);
 
-	gru_dbg(grudev, "gru %p, asid 0x%x\n", gru, asid);
+	gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
 	return asid;
 }
 
@@ -215,11 +216,12 @@ static int check_gru_resources(struct gr
  * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG
  * context.
  */
-static int gru_load_mm_tracker(struct gru_state *gru, struct gru_mm_struct *gms,
-			       int ctxnum)
+static int gru_load_mm_tracker(struct gru_state *gru,
+					struct gru_thread_state *gts)
 {
+	struct gru_mm_struct *gms = gts->ts_gms;
 	struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid];
-	unsigned short ctxbitmap = (1 << ctxnum);
+	unsigned short ctxbitmap = (1 << gts->ts_ctxnum);
 	int asid;
 
 	spin_lock(&gms->ms_asid_lock);
@@ -241,24 +243,26 @@ static int gru_load_mm_tracker(struct gr
 	spin_unlock(&gms->ms_asid_lock);
 
 	gru_dbg(grudev,
-		"gru %x, gms %p, ctxnum 0x%d, asid 0x%x, asidmap 0x%lx\n",
-		gru->gs_gid, gms, ctxnum, asid, gms->ms_asidmap[0]);
+		"gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
+		gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,
+		gms->ms_asidmap[0]);
 	return asid;
 }
 
 static void gru_unload_mm_tracker(struct gru_state *gru,
-				  struct gru_mm_struct *gms, int ctxnum)
+					struct gru_thread_state *gts)
 {
+	struct gru_mm_struct *gms = gts->ts_gms;
 	struct gru_mm_tracker *asids;
 	unsigned short ctxbitmap;
 
 	asids = &gms->ms_asids[gru->gs_gid];
-	ctxbitmap = (1 << ctxnum);
+	ctxbitmap = (1 << gts->ts_ctxnum);
 	spin_lock(&gms->ms_asid_lock);
 	BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
 	asids->mt_ctxbitmap ^= ctxbitmap;
-	gru_dbg(grudev, "gru %x, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
-		gru->gs_gid, gms, ctxnum, gms->ms_asidmap[0]);
+	gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
+		gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
 	spin_unlock(&gms->ms_asid_lock);
 }
 
@@ -399,7 +403,7 @@ static void gru_free_gru_context(struct 
 	struct gru_state *gru;
 
 	gru = gts->ts_gru;
-	gru_dbg(grudev, "gts %p, gru %p\n", gts, gru);
+	gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);
 
 	spin_lock(&gru->gs_lock);
 	gru->gs_gts[gts->ts_ctxnum] = NULL;
@@ -500,12 +504,12 @@ void gru_unload_context(struct gru_threa
 	zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 
+	gru_dbg(grudev, "gts %p\n", gts);
 	lock_cch_handle(cch);
 	if (cch_interrupt_sync(cch))
 		BUG();
-	gru_dbg(grudev, "gts %p\n", gts);
 
-	gru_unload_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum);
+	gru_unload_mm_tracker(gru, gts);
 	if (savestate)
 		gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr,
 					ctxnum, gts->ts_cbr_map,
@@ -534,7 +538,7 @@ static void gru_load_context(struct gru_
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 
 	lock_cch_handle(cch);
-	asid = gru_load_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum);
+	asid = gru_load_mm_tracker(gru, gts);
 	cch->tfm_fault_bit_enable =
 	    (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
 	     || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
@@ -688,7 +692,7 @@ static void gru_steal_context(struct gru
 		STAT(steal_context_failed);
 	}
 	gru_dbg(grudev,
-		"stole gru %x, ctxnum %d from gts %p. Need cb %d, ds %d;"
+		"stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
 		" avail cb %ld, ds %ld\n",
 		gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),
 		hweight64(gru->gs_dsr_map));
@@ -737,7 +741,7 @@ again:
 
 		STAT(assign_context);
 		gru_dbg(grudev,
-			"gseg %p, gts %p, gru %x, ctx %d, cbr %d, dsr %d\n",
+			"gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
 			gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts,
 			gts->ts_gru->gs_gid, gts->ts_ctxnum,
 			gts->ts_cbr_au_count, gts->ts_dsr_au_count);
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2009-01-15 08:17:50.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2009-01-15 08:18:06.000000000 -0600
@@ -62,7 +62,9 @@ static int statistics_show(struct seq_fi
 	printstat(s, asid_wrap);
 	printstat(s, asid_reuse);
 	printstat(s, intr);
+	printstat(s, intr_mm_lock_failed);
 	printstat(s, call_os);
+	printstat(s, call_os_offnode_reference);
 	printstat(s, call_os_check_for_bug);
 	printstat(s, call_os_wait_queue);
 	printstat(s, user_flush_tlb);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:06.000000000 -0600
@@ -184,7 +184,9 @@ struct gru_stats_s {
 	atomic_long_t asid_wrap;
 	atomic_long_t asid_reuse;
 	atomic_long_t intr;
+	atomic_long_t intr_mm_lock_failed;
 	atomic_long_t call_os;
+	atomic_long_t call_os_offnode_reference;
 	atomic_long_t call_os_check_for_bug;
 	atomic_long_t call_os_wait_queue;
 	atomic_long_t user_flush_tlb;
Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2009-01-15 08:18:06.000000000 -0600
@@ -210,7 +210,7 @@ void gru_flush_all_tlb(struct gru_state 
 {
 	struct gru_tlb_global_handle *tgh;
 
-	gru_dbg(grudev, "gru %p, gid %d\n", gru, gru->gs_gid);
+	gru_dbg(grudev, "gid %d\n", gru->gs_gid);
 	tgh = get_lock_tgh_handle(gru);
 	tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0xffff);
 	get_unlock_tgh_handle(tgh);

[Patch 06/14] Change GRU CCH commands from inline functions to outofline functions

[steiner]

[-- Attachment #1: uv_outofline_mcs_instructions --]
[-- Type: text/plain, Size: 12599 bytes --]

From: Jack Steiner <steiner@sgi.com>

Change the GRU instructions that manage contexts from inline functions
to out-of-line functions.  This simplifies adding statistics & error
checking to these functions.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 drivers/misc/sgi-gru/Makefile     |    2 
 drivers/misc/sgi-gru/gruhandles.c |  197 ++++++++++++++++++++++++++++++++++++++
 drivers/misc/sgi-gru/gruhandles.h |  174 +++------------------------------
 drivers/misc/sgi-gru/grutables.h  |    3 
 4 files changed, 217 insertions(+), 159 deletions(-)

Index: linux/drivers/misc/sgi-gru/Makefile
===================================================================
--- linux.orig/drivers/misc/sgi-gru/Makefile	2009-01-15 08:17:49.000000000 -0600
+++ linux/drivers/misc/sgi-gru/Makefile	2009-01-15 08:18:08.000000000 -0600
@@ -3,5 +3,5 @@ ifdef CONFIG_SGI_GRU_DEBUG
 endif
 
 obj-$(CONFIG_SGI_GRU) := gru.o
-gru-y := grufile.o grumain.o grufault.o grutlbpurge.o gruprocfs.o grukservices.o
+gru-y := grufile.o grumain.o grufault.o grutlbpurge.o gruprocfs.o grukservices.o gruhandles.o
 
Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/misc/sgi-gru/gruhandles.c	2009-01-15 08:18:08.000000000 -0600
@@ -0,0 +1,197 @@
+/*
+ *              GRU KERNEL MCS INSTRUCTIONS
+ *
+ *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include "gru.h"
+#include "grulib.h"
+#include "grutables.h"
+
+/* 10 sec */
+#ifdef CONFIG_IA64
+#include <asm/processor.h>
+#define GRU_OPERATION_TIMEOUT	(((cycles_t) local_cpu_data->itc_freq)*10)
+#else
+#include <asm/tsc.h>
+#define GRU_OPERATION_TIMEOUT	((cycles_t) tsc_khz*10*1000)
+#endif
+
+/* Extract the status field from a kernel handle */
+#define GET_MSEG_HANDLE_STATUS(h)	(((*(unsigned long *)(h)) >> 16) & 3)
+
+static void start_instruction(void *h)
+{
+	unsigned long *w0 = h;
+
+	wmb();		/* setting CMD bit must be last */
+	*w0 = *w0 | 1;
+	gru_flush_cache(h);
+}
+
+static int wait_instruction_complete(void *h, enum mcs_op opc)
+{
+	int status;
+	cycles_t start_time = get_cycles();
+
+	while (1) {
+		cpu_relax();
+		status = GET_MSEG_HANDLE_STATUS(h);
+		if (status != CCHSTATUS_ACTIVE)
+			break;
+		if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time))
+			panic("GRU %p is malfunctioning\n", h);
+	}
+	return status;
+}
+
+#if defined CONFIG_IA64
+static void cch_allocate_set_asids(
+		  struct gru_context_configuration_handle *cch, int asidval)
+{
+	int i;
+
+	for (i = 0; i < 8; i++) {
+		cch->asid[i] = (asidval++);
+#if 0
+		/* ZZZ hugepages not supported yet */
+		if (i == RGN_HPAGE)
+			cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift);
+		else
+#endif
+			cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT);
+	}
+}
+#elif defined CONFIG_X86_64
+static void cch_allocate_set_asids(
+		  struct gru_context_configuration_handle *cch, int asidval)
+{
+	int i;
+
+	for (i = 0; i < 8; i++) {
+		cch->asid[i] = asidval++;
+		cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) |
+			GRU_SIZEAVAIL(21);
+	}
+}
+#endif
+
+int cch_allocate(struct gru_context_configuration_handle *cch,
+			       int asidval, unsigned long cbrmap,
+			       unsigned long dsrmap)
+{
+	cch_allocate_set_asids(cch, asidval);
+	cch->dsr_allocation_map = dsrmap;
+	cch->cbr_allocation_map = cbrmap;
+	cch->opc = CCHOP_ALLOCATE;
+	start_instruction(cch);
+	return wait_instruction_complete(cch, cchop_allocate);
+}
+
+int cch_start(struct gru_context_configuration_handle *cch)
+{
+	cch->opc = CCHOP_START;
+	start_instruction(cch);
+	return wait_instruction_complete(cch, cchop_start);
+}
+
+int cch_interrupt(struct gru_context_configuration_handle *cch)
+{
+	cch->opc = CCHOP_INTERRUPT;
+	start_instruction(cch);
+	return wait_instruction_complete(cch, cchop_interrupt);
+}
+
+int cch_deallocate(struct gru_context_configuration_handle *cch)
+{
+	cch->opc = CCHOP_DEALLOCATE;
+	start_instruction(cch);
+	return wait_instruction_complete(cch, cchop_deallocate);
+}
+
+int cch_interrupt_sync(struct gru_context_configuration_handle
+				     *cch)
+{
+	cch->opc = CCHOP_INTERRUPT_SYNC;
+	start_instruction(cch);
+	return wait_instruction_complete(cch, cchop_interrupt_sync);
+}
+
+int tgh_invalidate(struct gru_tlb_global_handle *tgh,
+				 unsigned long vaddr, unsigned long vaddrmask,
+				 int asid, int pagesize, int global, int n,
+				 unsigned short ctxbitmap)
+{
+	tgh->vaddr = vaddr;
+	tgh->asid = asid;
+	tgh->pagesize = pagesize;
+	tgh->n = n;
+	tgh->global = global;
+	tgh->vaddrmask = vaddrmask;
+	tgh->ctxbitmap = ctxbitmap;
+	tgh->opc = TGHOP_TLBINV;
+	start_instruction(tgh);
+	return wait_instruction_complete(tgh, tghop_invalidate);
+}
+
+void tfh_write_only(struct gru_tlb_fault_handle *tfh,
+				  unsigned long pfn, unsigned long vaddr,
+				  int asid, int dirty, int pagesize)
+{
+	tfh->fillasid = asid;
+	tfh->fillvaddr = vaddr;
+	tfh->pfn = pfn;
+	tfh->dirty = dirty;
+	tfh->pagesize = pagesize;
+	tfh->opc = TFHOP_WRITE_ONLY;
+	start_instruction(tfh);
+}
+
+void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
+				     unsigned long paddr, int gaa,
+				     unsigned long vaddr, int asid, int dirty,
+				     int pagesize)
+{
+	tfh->fillasid = asid;
+	tfh->fillvaddr = vaddr;
+	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
+	tfh->gaa = gaa;
+	tfh->dirty = dirty;
+	tfh->pagesize = pagesize;
+	tfh->opc = TFHOP_WRITE_RESTART;
+	start_instruction(tfh);
+}
+
+void tfh_restart(struct gru_tlb_fault_handle *tfh)
+{
+	tfh->opc = TFHOP_RESTART;
+	start_instruction(tfh);
+}
+
+void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
+{
+	tfh->opc = TFHOP_USER_POLLING_MODE;
+	start_instruction(tfh);
+}
+
+void tfh_exception(struct gru_tlb_fault_handle *tfh)
+{
+	tfh->opc = TFHOP_EXCEPTION;
+	start_instruction(tfh);
+}
+
Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.h	2009-01-15 08:18:04.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruhandles.h	2009-01-15 08:18:08.000000000 -0600
@@ -495,164 +495,22 @@ enum gru_cbr_state {
 /* minimum TLB purge count to ensure a full purge */
 #define GRUMAXINVAL		1024UL
 
+int cch_allocate(struct gru_context_configuration_handle *cch,
+       int asidval, unsigned long cbrmap, unsigned long dsrmap);
 
-/* Extract the status field from a kernel handle */
-#define GET_MSEG_HANDLE_STATUS(h)	(((*(unsigned long *)(h)) >> 16) & 3)
-
-static inline void start_instruction(void *h)
-{
-	unsigned long *w0 = h;
-
-	wmb();		/* setting CMD bit must be last */
-	*w0 = *w0 | 1;
-	gru_flush_cache(h);
-}
-
-static inline int wait_instruction_complete(void *h)
-{
-	int status;
-
-	do {
-		cpu_relax();
-		barrier();
-		status = GET_MSEG_HANDLE_STATUS(h);
-	} while (status == CCHSTATUS_ACTIVE);
-	return status;
-}
-
-#if defined CONFIG_IA64
-static inline void cch_allocate_set_asids(
-		  struct gru_context_configuration_handle *cch, int asidval)
-{
-	int i;
-
-	for (i = 0; i <= RGN_HPAGE; i++) {  /*  assume HPAGE is last region */
-		cch->asid[i] = (asidval++);
-#if 0
-		/* ZZZ hugepages not supported yet */
-		if (i == RGN_HPAGE)
-			cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift);
-		else
-#endif
-			cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT);
-	}
-}
-#elif defined CONFIG_X86_64
-static inline void cch_allocate_set_asids(
-		  struct gru_context_configuration_handle *cch, int asidval)
-{
-	int i;
-
-	for (i = 0; i < 8; i++) {
-		cch->asid[i] = asidval++;
-		cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) |
-			GRU_SIZEAVAIL(21);
-	}
-}
-#endif
-
-static inline int cch_allocate(struct gru_context_configuration_handle *cch,
-			       int asidval, unsigned long cbrmap,
-			       unsigned long dsrmap)
-{
-	cch_allocate_set_asids(cch, asidval);
-	cch->dsr_allocation_map = dsrmap;
-	cch->cbr_allocation_map = cbrmap;
-	cch->opc = CCHOP_ALLOCATE;
-	start_instruction(cch);
-	return wait_instruction_complete(cch);
-}
-
-static inline int cch_start(struct gru_context_configuration_handle *cch)
-{
-	cch->opc = CCHOP_START;
-	start_instruction(cch);
-	return wait_instruction_complete(cch);
-}
-
-static inline int cch_interrupt(struct gru_context_configuration_handle *cch)
-{
-	cch->opc = CCHOP_INTERRUPT;
-	start_instruction(cch);
-	return wait_instruction_complete(cch);
-}
-
-static inline int cch_deallocate(struct gru_context_configuration_handle *cch)
-{
-	cch->opc = CCHOP_DEALLOCATE;
-	start_instruction(cch);
-	return wait_instruction_complete(cch);
-}
-
-static inline int cch_interrupt_sync(struct gru_context_configuration_handle
-				     *cch)
-{
-	cch->opc = CCHOP_INTERRUPT_SYNC;
-	start_instruction(cch);
-	return wait_instruction_complete(cch);
-}
-
-static inline int tgh_invalidate(struct gru_tlb_global_handle *tgh,
-				 unsigned long vaddr, unsigned long vaddrmask,
-				 int asid, int pagesize, int global, int n,
-				 unsigned short ctxbitmap)
-{
-	tgh->vaddr = vaddr;
-	tgh->asid = asid;
-	tgh->pagesize = pagesize;
-	tgh->n = n;
-	tgh->global = global;
-	tgh->vaddrmask = vaddrmask;
-	tgh->ctxbitmap = ctxbitmap;
-	tgh->opc = TGHOP_TLBINV;
-	start_instruction(tgh);
-	return wait_instruction_complete(tgh);
-}
-
-static inline void tfh_write_only(struct gru_tlb_fault_handle *tfh,
-				  unsigned long pfn, unsigned long vaddr,
-				  int asid, int dirty, int pagesize)
-{
-	tfh->fillasid = asid;
-	tfh->fillvaddr = vaddr;
-	tfh->pfn = pfn;
-	tfh->dirty = dirty;
-	tfh->pagesize = pagesize;
-	tfh->opc = TFHOP_WRITE_ONLY;
-	start_instruction(tfh);
-}
-
-static inline void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
-				     unsigned long paddr, int gaa,
-				     unsigned long vaddr, int asid, int dirty,
-				     int pagesize)
-{
-	tfh->fillasid = asid;
-	tfh->fillvaddr = vaddr;
-	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
-	tfh->gaa = gaa;
-	tfh->dirty = dirty;
-	tfh->pagesize = pagesize;
-	tfh->opc = TFHOP_WRITE_RESTART;
-	start_instruction(tfh);
-}
-
-static inline void tfh_restart(struct gru_tlb_fault_handle *tfh)
-{
-	tfh->opc = TFHOP_RESTART;
-	start_instruction(tfh);
-}
-
-static inline void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
-{
-	tfh->opc = TFHOP_USER_POLLING_MODE;
-	start_instruction(tfh);
-}
-
-static inline void tfh_exception(struct gru_tlb_fault_handle *tfh)
-{
-	tfh->opc = TFHOP_EXCEPTION;
-	start_instruction(tfh);
-}
+int cch_start(struct gru_context_configuration_handle *cch);
+int cch_interrupt(struct gru_context_configuration_handle *cch);
+int cch_deallocate(struct gru_context_configuration_handle *cch);
+int cch_interrupt_sync(struct gru_context_configuration_handle *cch);
+int tgh_invalidate(struct gru_tlb_global_handle *tgh, unsigned long vaddr,
+	unsigned long vaddrmask, int asid, int pagesize, int global, int n,
+	unsigned short ctxbitmap);
+void tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long pfn,
+	unsigned long vaddr, int asid, int dirty, int pagesize);
+void tfh_write_restart(struct gru_tlb_fault_handle *tfh, unsigned long paddr,
+	int gaa, unsigned long vaddr, int asid, int dirty, int pagesize);
+void tfh_restart(struct gru_tlb_fault_handle *tfh);
+void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh);
+void tfh_exception(struct gru_tlb_fault_handle *tfh);
 
 #endif /* __GRUHANDLES_H__ */
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:06.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:08.000000000 -0600
@@ -239,6 +239,9 @@ struct gru_stats_s {
 
 };
 
+enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
+	cchop_deallocate, tghop_invalidate, mcsop_last};
+
 #define OPT_DPRINT	1
 #define OPT_STATS	2
 #define GRU_QUICKLOOK	4

[Patch 07/14] Add statistics to the GRU context management functions

[steiner]

[-- Attachment #1: uv_outofline_mcs_instructions_statistics --]
[-- Type: text/plain, Size: 4684 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add statistics to the GRU context management functions.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gruhandles.c |   12 +++++++++++
 drivers/misc/sgi-gru/gruprocfs.c  |   39 ++++++++++++++++++++++++++++++++++++++
 drivers/misc/sgi-gru/grutables.h  |    8 +++++++
 3 files changed, 59 insertions(+)

Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.c	2009-01-15 08:18:08.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruhandles.c	2009-01-15 08:18:09.000000000 -0600
@@ -35,6 +35,16 @@
 /* Extract the status field from a kernel handle */
 #define GET_MSEG_HANDLE_STATUS(h)	(((*(unsigned long *)(h)) >> 16) & 3)
 
+struct mcs_op_statistic mcs_op_statistics[mcsop_last];
+
+static void update_mcs_stats(enum mcs_op op, unsigned long clks)
+{
+	atomic_long_inc(&mcs_op_statistics[op].count);
+	atomic_long_add(clks, &mcs_op_statistics[op].total);
+	if (mcs_op_statistics[op].max < clks)
+		mcs_op_statistics[op].max = clks;
+}
+
 static void start_instruction(void *h)
 {
 	unsigned long *w0 = h;
@@ -57,6 +67,8 @@ static int wait_instruction_complete(voi
 		if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time))
 			panic("GRU %p is malfunctioning\n", h);
 	}
+	if (gru_options & OPT_STATS)
+		update_mcs_stats(opc, get_cycles() - start_time);
 	return status;
 }
 
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2009-01-15 08:18:06.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2009-01-15 08:18:09.000000000 -0600
@@ -122,6 +122,30 @@ static ssize_t statistics_write(struct f
 	return count;
 }
 
+static int mcs_statistics_show(struct seq_file *s, void *p)
+{
+	int op;
+	unsigned long total, count, max;
+	static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt",
+		"cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"};
+
+	for (op = 0; op < mcsop_last; op++) {
+		count = atomic_long_read(&mcs_op_statistics[op].count);
+		total = atomic_long_read(&mcs_op_statistics[op].total);
+		max = mcs_op_statistics[op].max;
+		seq_printf(s, "%-20s%12ld%12ld%12ld\n", id[op], count,
+			   count ? total / count : 0, max);
+	}
+	return 0;
+}
+
+static ssize_t mcs_statistics_write(struct file *file,
+			const char __user *userbuf, size_t count, loff_t *data)
+{
+	memset(mcs_op_statistics, 0, sizeof(mcs_op_statistics));
+	return count;
+}
+
 static int options_show(struct seq_file *s, void *p)
 {
 	seq_printf(s, "0x%lx\n", gru_options);
@@ -137,6 +161,7 @@ static ssize_t options_write(struct file
 	if (copy_from_user
 	    (buf, userbuf, count < sizeof(buf) ? count : sizeof(buf)))
 		return -EFAULT;
+	buf[count - 1] = '\0';
 	if (!strict_strtoul(buf, 10, &val))
 		gru_options = val;
 
@@ -233,6 +258,11 @@ static int statistics_open(struct inode 
 	return single_open(file, statistics_show, NULL);
 }
 
+static int mcs_statistics_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mcs_statistics_show, NULL);
+}
+
 static int options_open(struct inode *inode, struct file *file)
 {
 	return single_open(file, options_show, NULL);
@@ -257,6 +287,14 @@ static const struct file_operations stat
 	.release 	= single_release,
 };
 
+static const struct file_operations mcs_statistics_fops = {
+	.open 		= mcs_statistics_open,
+	.read 		= seq_read,
+	.write 		= mcs_statistics_write,
+	.llseek 	= seq_lseek,
+	.release 	= single_release,
+};
+
 static const struct file_operations options_fops = {
 	.open 		= options_open,
 	.read 		= seq_read,
@@ -285,6 +323,7 @@ static struct proc_entry {
 	struct proc_dir_entry *entry;
 } proc_files[] = {
 	{"statistics", 0644, &statistics_fops},
+	{"mcs_statistics", 0644, &mcs_statistics_fops},
 	{"debug_options", 0644, &options_fops},
 	{"cch_status", 0444, &cch_fops},
 	{"gru_status", 0444, &gru_fops},
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:08.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:09.000000000 -0600
@@ -242,6 +242,14 @@ struct gru_stats_s {
 enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
 	cchop_deallocate, tghop_invalidate, mcsop_last};
 
+struct mcs_op_statistic {
+	atomic_long_t	count;
+	atomic_long_t	total;
+	unsigned long	max;
+};
+
+extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];
+
 #define OPT_DPRINT	1
 #define OPT_STATS	2
 #define GRU_QUICKLOOK	4

[Patch 08/14] Add support for a user to explicitly unload a GRU context

[steiner]

[-- Attachment #1: uv_gru_unload_all_contexts --]
[-- Type: text/plain, Size: 2020 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add support for a user to explicitly unload a GRU context.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c |   36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:06.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:10.000000000 -0600
@@ -32,6 +32,7 @@
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/security.h>
 #include <asm/pgtable.h>
 #include "gru.h"
 #include "grutables.h"
@@ -575,6 +576,38 @@ int gru_get_exception_detail(unsigned lo
 /*
  * User request to unload a context. Content is saved for possible reload.
  */
+static int gru_unload_all_contexts(void)
+{
+	struct gru_thread_state *gts;
+	struct gru_state *gru;
+	int maxgid, gid, ctxnum;
+	int nodesperblade;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	if (num_online_nodes() > 1 &&
+			(uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
+		nodesperblade = 2;
+	else
+		nodesperblade = 1;
+	maxgid = GRU_CHIPLETS_PER_BLADE * num_online_nodes() / nodesperblade;
+	for (gid = 0; gid < maxgid; gid++) {
+		gru = GID_TO_GRU(gid);
+		spin_lock(&gru->gs_lock);
+		for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
+			gts = gru->gs_gts[ctxnum];
+			if (gts && mutex_trylock(&gts->ts_ctxlock)) {
+				spin_unlock(&gru->gs_lock);
+				gru_unload_context(gts, 1);
+				gru_unlock_gts(gts);
+				spin_lock(&gru->gs_lock);
+			}
+		}
+		spin_unlock(&gru->gs_lock);
+	}
+	return 0;
+}
+
 int gru_user_unload_context(unsigned long arg)
 {
 	struct gru_thread_state *gts;
@@ -586,6 +619,9 @@ int gru_user_unload_context(unsigned lon
 
 	gru_dbg(grudev, "gseg 0x%lx\n", req.gseg);
 
+	if (!req.gseg)
+		return gru_unload_all_contexts();
+
 	gts = gru_find_lock_gts(req.gseg);
 	if (!gts)
 		return -EINVAL;

[Patch 09/14] ASID (context management) bug fixes

[steiner]

[-- Attachment #1: uv_gru_asid_fixes --]
[-- Type: text/plain, Size: 3724 bytes --]

From: Jack Steiner <steiner@sgi.com>

This patch fixes bugs related to ASID (context id) management
in the GRU driver.  These changes are all internal to the SGI
GRU driver and have no effect on the base kernel.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufile.c |    1 +
 drivers/misc/sgi-gru/grumain.c |   13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2009-01-15 08:18:06.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufile.c	2009-01-15 08:18:11.000000000 -0600
@@ -286,6 +286,7 @@ static void gru_init_chiplet(struct gru_
 	gru->gs_blade_id = bid;
 	gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
 	gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
+	gru->gs_asid_limit = MAX_ASID;
 	gru_tgh_flush_init(gru);
 	gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n",
 		bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr,
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2009-01-15 08:18:06.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grumain.c	2009-01-15 08:18:11.000000000 -0600
@@ -79,7 +79,6 @@ static int gru_wrap_asid(struct gru_stat
 	gru_dbg(grudev, "gid %d\n", gru->gs_gid);
 	STAT(asid_wrap);
 	gru->gs_asid_gen++;
-	gru_flush_all_tlb(gru);
 	return MIN_ASID;
 }
 
@@ -93,6 +92,7 @@ static int gru_reset_asid_limit(struct g
 	limit = MAX_ASID;
 	if (asid >= limit)
 		asid = gru_wrap_asid(gru);
+	gru_flush_all_tlb(gru);
 	gid = gru->gs_gid;
 again:
 	for (i = 0; i < GRU_NUM_CCH; i++) {
@@ -131,12 +131,10 @@ static int gru_assign_asid(struct gru_st
 {
 	int asid;
 
-	spin_lock(&gru->gs_asid_lock);
 	gru->gs_asid += ASID_INC;
 	asid = gru->gs_asid;
 	if (asid >= gru->gs_asid_limit)
 		asid = gru_reset_asid_limit(gru, asid);
-	spin_unlock(&gru->gs_asid_lock);
 
 	gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
 	return asid;
@@ -227,7 +225,9 @@ static int gru_load_mm_tracker(struct gr
 	spin_lock(&gms->ms_asid_lock);
 	asid = asids->mt_asid;
 
-	if (asid == 0 || asids->mt_asid_gen != gru->gs_asid_gen) {
+	spin_lock(&gru->gs_asid_lock);
+	if (asid == 0 || (asids->mt_ctxbitmap == 0 && asids->mt_asid_gen !=
+			  gru->gs_asid_gen)) {
 		asid = gru_assign_asid(gru);
 		asids->mt_asid = asid;
 		asids->mt_asid_gen = gru->gs_asid_gen;
@@ -235,6 +235,7 @@ static int gru_load_mm_tracker(struct gr
 	} else {
 		STAT(asid_reuse);
 	}
+	spin_unlock(&gru->gs_asid_lock);
 
 	BUG_ON(asids->mt_ctxbitmap & ctxbitmap);
 	asids->mt_ctxbitmap |= ctxbitmap;
@@ -259,10 +260,12 @@ static void gru_unload_mm_tracker(struct
 	asids = &gms->ms_asids[gru->gs_gid];
 	ctxbitmap = (1 << gts->ts_ctxnum);
 	spin_lock(&gms->ms_asid_lock);
+	spin_lock(&gru->gs_asid_lock);
 	BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
 	asids->mt_ctxbitmap ^= ctxbitmap;
 	gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
 		gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
+	spin_unlock(&gru->gs_asid_lock);
 	spin_unlock(&gms->ms_asid_lock);
 }
 
@@ -412,6 +415,7 @@ static void gru_free_gru_context(struct 
 	__clear_bit(gts->ts_ctxnum, &gru->gs_context_map);
 	gts->ts_ctxnum = NULLCTX;
 	gts->ts_gru = NULL;
+	gts->ts_blade = -1;
 	spin_unlock(&gru->gs_lock);
 
 	gts_drop(gts);
@@ -731,6 +735,7 @@ again:
 		}
 		reserve_gru_resources(gru, gts);
 		gts->ts_gru = gru;
+		gts->ts_blade = gru->gs_blade_id;
 		gts->ts_ctxnum =
 		    find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);
 		BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH);

[Patch 10/14] Restructure the GRU vtop functions

[steiner]

[-- Attachment #1: uv_gru_vtop --]
[-- Type: text/plain, Size: 3003 bytes --]

From: Jack Steiner <steiner@sgi.com>

Restructure the GRU vtop functions in preparation for future changes.
This patch simply moves code around & does not change the algorithm.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c |   68 +++++++++++++++++++++++++---------------
 1 file changed, 44 insertions(+), 24 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:10.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:12.000000000 -0600
@@ -267,6 +267,44 @@ err:
 	return 1;
 }
 
+static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr,
+		    int write, int atomic, unsigned long *gpa, int *pageshift)
+{
+	struct mm_struct *mm = gts->ts_mm;
+	struct vm_area_struct *vma;
+	unsigned long paddr;
+	int ret, ps;
+
+	vma = find_vma(mm, vaddr);
+	if (!vma)
+		goto inval;
+
+	/*
+	 * Atomic lookup is faster & usually works even if called in non-atomic
+	 * context.
+	 */
+	rmb();	/* Must/check ms_range_active before loading PTEs */
+	ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps);
+	if (ret) {
+		if (atomic)
+			goto upm;
+		if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps))
+			goto inval;
+	}
+	if (is_gru_paddr(paddr))
+		goto inval;
+	paddr = paddr & ~((1UL << ps) - 1);
+	*gpa = uv_soc_phys_ram_to_gpa(paddr);
+	*pageshift = ps;
+	return 0;
+
+inval:
+	return -1;
+upm:
+	return -2;
+}
+
+
 /*
  * Drop a TLB entry into the GRU. The fault is described by info in an TFH.
  *	Input:
@@ -281,10 +319,8 @@ static int gru_try_dropin(struct gru_thr
 			  struct gru_tlb_fault_handle *tfh,
 			  unsigned long __user *cb)
 {
-	struct mm_struct *mm = gts->ts_mm;
-	struct vm_area_struct *vma;
-	int pageshift, asid, write, ret;
-	unsigned long paddr, gpa, vaddr;
+	int pageshift = 0, asid, write, ret, atomic = !cb;
+	unsigned long gpa = 0, vaddr = 0;
 
 	/*
 	 * NOTE: The GRU contains magic hardware that eliminates races between
@@ -318,28 +354,12 @@ static int gru_try_dropin(struct gru_thr
 	if (atomic_read(&gts->ts_gms->ms_range_active))
 		goto failactive;
 
-	vma = find_vma(mm, vaddr);
-	if (!vma)
-		goto failinval;
-
-	/*
-	 * Atomic lookup is faster & usually works even if called in non-atomic
-	 * context.
-	 */
-	rmb();	/* Must/check ms_range_active before loading PTEs */
-	ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &pageshift);
-	if (ret) {
-		if (!cb)
-			goto failupm;
-		if (non_atomic_pte_lookup(vma, vaddr, write, &paddr,
-					  &pageshift))
-			goto failinval;
-	}
-	if (is_gru_paddr(paddr))
+	ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
+	if (ret == -1)
 		goto failinval;
+	if (ret == -2)
+		goto failupm;
 
-	paddr = paddr & ~((1UL << pageshift) - 1);
-	gpa = uv_soc_phys_ram_to_gpa(paddr);
 	gru_cb_set_istatus_active(cb);
 	tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
 			  GRU_PAGESIZE(pageshift));

[Patch 11/14] Add support to the GRU driver for message queue interrupts

[steiner]

[-- Attachment #1: uv_gru_message_queue --]
[-- Type: text/plain, Size: 37044 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add support to the GRU driver for message queue interrupts.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Dean Nelson <dcn@sgi.com>

---

 drivers/misc/sgi-gru/grukservices.c |  104 +++++++++-------
 drivers/misc/sgi-gru/grukservices.h |   33 +++--
 drivers/misc/sgi-xp/xpc.h           |   27 ++--
 drivers/misc/sgi-xp/xpc_channel.c   |    8 +
 drivers/misc/sgi-xp/xpc_main.c      |    6 
 drivers/misc/sgi-xp/xpc_sn2.c       |   10 +
 drivers/misc/sgi-xp/xpc_uv.c        |  201 ++++++++++++++++++++++++--------
 7 files changed, 274 insertions(+), 115 deletions(-)

Index: linux/drivers/misc/sgi-gru/grukservices.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.c	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grukservices.c	2009-01-21 12:20:53.000000000 -0600
@@ -52,8 +52,10 @@
  */
 
 /* Blade percpu resources PERMANENTLY reserved for kernel use */
-#define GRU_NUM_KERNEL_CBR      1
+#define GRU_NUM_KERNEL_CBR	1
 #define GRU_NUM_KERNEL_DSR_BYTES 256
+#define GRU_NUM_KERNEL_DSR_CL	(GRU_NUM_KERNEL_DSR_BYTES /		\
+					GRU_CACHE_LINE_BYTES)
 #define KERNEL_CTXNUM           15
 
 /* GRU instruction attributes for all instructions */
@@ -94,7 +96,6 @@ struct message_header {
 	char	fill;
 };
 
-#define QLINES(mq)	((mq) + offsetof(struct message_queue, qlines))
 #define HSTATUS(mq, h)	((mq) + offsetof(struct message_queue, hstatus[h]))
 
 static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr)
@@ -250,7 +251,8 @@ static inline void restore_present2(void
  * Create a message queue.
  * 	qlines - message queue size in cache lines. Includes 2-line header.
  */
-int gru_create_message_queue(void *p, unsigned int bytes)
+int gru_create_message_queue(struct gru_message_queue_desc *mqd,
+		void *p, unsigned int bytes, int nasid, int vector, int apicid)
 {
 	struct message_queue *mq = p;
 	unsigned int qlines;
@@ -265,6 +267,12 @@ int gru_create_message_queue(void *p, un
 	mq->hstatus[0] = 0;
 	mq->hstatus[1] = 1;
 	mq->head = gru_mesq_head(2, qlines / 2 + 1);
+	mqd->mq = mq;
+	mqd->mq_gpa = uv_gpa(mq);
+	mqd->qlines = qlines;
+	mqd->interrupt_pnode = UV_NASID_TO_PNODE(nasid);
+	mqd->interrupt_vector = vector;
+	mqd->interrupt_apicid = apicid;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(gru_create_message_queue);
@@ -277,8 +285,8 @@ EXPORT_SYMBOL_GPL(gru_create_message_que
  *		-1 - if mesq sent successfully but queue not full
  *		>0 - unexpected error. MQE_xxx returned
  */
-static int send_noop_message(void *cb,
-				unsigned long mq, void *mesg)
+static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
+				void *mesg)
 {
 	const struct message_header noop_header = {
 					.present = MQS_NOOP, .lines = 1};
@@ -289,7 +297,7 @@ static int send_noop_message(void *cb,
 	STAT(mesq_noop);
 	save_mhdr = *mhdr;
 	*mhdr = noop_header;
-	gru_mesq(cb, mq, gru_get_tri(mhdr), 1, IMA);
+	gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);
 	ret = gru_wait(cb);
 
 	if (ret) {
@@ -313,7 +321,7 @@ static int send_noop_message(void *cb,
 			break;
 		case CBSS_PUT_NACKED:
 			STAT(mesq_noop_put_nacked);
-			m = mq + (gru_get_amo_value_head(cb) << 6);
+			m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
 			gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
 						IMA);
 			if (gru_wait(cb) == CBS_IDLE)
@@ -333,30 +341,20 @@ static int send_noop_message(void *cb,
 /*
  * Handle a gru_mesq full.
  */
-static int send_message_queue_full(void *cb,
-			   unsigned long mq, void *mesg, int lines)
+static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,
+				void *mesg, int lines)
 {
 	union gru_mesqhead mqh;
 	unsigned int limit, head;
 	unsigned long avalue;
-	int half, qlines, save;
+	int half, qlines;
 
 	/* Determine if switching to first/second half of q */
 	avalue = gru_get_amo_value(cb);
 	head = gru_get_amo_value_head(cb);
 	limit = gru_get_amo_value_limit(cb);
 
-	/*
-	 * Fetch "qlines" from the queue header. Since the queue may be
-	 * in memory that can't be accessed using socket addresses, use
-	 * the GRU to access the data. Use DSR space from the message.
-	 */
-	save = *(int *)mesg;
-	gru_vload(cb, QLINES(mq), gru_get_tri(mesg), XTYPE_W, 1, 1, IMA);
-	if (gru_wait(cb) != CBS_IDLE)
-		goto cberr;
-	qlines = *(int *)mesg;
-	*(int *)mesg = save;
+	qlines = mqd->qlines;
 	half = (limit != qlines);
 
 	if (half)
@@ -365,7 +363,7 @@ static int send_message_queue_full(void 
 		mqh = gru_mesq_head(2, qlines / 2 + 1);
 
 	/* Try to get lock for switching head pointer */
-	gru_gamir(cb, EOP_IR_CLR, HSTATUS(mq, half), XTYPE_DW, IMA);
+	gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);
 	if (gru_wait(cb) != CBS_IDLE)
 		goto cberr;
 	if (!gru_get_amo_value(cb)) {
@@ -375,8 +373,8 @@ static int send_message_queue_full(void 
 
 	/* Got the lock. Send optional NOP if queue not full, */
 	if (head != limit) {
-		if (send_noop_message(cb, mq, mesg)) {
-			gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half),
+		if (send_noop_message(cb, mqd, mesg)) {
+			gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),
 					XTYPE_DW, IMA);
 			if (gru_wait(cb) != CBS_IDLE)
 				goto cberr;
@@ -387,14 +385,16 @@ static int send_message_queue_full(void 
 	}
 
 	/* Then flip queuehead to other half of queue. */
-	gru_gamer(cb, EOP_ERR_CSWAP, mq, XTYPE_DW, mqh.val, avalue, IMA);
+	gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,
+							IMA);
 	if (gru_wait(cb) != CBS_IDLE)
 		goto cberr;
 
 	/* If not successfully in swapping queue head, clear the hstatus lock */
 	if (gru_get_amo_value(cb) != avalue) {
 		STAT(mesq_qf_switch_head_failed);
-		gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half), XTYPE_DW, IMA);
+		gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,
+							IMA);
 		if (gru_wait(cb) != CBS_IDLE)
 			goto cberr;
 	}
@@ -404,15 +404,25 @@ cberr:
 	return MQE_UNEXPECTED_CB_ERR;
 }
 
+/*
+ * Send a cross-partition interrupt to the SSI that contains the target
+ * message queue. Normally, the interrupt is automatically delivered by hardware
+ * but some error conditions require explicit delivery.
+ */
+static void send_message_queue_interrupt(struct gru_message_queue_desc *mqd)
+{
+	if (mqd->interrupt_vector)
+		uv_hub_send_ipi(mqd->interrupt_pnode, mqd->interrupt_apicid,
+				mqd->interrupt_vector);
+}
+
 
 /*
  * Handle a gru_mesq failure. Some of these failures are software recoverable
  * or retryable.
  */
-static int send_message_failure(void *cb,
-				unsigned long mq,
-				void *mesg,
-				int lines)
+static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd,
+				void *mesg, int lines)
 {
 	int substatus, ret = 0;
 	unsigned long m;
@@ -429,7 +439,7 @@ static int send_message_failure(void *cb
 		break;
 	case CBSS_QLIMIT_REACHED:
 		STAT(mesq_send_qlimit_reached);
-		ret = send_message_queue_full(cb, mq, mesg, lines);
+		ret = send_message_queue_full(cb, mqd, mesg, lines);
 		break;
 	case CBSS_AMO_NACKED:
 		STAT(mesq_send_amo_nacked);
@@ -437,12 +447,14 @@ static int send_message_failure(void *cb
 		break;
 	case CBSS_PUT_NACKED:
 		STAT(mesq_send_put_nacked);
-		m = mq + (gru_get_amo_value_head(cb) << 6);
+		m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
 		gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
-		if (gru_wait(cb) == CBS_IDLE)
+		if (gru_wait(cb) == CBS_IDLE) {
 			ret = MQE_OK;
-		else
+			send_message_queue_interrupt(mqd);
+		} else {
 			ret = MQE_UNEXPECTED_CB_ERR;
+		}
 		break;
 	default:
 		BUG();
@@ -452,12 +464,12 @@ static int send_message_failure(void *cb
 
 /*
  * Send a message to a message queue
- * 	cb	GRU control block to use to send message
- * 	mq	message queue
+ * 	mqd	message queue descriptor
  * 	mesg	message. ust be vaddr within a GSEG
  * 	bytes	message size (<= 2 CL)
  */
-int gru_send_message_gpa(unsigned long mq, void *mesg, unsigned int bytes)
+int gru_send_message_gpa(struct gru_message_queue_desc *mqd, void *mesg,
+				unsigned int bytes)
 {
 	struct message_header *mhdr;
 	void *cb;
@@ -481,10 +493,10 @@ int gru_send_message_gpa(unsigned long m
 
 	do {
 		ret = MQE_OK;
-		gru_mesq(cb, mq, gru_get_tri(mhdr), clines, IMA);
+		gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), clines, IMA);
 		istatus = gru_wait(cb);
 		if (istatus != CBS_IDLE)
-			ret = send_message_failure(cb, mq, dsr, clines);
+			ret = send_message_failure(cb, mqd, dsr, clines);
 	} while (ret == MQIE_AGAIN);
 	gru_free_cpu_resources(cb, dsr);
 
@@ -497,9 +509,9 @@ EXPORT_SYMBOL_GPL(gru_send_message_gpa);
 /*
  * Advance the receive pointer for the queue to the next message.
  */
-void gru_free_message(void *rmq, void *mesg)
+void gru_free_message(struct gru_message_queue_desc *mqd, void *mesg)
 {
-	struct message_queue *mq = rmq;
+	struct message_queue *mq = mqd->mq;
 	struct message_header *mhdr = mq->next;
 	void *next, *pnext;
 	int half = -1;
@@ -529,16 +541,16 @@ EXPORT_SYMBOL_GPL(gru_free_message);
  * present. User must call next_message() to move to next message.
  * 	rmq	message queue
  */
-void *gru_get_next_message(void *rmq)
+void *gru_get_next_message(struct gru_message_queue_desc *mqd)
 {
-	struct message_queue *mq = rmq;
+	struct message_queue *mq = mqd->mq;
 	struct message_header *mhdr = mq->next;
 	int present = mhdr->present;
 
 	/* skip NOOP messages */
 	STAT(mesq_receive);
 	while (present == MQS_NOOP) {
-		gru_free_message(rmq, mhdr);
+		gru_free_message(mqd, mhdr);
 		mhdr = mq->next;
 		present = mhdr->present;
 	}
@@ -576,7 +588,7 @@ int gru_copy_gpa(unsigned long dest_gpa,
 	if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
 		return MQE_BUG_NO_RESOURCES;
 	gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr),
-		  XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_BYTES, IMA);
+		  XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_CL, IMA);
 	ret = gru_wait(cb);
 	gru_free_cpu_resources(cb, dsr);
 	return ret;
@@ -611,7 +623,7 @@ static int quicktest(struct gru_state *g
 
 	if (word0 != word1 || word0 != MAGIC) {
 		printk
-		    ("GRU quicktest err: gru %d, found 0x%lx, expected 0x%lx\n",
+		    ("GRU quicktest err: gid %d, found 0x%lx, expected 0x%lx\n",
 		     gru->gs_gid, word1, MAGIC);
 		BUG();		/* ZZZ should not be fatal */
 	}
Index: linux/drivers/misc/sgi-gru/grukservices.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.h	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grukservices.h	2009-01-21 11:40:09.000000000 -0600
@@ -41,6 +41,15 @@
  * 	- gru_create_message_queue() needs interrupt vector info
  */
 
+struct gru_message_queue_desc {
+	void		*mq;			/* message queue vaddress */
+	unsigned long	mq_gpa;			/* global address of mq */
+	int		qlines;			/* queue size in CL */
+	int		interrupt_vector;	/* interrupt vector */
+	int		interrupt_pnode;	/* pnode for interrupt */
+	int		interrupt_apicid;	/* lapicid for interrupt */
+};
+
 /*
  * Initialize a user allocated chunk of memory to be used as
  * a message queue. The caller must ensure that the queue is
@@ -51,14 +60,19 @@
  * to manage the queue.
  *
  *  Input:
- * 	p	pointer to user allocated memory.
+ * 	mqd	pointer to message queue descriptor
+ * 	p	pointer to user allocated mesq memory.
  * 	bytes	size of message queue in bytes
+ *      vector	interrupt vector (zero if no interrupts)
+ *      nasid	nasid of blade where interrupt is delivered
+ *      apicid	apicid of cpu for interrupt
  *
  *  Errors:
  *  	0	OK
  *  	>0	error
  */
-extern int gru_create_message_queue(void *p, unsigned int bytes);
+extern int gru_create_message_queue(struct gru_message_queue_desc *mqd,
+		void *p, unsigned int bytes, int nasid, int vector, int apicid);
 
 /*
  * Send a message to a message queue.
@@ -68,7 +82,7 @@ extern int gru_create_message_queue(void
  *
  *
  *   Input:
- * 	xmq	message queue - must be a UV global physical address
+ * 	mqd	pointer to message queue descriptor
  * 	mesg	pointer to message. Must be 64-bit aligned
  * 	bytes	size of message in bytes
  *
@@ -77,8 +91,8 @@ extern int gru_create_message_queue(void
  *     >0	Send failure - see error codes below
  *
  */
-extern int gru_send_message_gpa(unsigned long mq_gpa, void *mesg,
-						unsigned int bytes);
+extern int gru_send_message_gpa(struct gru_message_queue_desc *mqd,
+			void *mesg, unsigned int bytes);
 
 /* Status values for gru_send_message() */
 #define MQE_OK			0	/* message sent successfully */
@@ -94,10 +108,11 @@ extern int gru_send_message_gpa(unsigned
  * API extensions may allow for out-of-order freeing.
  *
  *   Input
- * 	mq	message queue
+ * 	mqd	pointer to message queue descriptor
  * 	mesq	message being freed
  */
-extern void gru_free_message(void *mq, void *mesq);
+extern void gru_free_message(struct gru_message_queue_desc *mqd,
+			     void *mesq);
 
 /*
  * Get next message from message queue. Returns pointer to
@@ -106,13 +121,13 @@ extern void gru_free_message(void *mq, v
  * in order to move the queue pointers to next message.
  *
  *   Input
- * 	mq	message queue
+ * 	mqd	pointer to message queue descriptor
  *
  *   Output:
  *	p	pointer to message
  *	NULL	no message available
  */
-extern void *gru_get_next_message(void *mq);
+extern void *gru_get_next_message(struct gru_message_queue_desc *mqd);
 
 
 /*
Index: linux/drivers/misc/sgi-xp/xpc.h
===================================================================
--- linux.orig/drivers/misc/sgi-xp/xpc.h	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-xp/xpc.h	2009-01-21 11:40:09.000000000 -0600
@@ -92,7 +92,9 @@ struct xpc_rsvd_page {
 	u8 pad1[3];		/* align to next u64 in 1st 64-byte cacheline */
 	union {
 		unsigned long vars_pa;	/* phys address of struct xpc_vars */
-		unsigned long activate_mq_gpa; /* gru phy addr of activate_mq */
+		unsigned long activate_gru_mq_desc_gpa; /* phys addr of */
+							/* activate mq's */
+							/* gru mq descriptor */
 	} sn;
 	unsigned long ts_jiffies; /* timestamp when rsvd pg was setup by XPC */
 	u64 pad2[10];		/* align to last u64 in 2nd 64-byte cacheline */
@@ -189,7 +191,9 @@ struct xpc_gru_mq_uv {
 	int irq;		/* irq raised when message is received in mq */
 	int mmr_blade;		/* blade where watchlist was allocated from */
 	unsigned long mmr_offset; /* offset of irq mmr located on mmr_blade */
+	unsigned long mmr_value; /* value of irq mmr located on mmr_blade */
 	int watchlist_num;	/* number of watchlist allocatd by BIOS */
+	void *gru_mq_desc;	/* opaque structure used by the GRU driver */
 };
 
 /*
@@ -197,6 +201,7 @@ struct xpc_gru_mq_uv {
  * heartbeat, partition active state, and channel state. This is UV only.
  */
 struct xpc_activate_mq_msghdr_uv {
+	unsigned int gru_msg_hdr; /* FOR GRU INTERNAL USE ONLY */
 	short partid;		/* sender's partid */
 	u8 act_state;		/* sender's act_state at time msg sent */
 	u8 type;		/* message's type */
@@ -232,7 +237,7 @@ struct xpc_activate_mq_msg_heartbeat_req
 struct xpc_activate_mq_msg_activate_req_uv {
 	struct xpc_activate_mq_msghdr_uv hdr;
 	unsigned long rp_gpa;
-	unsigned long activate_mq_gpa;
+	unsigned long activate_gru_mq_desc_gpa;
 };
 
 struct xpc_activate_mq_msg_deactivate_req_uv {
@@ -263,7 +268,7 @@ struct xpc_activate_mq_msg_chctl_openrep
 	short ch_number;
 	short remote_nentries;	/* ??? Is this needed? What is? */
 	short local_nentries;	/* ??? Is this needed? What is? */
-	unsigned long local_notify_mq_gpa;
+	unsigned long notify_gru_mq_desc_gpa;
 };
 
 /*
@@ -510,8 +515,8 @@ struct xpc_channel_sn2 {
 };
 
 struct xpc_channel_uv {
-	unsigned long remote_notify_mq_gpa;	/* gru phys address of remote */
-						/* partition's notify mq */
+	void *cached_notify_gru_mq_desc; /* remote partition's notify mq's */
+					 /* gru mq descriptor */
 
 	struct xpc_send_msg_slot_uv *send_msg_slots;
 	struct xpc_notify_mq_msg_uv *recv_msg_slots;
@@ -681,8 +686,12 @@ struct xpc_partition_sn2 {
 };
 
 struct xpc_partition_uv {
-	unsigned long remote_activate_mq_gpa;	/* gru phys address of remote */
-						/* partition's activate mq */
+	unsigned long activate_gru_mq_desc_gpa;	/* phys addr of parititon's */
+						/* activate mq's gru mq */
+						/* descriptor */
+	void *cached_activate_gru_mq_desc; /* cached copy of partition's */
+					   /* activate mq's gru mq descriptor */
+	struct mutex cached_activate_gru_mq_desc_mutex;
 	spinlock_t flags_lock;	/* protect updating of flags */
 	unsigned int flags;	/* general flags */
 	u8 remote_act_state;	/* remote partition's act_state */
@@ -693,8 +702,9 @@ struct xpc_partition_uv {
 
 /* struct xpc_partition_uv flags */
 
-#define XPC_P_HEARTBEAT_OFFLINE_UV	0x00000001
-#define XPC_P_ENGAGED_UV		0x00000002
+#define XPC_P_HEARTBEAT_OFFLINE_UV		0x00000001
+#define XPC_P_ENGAGED_UV			0x00000002
+#define XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV	0x00000004
 
 /* struct xpc_partition_uv act_state change requests */
 
@@ -803,6 +813,7 @@ extern void xpc_activate_kthreads(struct
 extern void xpc_create_kthreads(struct xpc_channel *, int, int);
 extern void xpc_disconnect_wait(int);
 extern int (*xpc_setup_partitions_sn) (void);
+extern void (*xpc_teardown_partitions_sn) (void);
 extern enum xp_retval (*xpc_get_partition_rsvd_page_pa) (void *, u64 *,
 							 unsigned long *,
 							 size_t *);
@@ -845,8 +856,8 @@ extern void (*xpc_send_chctl_openrequest
 					   unsigned long *);
 extern void (*xpc_send_chctl_openreply) (struct xpc_channel *, unsigned long *);
 
-extern void (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *,
-					    unsigned long);
+extern enum xp_retval (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *,
+						      unsigned long);
 
 extern enum xp_retval (*xpc_send_payload) (struct xpc_channel *, u32, void *,
 					   u16, u8, xpc_notify_func, void *);
Index: linux/drivers/misc/sgi-xp/xpc_channel.c
===================================================================
--- linux.orig/drivers/misc/sgi-xp/xpc_channel.c	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-xp/xpc_channel.c	2009-01-21 11:40:09.000000000 -0600
@@ -186,6 +186,7 @@ xpc_process_openclose_chctl_flags(struct
 	    &part->remote_openclose_args[ch_number];
 	struct xpc_channel *ch = &part->channels[ch_number];
 	enum xp_retval reason;
+	enum xp_retval ret;
 
 	spin_lock_irqsave(&ch->lock, irq_flags);
 
@@ -402,8 +403,13 @@ again:
 		DBUG_ON(args->local_nentries == 0);
 		DBUG_ON(args->remote_nentries == 0);
 
+		ret = xpc_save_remote_msgqueue_pa(ch, args->local_msgqueue_pa);
+		if (ret != xpSuccess) {
+			XPC_DISCONNECT_CHANNEL(ch, ret, &irq_flags);
+			spin_unlock_irqrestore(&ch->lock, irq_flags);
+			return;
+		}
 		ch->flags |= XPC_C_ROPENREPLY;
-		xpc_save_remote_msgqueue_pa(ch, args->local_msgqueue_pa);
 
 		if (args->local_nentries < ch->remote_nentries) {
 			dev_dbg(xpc_chan, "XPC_CHCTL_OPENREPLY: new "
Index: linux/drivers/misc/sgi-xp/xpc_main.c
===================================================================
--- linux.orig/drivers/misc/sgi-xp/xpc_main.c	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-xp/xpc_main.c	2009-01-21 12:20:51.000000000 -0600
@@ -171,6 +171,7 @@ static struct notifier_block xpc_die_not
 };
 
 int (*xpc_setup_partitions_sn) (void);
+void (*xpc_teardown_partitions_sn) (void);
 enum xp_retval (*xpc_get_partition_rsvd_page_pa) (void *buf, u64 *cookie,
 						  unsigned long *rp_pa,
 						  size_t *len);
@@ -217,8 +218,8 @@ void (*xpc_send_chctl_openrequest) (stru
 void (*xpc_send_chctl_openreply) (struct xpc_channel *ch,
 				  unsigned long *irq_flags);
 
-void (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *ch,
-				     unsigned long msgqueue_pa);
+enum xp_retval (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *ch,
+					       unsigned long msgqueue_pa);
 
 enum xp_retval (*xpc_send_payload) (struct xpc_channel *ch, u32 flags,
 				    void *payload, u16 payload_size,
@@ -998,6 +999,7 @@ xpc_setup_partitions(void)
 static void
 xpc_teardown_partitions(void)
 {
+	xpc_teardown_partitions_sn();
 	kfree(xpc_partitions);
 }
 
Index: linux/drivers/misc/sgi-xp/xpc_sn2.c
===================================================================
--- linux.orig/drivers/misc/sgi-xp/xpc_sn2.c	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-xp/xpc_sn2.c	2009-01-21 12:20:51.000000000 -0600
@@ -66,6 +66,12 @@ xpc_setup_partitions_sn_sn2(void)
 	return 0;
 }
 
+static void
+xpc_teardown_partitions_sn_sn2(void)
+{
+	/* nothing needs to be done */
+}
+
 /* SH_IPI_ACCESS shub register value on startup */
 static u64 xpc_sh1_IPI_access_sn2;
 static u64 xpc_sh2_IPI_access0_sn2;
@@ -436,11 +442,12 @@ xpc_send_chctl_local_msgrequest_sn2(stru
 	XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
 }
 
-static void
+static enum xp_retval
 xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
 				unsigned long msgqueue_pa)
 {
 	ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
+	return xpSuccess;
 }
 
 /*
@@ -2305,6 +2312,7 @@ xpc_init_sn2(void)
 	size_t buf_size;
 
 	xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2;
+	xpc_teardown_partitions_sn = xpc_teardown_partitions_sn_sn2;
 	xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2;
 	xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2;
 	xpc_increment_heartbeat = xpc_increment_heartbeat_sn2;
Index: linux/drivers/misc/sgi-xp/xpc_uv.c
===================================================================
--- linux.orig/drivers/misc/sgi-xp/xpc_uv.c	2009-01-21 11:40:00.000000000 -0600
+++ linux/drivers/misc/sgi-xp/xpc_uv.c	2009-01-21 12:20:51.000000000 -0600
@@ -31,6 +31,21 @@
 #include "../sgi-gru/grukservices.h"
 #include "xpc.h"
 
+#if defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
+struct uv_IO_APIC_route_entry {
+	__u64	vector		:  8,
+		delivery_mode	:  3,
+		dest_mode	:  1,
+		delivery_status	:  1,
+		polarity	:  1,
+		__reserved_1	:  1,
+		trigger		:  1,
+		mask		:  1,
+		__reserved_2	: 15,
+		dest		: 32;
+};
+#endif
+
 static atomic64_t xpc_heartbeat_uv;
 static DECLARE_BITMAP(xpc_heartbeating_to_mask_uv, XP_MAX_NPARTITIONS_UV);
 
@@ -56,26 +71,52 @@ xpc_setup_partitions_sn_uv(void)
 	for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
 		part_uv = &xpc_partitions[partid].sn.uv;
 
+		mutex_init(&part_uv->cached_activate_gru_mq_desc_mutex);
 		spin_lock_init(&part_uv->flags_lock);
 		part_uv->remote_act_state = XPC_P_AS_INACTIVE;
 	}
 	return 0;
 }
 
+static void
+xpc_teardown_partitions_sn_uv(void)
+{
+	short partid;
+	struct xpc_partition_uv *part_uv;
+	unsigned long irq_flags;
+
+	for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
+		part_uv = &xpc_partitions[partid].sn.uv;
+
+		if (part_uv->cached_activate_gru_mq_desc != NULL) {
+			mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
+			spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
+			part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
+			spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
+			kfree(part_uv->cached_activate_gru_mq_desc);
+			part_uv->cached_activate_gru_mq_desc = NULL;
+			mutex_unlock(&part_uv->
+				     cached_activate_gru_mq_desc_mutex);
+		}
+	}
+}
+
 static int
 xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name)
 {
+	int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
+
 #if defined CONFIG_X86_64
 	mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset);
 	if (mq->irq < 0) {
 		dev_err(xpc_part, "uv_setup_irq() returned error=%d\n",
-			mq->irq);
+			-mq->irq);
+		return mq->irq;
 	}
 
-#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
-	int mmr_pnode;
-	unsigned long mmr_value;
+	mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset);
 
+#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
 	if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0)
 		mq->irq = SGI_XPC_ACTIVATE;
 	else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0)
@@ -83,10 +124,8 @@ xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_
 	else
 		return -EINVAL;
 
-	mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
-	mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq;
-
-	uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value);
+	mq->mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq;
+	uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mq->mmr_value);
 #else
 	#error not a supported configuration
 #endif
@@ -127,7 +166,7 @@ xpc_gru_mq_watchlist_alloc_uv(struct xpc
 		return ret;
 	}
 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
-	ret = sn_mq_watchlist_alloc(mq->mmr_blade, uv_gpa(mq->address),
+	ret = sn_mq_watchlist_alloc(mq->mmr_blade, (void *)uv_gpa(mq->address),
 				    mq->order, &mq->mmr_offset);
 	if (ret < 0) {
 		dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n",
@@ -168,12 +207,22 @@ xpc_create_gru_mq_uv(unsigned int mq_siz
 	int pg_order;
 	struct page *page;
 	struct xpc_gru_mq_uv *mq;
+	struct uv_IO_APIC_route_entry *mmr_value;
 
 	mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL);
 	if (mq == NULL) {
 		dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
 			"a xpc_gru_mq_uv structure\n");
 		ret = -ENOMEM;
+		goto out_0;
+	}
+
+	mq->gru_mq_desc = kzalloc(sizeof(struct gru_message_queue_desc),
+				  GFP_KERNEL);
+	if (mq->gru_mq_desc == NULL) {
+		dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
+			"a gru_message_queue_desc structure\n");
+		ret = -ENOMEM;
 		goto out_1;
 	}
 
@@ -194,14 +243,6 @@ xpc_create_gru_mq_uv(unsigned int mq_siz
 	}
 	mq->address = page_address(page);
 
-	ret = gru_create_message_queue(mq->address, mq_size);
-	if (ret != 0) {
-		dev_err(xpc_part, "gru_create_message_queue() returned "
-			"error=%d\n", ret);
-		ret = -EINVAL;
-		goto out_3;
-	}
-
 	/* enable generation of irq when GRU mq operation occurs to this mq */
 	ret = xpc_gru_mq_watchlist_alloc_uv(mq);
 	if (ret != 0)
@@ -214,10 +255,20 @@ xpc_create_gru_mq_uv(unsigned int mq_siz
 	ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL);
 	if (ret != 0) {
 		dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n",
-			mq->irq, ret);
+			mq->irq, -ret);
 		goto out_5;
 	}
 
+	mmr_value = (struct uv_IO_APIC_route_entry *)&mq->mmr_value;
+	ret = gru_create_message_queue(mq->gru_mq_desc, mq->address, mq_size,
+				       nid, mmr_value->vector, mmr_value->dest);
+	if (ret != 0) {
+		dev_err(xpc_part, "gru_create_message_queue() returned "
+			"error=%d\n", ret);
+		ret = -EINVAL;
+		goto out_6;
+	}
+
 	/* allow other partitions to access this GRU mq */
 	xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size);
 	if (xp_ret != xpSuccess) {
@@ -237,8 +288,10 @@ out_4:
 out_3:
 	free_pages((unsigned long)mq->address, pg_order);
 out_2:
-	kfree(mq);
+	kfree(mq->gru_mq_desc);
 out_1:
+	kfree(mq);
+out_0:
 	return ERR_PTR(ret);
 }
 
@@ -268,13 +321,14 @@ xpc_destroy_gru_mq_uv(struct xpc_gru_mq_
 }
 
 static enum xp_retval
-xpc_send_gru_msg(unsigned long mq_gpa, void *msg, size_t msg_size)
+xpc_send_gru_msg(struct gru_message_queue_desc *gru_mq_desc, void *msg,
+		 size_t msg_size)
 {
 	enum xp_retval xp_ret;
 	int ret;
 
 	while (1) {
-		ret = gru_send_message_gpa(mq_gpa, msg, msg_size);
+		ret = gru_send_message_gpa(gru_mq_desc, msg, msg_size);
 		if (ret == MQE_OK) {
 			xp_ret = xpSuccess;
 			break;
@@ -421,7 +475,15 @@ xpc_handle_activate_mq_msg_uv(struct xpc
 		part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV;
 		part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */
 		part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies;
-		part_uv->remote_activate_mq_gpa = msg->activate_mq_gpa;
+
+		if (msg->activate_gru_mq_desc_gpa !=
+		    part_uv->activate_gru_mq_desc_gpa) {
+			spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
+			part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
+			spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
+			part_uv->activate_gru_mq_desc_gpa =
+			    msg->activate_gru_mq_desc_gpa;
+		}
 		spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
 
 		(*wakeup_hb_checker)++;
@@ -498,7 +560,7 @@ xpc_handle_activate_mq_msg_uv(struct xpc
 		args = &part->remote_openclose_args[msg->ch_number];
 		args->remote_nentries = msg->remote_nentries;
 		args->local_nentries = msg->local_nentries;
-		args->local_msgqueue_pa = msg->local_notify_mq_gpa;
+		args->local_msgqueue_pa = msg->notify_gru_mq_desc_gpa;
 
 		spin_lock_irqsave(&part->chctl_lock, irq_flags);
 		part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY;
@@ -558,9 +620,10 @@ xpc_handle_activate_IRQ_uv(int irq, void
 	short partid;
 	struct xpc_partition *part;
 	int wakeup_hb_checker = 0;
+	int part_referenced;
 
 	while (1) {
-		msg_hdr = gru_get_next_message(xpc_activate_mq_uv->address);
+		msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc);
 		if (msg_hdr == NULL)
 			break;
 
@@ -571,14 +634,15 @@ xpc_handle_activate_IRQ_uv(int irq, void
 				partid);
 		} else {
 			part = &xpc_partitions[partid];
-			if (xpc_part_ref(part)) {
-				xpc_handle_activate_mq_msg_uv(part, msg_hdr,
-							    &wakeup_hb_checker);
+
+			part_referenced = xpc_part_ref(part);
+			xpc_handle_activate_mq_msg_uv(part, msg_hdr,
+						      &wakeup_hb_checker);
+			if (part_referenced)
 				xpc_part_deref(part);
-			}
 		}
 
-		gru_free_message(xpc_activate_mq_uv->address, msg_hdr);
+		gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr);
 	}
 
 	if (wakeup_hb_checker)
@@ -588,21 +652,73 @@ xpc_handle_activate_IRQ_uv(int irq, void
 }
 
 static enum xp_retval
+xpc_cache_remote_gru_mq_desc_uv(struct gru_message_queue_desc *gru_mq_desc,
+				unsigned long gru_mq_desc_gpa)
+{
+	enum xp_retval ret;
+
+	ret = xp_remote_memcpy(uv_gpa(gru_mq_desc), gru_mq_desc_gpa,
+			       sizeof(struct gru_message_queue_desc));
+	if (ret == xpSuccess)
+		gru_mq_desc->mq = NULL;
+
+	return ret;
+}
+
+static enum xp_retval
 xpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size,
 			 int msg_type)
 {
 	struct xpc_activate_mq_msghdr_uv *msg_hdr = msg;
+	struct xpc_partition_uv *part_uv = &part->sn.uv;
+	struct gru_message_queue_desc *gru_mq_desc;
+	unsigned long irq_flags;
+	enum xp_retval ret;
 
 	DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV);
 
 	msg_hdr->type = msg_type;
-	msg_hdr->partid = XPC_PARTID(part);
+	msg_hdr->partid = xp_partition_id;
 	msg_hdr->act_state = part->act_state;
 	msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies;
 
+	mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
+again:
+	if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) {
+		gru_mq_desc = part_uv->cached_activate_gru_mq_desc;
+		if (gru_mq_desc == NULL) {
+			gru_mq_desc = kmalloc(sizeof(struct
+					      gru_message_queue_desc),
+					      GFP_KERNEL);
+			if (gru_mq_desc == NULL) {
+				ret = xpNoMemory;
+				goto done;
+			}
+			part_uv->cached_activate_gru_mq_desc = gru_mq_desc;
+		}
+
+		ret = xpc_cache_remote_gru_mq_desc_uv(gru_mq_desc,
+						      part_uv->
+						      activate_gru_mq_desc_gpa);
+		if (ret != xpSuccess)
+			goto done;
+
+		spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
+		part_uv->flags |= XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
+		spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
+	}
+
 	/* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */
-	return xpc_send_gru_msg(part->sn.uv.remote_activate_mq_gpa, msg,
-				msg_size);
+	ret = xpc_send_gru_msg(part_uv->cached_activate_gru_mq_desc, msg,
+			       msg_size);
+	if (ret != xpSuccess) {
+		smp_rmb();	/* ensure a fresh copy of part_uv->flags */
+		if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV))
+			goto again;
+	}
+done:
+	mutex_unlock(&part_uv->cached_activate_gru_mq_desc_mutex);
+	return ret;
 }
 
 static void
@@ -620,7 +736,7 @@ static void
 xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags,
 			 void *msg, size_t msg_size, int msg_type)
 {
-	struct xpc_partition *part = &xpc_partitions[ch->number];
+	struct xpc_partition *part = &xpc_partitions[ch->partid];
 	enum xp_retval ret;
 
 	ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
@@ -692,7 +808,8 @@ xpc_get_partition_rsvd_page_pa_uv(void *
 static int
 xpc_setup_rsvd_page_sn_uv(struct xpc_rsvd_page *rp)
 {
-	rp->sn.activate_mq_gpa = uv_gpa(xpc_activate_mq_uv->address);
+	rp->sn.activate_gru_mq_desc_gpa =
+	    uv_gpa(xpc_activate_mq_uv->gru_mq_desc);
 	return 0;
 }
 
@@ -787,7 +904,8 @@ xpc_request_partition_activation_uv(stru
 
 	part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */
 	part->remote_rp_ts_jiffies = remote_rp->ts_jiffies;
-	part->sn.uv.remote_activate_mq_gpa = remote_rp->sn.activate_mq_gpa;
+	part->sn.uv.activate_gru_mq_desc_gpa =
+	    remote_rp->sn.activate_gru_mq_desc_gpa;
 
 	/*
 	 * ??? Is it a good idea to make this conditional on what is
@@ -795,7 +913,8 @@ xpc_request_partition_activation_uv(stru
 	 */
 	if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) {
 		msg.rp_gpa = uv_gpa(xpc_rsvd_page);
-		msg.activate_mq_gpa = xpc_rsvd_page->sn.activate_mq_gpa;
+		msg.activate_gru_mq_desc_gpa =
+		    xpc_rsvd_page->sn.activate_gru_mq_desc_gpa;
 		xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
 					   XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV);
 	}
@@ -857,7 +976,8 @@ xpc_get_fifo_entry_uv(struct xpc_fifo_he
 		if (head->first == NULL)
 			head->last = NULL;
 	}
-	head->n_entries++;
+	head->n_entries--;
+	BUG_ON(head->n_entries < 0);
 	spin_unlock_irqrestore(&head->lock, irq_flags);
 	first->next = NULL;
 	return first;
@@ -876,8 +996,7 @@ xpc_put_fifo_entry_uv(struct xpc_fifo_he
 	else
 		head->first = last;
 	head->last = last;
-	head->n_entries--;
-	BUG_ON(head->n_entries < 0);
+	head->n_entries++;
 	spin_unlock_irqrestore(&head->lock, irq_flags);
 }
 
@@ -1037,6 +1156,12 @@ xpc_setup_msg_structures_uv(struct xpc_c
 
 	DBUG_ON(ch->flags & XPC_C_SETUP);
 
+	ch_uv->cached_notify_gru_mq_desc = kmalloc(sizeof(struct
+						   gru_message_queue_desc),
+						   GFP_KERNEL);
+	if (ch_uv->cached_notify_gru_mq_desc == NULL)
+		return xpNoMemory;
+
 	ret = xpc_allocate_send_msg_slot_uv(ch);
 	if (ret == xpSuccess) {
 
@@ -1060,7 +1185,8 @@ xpc_teardown_msg_structures_uv(struct xp
 
 	DBUG_ON(!spin_is_locked(&ch->lock));
 
-	ch_uv->remote_notify_mq_gpa = 0;
+	kfree(ch_uv->cached_notify_gru_mq_desc);
+	ch_uv->cached_notify_gru_mq_desc = NULL;
 
 	if (ch->flags & XPC_C_SETUP) {
 		xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
@@ -1111,7 +1237,7 @@ xpc_send_chctl_openreply_uv(struct xpc_c
 	msg.ch_number = ch->number;
 	msg.local_nentries = ch->local_nentries;
 	msg.remote_nentries = ch->remote_nentries;
-	msg.local_notify_mq_gpa = uv_gpa(xpc_notify_mq_uv);
+	msg.notify_gru_mq_desc_gpa = uv_gpa(xpc_notify_mq_uv->gru_mq_desc);
 	xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
 				    XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV);
 }
@@ -1128,11 +1254,15 @@ xpc_send_chctl_local_msgrequest_uv(struc
 	xpc_wakeup_channel_mgr(part);
 }
 
-static void
+static enum xp_retval
 xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch,
-			       unsigned long msgqueue_pa)
+			       unsigned long gru_mq_desc_gpa)
 {
-	ch->sn.uv.remote_notify_mq_gpa = msgqueue_pa;
+	struct xpc_channel_uv *ch_uv = &ch->sn.uv;
+
+	DBUG_ON(ch_uv->cached_notify_gru_mq_desc == NULL);
+	return xpc_cache_remote_gru_mq_desc_uv(ch_uv->cached_notify_gru_mq_desc,
+					       gru_mq_desc_gpa);
 }
 
 static void
@@ -1340,7 +1470,8 @@ xpc_handle_notify_IRQ_uv(int irq, void *
 	short partid;
 	struct xpc_partition *part;
 
-	while ((msg = gru_get_next_message(xpc_notify_mq_uv)) != NULL) {
+	while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) !=
+	       NULL) {
 
 		partid = msg->hdr.partid;
 		if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
@@ -1355,7 +1486,7 @@ xpc_handle_notify_IRQ_uv(int irq, void *
 			}
 		}
 
-		gru_free_message(xpc_notify_mq_uv, msg);
+		gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg);
 	}
 
 	return IRQ_HANDLED;
@@ -1439,7 +1570,8 @@ xpc_send_payload_uv(struct xpc_channel *
 	msg->hdr.msg_slot_number = msg_slot->msg_slot_number;
 	memcpy(&msg->payload, payload, payload_size);
 
-	ret = xpc_send_gru_msg(ch->sn.uv.remote_notify_mq_gpa, msg, msg_size);
+	ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
+			       msg_size);
 	if (ret == xpSuccess)
 		goto out_1;
 
@@ -1530,7 +1662,7 @@ xpc_received_payload_uv(struct xpc_chann
 	msg->hdr.partid = xp_partition_id;
 	msg->hdr.size = 0;	/* size of zero indicates this is an ACK */
 
-	ret = xpc_send_gru_msg(ch->sn.uv.remote_notify_mq_gpa, msg,
+	ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
 			       sizeof(struct xpc_notify_mq_msghdr_uv));
 	if (ret != xpSuccess)
 		XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
@@ -1542,6 +1674,7 @@ int
 xpc_init_uv(void)
 {
 	xpc_setup_partitions_sn = xpc_setup_partitions_sn_uv;
+	xpc_teardown_partitions_sn = xpc_teardown_partitions_sn_uv;
 	xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv;
 	xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv;
 	xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_uv;

[Patch 12/14] macro for scanning all gru chiplets

[steiner]

[-- Attachment #1: uv_gru_max_gid --]
[-- Type: text/plain, Size: 4641 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add macro for scanning all active GRU chiplets. Maximum
chiplet id is saved during GRU initialization.

Signed-off-by: Jack Steiner <steiner@sgi.com>
---
 drivers/misc/sgi-gru/grufault.c  |   11 ++---------
 drivers/misc/sgi-gru/grufile.c   |    3 +++
 drivers/misc/sgi-gru/gruprocfs.c |    4 ++--
 drivers/misc/sgi-gru/grutables.h |    9 +++++++--
 4 files changed, 14 insertions(+), 13 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:12.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufault.c	2009-01-15 08:18:13.000000000 -0600
@@ -600,18 +600,11 @@ static int gru_unload_all_contexts(void)
 {
 	struct gru_thread_state *gts;
 	struct gru_state *gru;
-	int maxgid, gid, ctxnum;
-	int nodesperblade;
+	int gid, ctxnum;
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
-	if (num_online_nodes() > 1 &&
-			(uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
-		nodesperblade = 2;
-	else
-		nodesperblade = 1;
-	maxgid = GRU_CHIPLETS_PER_BLADE * num_online_nodes() / nodesperblade;
-	for (gid = 0; gid < maxgid; gid++) {
+	foreach_gid(gid) {
 		gru = GID_TO_GRU(gid);
 		spin_lock(&gru->gs_lock);
 		for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2009-01-15 08:18:11.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufile.c	2009-01-15 08:18:13.000000000 -0600
@@ -59,6 +59,7 @@
 struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly;
 unsigned long gru_start_paddr __read_mostly;
 unsigned long gru_end_paddr __read_mostly;
+unsigned int gru_max_gids __read_mostly;
 struct gru_stats_s gru_stats;
 
 /* Guaranteed user available resources on each node */
@@ -288,6 +289,8 @@ static void gru_init_chiplet(struct gru_
 	gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
 	gru->gs_asid_limit = MAX_ASID;
 	gru_tgh_flush_init(gru);
+	if (gru->gs_gid >= gru_max_gids)
+		gru_max_gids = gru->gs_gid + 1;
 	gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n",
 		bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr,
 		gru->gs_gru_base_paddr);
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2009-01-15 08:18:09.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2009-01-15 08:18:13.000000000 -0600
@@ -226,7 +226,7 @@ static void seq_stop(struct seq_file *fi
 
 static void *seq_start(struct seq_file *file, loff_t *gid)
 {
-	if (*gid < GRU_MAX_GRUS)
+	if (*gid < gru_max_gids)
 		return gid;
 	return NULL;
 }
@@ -234,7 +234,7 @@ static void *seq_start(struct seq_file *
 static void *seq_next(struct seq_file *file, void *data, loff_t *gid)
 {
 	(*gid)++;
-	if (*gid < GRU_MAX_GRUS)
+	if (*gid < gru_max_gids)
 		return gid;
 	return NULL;
 }
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:09.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-15 08:18:13.000000000 -0600
@@ -153,6 +153,7 @@
 extern struct gru_stats_s gru_stats;
 extern struct gru_blade_state *gru_base[];
 extern unsigned long gru_start_paddr, gru_end_paddr;
+extern unsigned int gru_max_gids;
 
 #define GRU_MAX_BLADES		MAX_NUMNODES
 #define GRU_MAX_GRUS		(GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE)
@@ -406,12 +407,12 @@ struct gru_state {
 							   gru segments (64) */
 	void			*gs_gru_base_vaddr;	/* Virtual address of
 							   gru segments (64) */
-	unsigned char		gs_gid;			/* unique GRU number */
+	unsigned short		gs_gid;			/* unique GRU number */
+	unsigned short		gs_blade_id;		/* blade of GRU */
 	unsigned char		gs_tgh_local_shift;	/* used to pick TGH for
 							   local flush */
 	unsigned char		gs_tgh_first_remote;	/* starting TGH# for
 							   remote flush */
-	unsigned short		gs_blade_id;		/* blade of GRU */
 	spinlock_t		gs_asid_lock;		/* lock used for
 							   assigning asids */
 	spinlock_t		gs_lock;		/* lock used for
@@ -506,6 +507,10 @@ struct gru_blade_state {
 			(i) < GRU_CHIPLETS_PER_BLADE;			\
 			(i)++, (gru)++)
 
+/* Scan all GRUs */
+#define foreach_gid(gid)						\
+	for ((gid) = 0; (gid) < gru_max_gids; (gid)++)
+
 /* Scan all active GTSs on a gru. Note: must hold ss_lock to use this macro. */
 #define for_each_gts_on_gru(gts, gru, ctxnum)				\
 	for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++)		\

[Patch 13/14] Fix bugs related to module unload of the GRU driver

[steiner]

[-- Attachment #1: uv_gru_module_unload --]
[-- Type: text/plain, Size: 2600 bytes --]

From: Jack Steiner <steiner@sgi.com>

Fix bugs related to module unload of the GRU driver.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufile.c      |    5 ++++-
 drivers/misc/sgi-gru/grukservices.c |   19 +++++++++++++++++++
 drivers/misc/sgi-gru/grutables.h    |    1 +
 3 files changed, 24 insertions(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2009-01-21 12:53:18.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufile.c	2009-01-21 12:53:19.000000000 -0600
@@ -452,7 +452,7 @@ exit1:
 
 static void __exit gru_exit(void)
 {
-	int i, bid;
+	int i, bid, gid;
 	int order = get_order(sizeof(struct gru_state) *
 			      GRU_CHIPLETS_PER_BLADE);
 
@@ -462,6 +462,9 @@ static void __exit gru_exit(void)
 	for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++)
 		free_irq(IRQ_GRU + i, NULL);
 
+	foreach_gid(gid)
+		gru_kservices_exit(GID_TO_GRU(gid));
+
 	for (bid = 0; bid < GRU_MAX_BLADES; bid++)
 		free_pages((unsigned long)gru_base[bid], order);
 
Index: linux/drivers/misc/sgi-gru/grukservices.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.c	2009-01-21 12:53:17.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grukservices.c	2009-01-21 12:53:24.000000000 -0600
@@ -690,3 +690,22 @@ int gru_kservices_init(struct gru_state 
 		quicktest(gru);
 	return 0;
 }
+
+void gru_kservices_exit(struct gru_state *gru)
+{
+	struct gru_context_configuration_handle *cch;
+	struct gru_blade_state *bs;
+
+	bs = gru->gs_blade;
+	if (gru != &bs->bs_grus[1])
+		return;
+
+	cch = get_cch(gru->gs_gru_base_vaddr, KERNEL_CTXNUM);
+	lock_cch_handle(cch);
+	if (cch_interrupt_sync(cch))
+		BUG();
+	if (cch_deallocate(cch))
+		BUG();
+	unlock_cch_handle(cch);
+}
+
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-21 12:53:18.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-21 12:53:24.000000000 -0600
@@ -600,6 +600,7 @@ extern void gru_unload_context(struct gr
 extern void gts_drop(struct gru_thread_state *gts);
 extern void gru_tgh_flush_init(struct gru_state *gru);
 extern int gru_kservices_init(struct gru_state *gru);
+extern void gru_kservices_exit(struct gru_state *gru);
 extern irqreturn_t gru_intr(int irq, void *dev_id);
 extern int gru_handle_user_call_os(unsigned long address);
 extern int gru_user_flush_tlb(unsigned long arg);

[Patch 14/14] Support multiple pagesizes in GRU

[steiner]

[-- Attachment #1: uv_gru_large_pages --]
[-- Type: text/plain, Size: 8131 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add multiple pagesize support to the GRU driver.

Signed-off-by: Jack Steiner <steiner@sgi.com>
---
 drivers/misc/sgi-gru/grufault.c     |   15 +++++++++++++++
 drivers/misc/sgi-gru/gruhandles.c   |   34 ++++------------------------------
 drivers/misc/sgi-gru/gruhandles.h   |    2 +-
 drivers/misc/sgi-gru/grukservices.c |    2 +-
 drivers/misc/sgi-gru/grumain.c      |   17 +++++++++++------
 drivers/misc/sgi-gru/grutables.h    |    3 +++
 6 files changed, 35 insertions(+), 38 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2009-01-22 11:33:08.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grufault.c	2009-01-22 11:33:56.000000000 -0600
@@ -360,6 +360,13 @@ static int gru_try_dropin(struct gru_thr
 	if (ret == -2)
 		goto failupm;
 
+	if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) {
+		gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift);
+		if (atomic || !gru_update_cch(gts, 0)) {
+			gts->ts_force_cch_reload = 1;
+			goto failupm;
+		}
+	}
 	gru_cb_set_istatus_active(cb);
 	tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
 			  GRU_PAGESIZE(pageshift));
@@ -535,6 +542,14 @@ int gru_handle_user_call_os(unsigned lon
 		gts->ts_force_unload = 1;
 	}
 
+	/*
+	 * CCH may contain stale data if ts_force_cch_reload is set.
+	 */
+	if (gts->ts_gru && gts->ts_force_cch_reload) {
+		gru_update_cch(gts, 0);
+		gts->ts_force_cch_reload = 0;
+	}
+
 	ret = -EAGAIN;
 	cbrnum = thread_cbr_number(gts, ucbnum);
 	if (gts->ts_force_unload) {
Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.c	2009-01-22 11:33:03.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruhandles.c	2009-01-22 11:39:01.000000000 -0600
@@ -72,42 +72,16 @@ static int wait_instruction_complete(voi
 	return status;
 }
 
-#if defined CONFIG_IA64
-static void cch_allocate_set_asids(
-		  struct gru_context_configuration_handle *cch, int asidval)
+int cch_allocate(struct gru_context_configuration_handle *cch,
+		int asidval, int sizeavail, unsigned long cbrmap,
+		unsigned long dsrmap)
 {
 	int i;
 
 	for (i = 0; i < 8; i++) {
 		cch->asid[i] = (asidval++);
-#if 0
-		/* ZZZ hugepages not supported yet */
-		if (i == RGN_HPAGE)
-			cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift);
-		else
-#endif
-			cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT);
+		cch->sizeavail[i] = sizeavail;
 	}
-}
-#elif defined CONFIG_X86_64
-static void cch_allocate_set_asids(
-		  struct gru_context_configuration_handle *cch, int asidval)
-{
-	int i;
-
-	for (i = 0; i < 8; i++) {
-		cch->asid[i] = asidval++;
-		cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) |
-			GRU_SIZEAVAIL(21);
-	}
-}
-#endif
-
-int cch_allocate(struct gru_context_configuration_handle *cch,
-			       int asidval, unsigned long cbrmap,
-			       unsigned long dsrmap)
-{
-	cch_allocate_set_asids(cch, asidval);
 	cch->dsr_allocation_map = dsrmap;
 	cch->cbr_allocation_map = cbrmap;
 	cch->opc = CCHOP_ALLOCATE;
Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.h	2009-01-22 11:33:02.000000000 -0600
+++ linux/drivers/misc/sgi-gru/gruhandles.h	2009-01-22 11:33:56.000000000 -0600
@@ -496,7 +496,7 @@ enum gru_cbr_state {
 #define GRUMAXINVAL		1024UL
 
 int cch_allocate(struct gru_context_configuration_handle *cch,
-       int asidval, unsigned long cbrmap, unsigned long dsrmap);
+       int asidval, int sizeavail, unsigned long cbrmap, unsigned long dsrmap);
 
 int cch_start(struct gru_context_configuration_handle *cch);
 int cch_interrupt(struct gru_context_configuration_handle *cch);
Index: linux/drivers/misc/sgi-gru/grukservices.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.c	2009-01-22 11:33:09.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grukservices.c	2009-01-22 11:33:56.000000000 -0600
@@ -672,7 +672,7 @@ int gru_kservices_init(struct gru_state 
 	cch->tlb_int_enable = 0;
 	cch->tfm_done_bit_enable = 0;
 	cch->unmap_enable = 1;
-	err = cch_allocate(cch, 0, cbr_map, dsr_map);
+	err = cch_allocate(cch, 0, 0, cbr_map, dsr_map);
 	if (err) {
 		gru_dbg(grudev,
 			"Unable to allocate kernel CCH: gid %d, err %d\n",
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2009-01-22 11:33:05.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grumain.c	2009-01-22 11:33:56.000000000 -0600
@@ -326,6 +326,7 @@ static struct gru_thread_state *gru_allo
 	gts->ts_vma = vma;
 	gts->ts_tlb_int_select = -1;
 	gts->ts_gms = gru_register_mmu_notifier();
+	gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT);
 	if (!gts->ts_gms)
 		goto err;
 
@@ -552,7 +553,8 @@ static void gru_load_context(struct gru_
 		cch->tlb_int_select = gts->ts_tlb_int_select;
 	}
 	cch->tfm_done_bit_enable = 0;
-	err = cch_allocate(cch, asid, gts->ts_cbr_map, gts->ts_dsr_map);
+	err = cch_allocate(cch, asid, gts->ts_sizeavail, gts->ts_cbr_map,
+				gts->ts_dsr_map);
 	if (err) {
 		gru_dbg(grudev,
 			"err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
@@ -573,11 +575,12 @@ static void gru_load_context(struct gru_
 /*
  * Update fields in an active CCH:
  * 	- retarget interrupts on local blade
+ * 	- update sizeavail mask
  * 	- force a delayed context unload by clearing the CCH asids. This
  * 	  forces TLB misses for new GRU instructions. The context is unloaded
  * 	  when the next TLB miss occurs.
  */
-static int gru_update_cch(struct gru_thread_state *gts, int int_select)
+int gru_update_cch(struct gru_thread_state *gts, int force_unload)
 {
 	struct gru_context_configuration_handle *cch;
 	struct gru_state *gru = gts->ts_gru;
@@ -591,9 +594,11 @@ static int gru_update_cch(struct gru_thr
 			goto exit;
 		if (cch_interrupt(cch))
 			BUG();
-		if (int_select >= 0) {
-			gts->ts_tlb_int_select = int_select;
-			cch->tlb_int_select = int_select;
+		if (!force_unload) {
+			for (i = 0; i < 8; i++)
+				cch->sizeavail[i] = gts->ts_sizeavail;
+			gts->ts_tlb_int_select = gru_cpu_fault_map_id();
+			cch->tlb_int_select = gru_cpu_fault_map_id();
 		} else {
 			for (i = 0; i < 8; i++)
 				cch->asid[i] = 0;
@@ -625,7 +630,7 @@ static int gru_retarget_intr(struct gru_
 
 	gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select,
 		gru_cpu_fault_map_id());
-	return gru_update_cch(gts, gru_cpu_fault_map_id());
+	return gru_update_cch(gts, 0);
 }
 
 
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2009-01-22 11:33:09.000000000 -0600
+++ linux/drivers/misc/sgi-gru/grutables.h	2009-01-22 11:33:56.000000000 -0600
@@ -361,6 +361,7 @@ struct gru_thread_state {
 	long			ts_user_options;/* misc user option flags */
 	pid_t			ts_tgid_owner;	/* task that is using the
 						   context - for migration */
+	unsigned short		ts_sizeavail;	/* Pagesizes in use */
 	int			ts_tsid;	/* thread that owns the
 						   structure */
 	int			ts_tlb_int_select;/* target cpu if interrupts
@@ -374,6 +375,7 @@ struct gru_thread_state {
 						   required for contest */
 	char			ts_blade;	/* If >= 0, migrate context if
 						   ref from diferent blade */
+	char			ts_force_cch_reload;
 	char			ts_force_unload;/* force context to be unloaded
 						   after migration */
 	char			ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each
@@ -597,6 +599,7 @@ extern struct gru_thread_state *gru_find
 extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct
 				*vma, int tsid);
 extern void gru_unload_context(struct gru_thread_state *gts, int savestate);
+extern int gru_update_cch(struct gru_thread_state *gts, int force_unload);
 extern void gts_drop(struct gru_thread_state *gts);
 extern void gru_tgh_flush_init(struct gru_state *gru);
 extern int gru_kservices_init(struct gru_state *gru);