[PATCH] move cnodeid_to_nasid_table out of pda

[PATCH] move cnodeid_to_nasid_table out of pda

[Dean Nelson]

Another step in the effort to eliminate the SN pda structure.
This patch moves the cnodeid_to_nasid_table field out of the pda,
making it a standalone per-cpu data item, and exports it so it can
be accessed by kernel modules.

Signed-off-by: Dean Nelson <dcn@sgi.com>


Index: linux-2.6/include/asm-ia64/sn/pda.h
=================================--- linux-2.6.orig/include/asm-ia64/sn/pda.h	2005-03-21 09:33:28.657196704 -0600
+++ linux-2.6/include/asm-ia64/sn/pda.h	2005-03-21 09:33:28.783171960 -0600
@@ -49,7 +49,6 @@
 
 	unsigned long	sn_soft_irr[4];
 	unsigned long	sn_in_service_ivecs[4];
-	short		cnodeid_to_nasid_table[MAX_NUMNODES];
 	int		sn_lb_int_war_ticks;
 	int		sn_last_irq;
 	int		sn_first_irq;
Index: linux-2.6/include/asm-ia64/sn/sn_cpuid.h
=================================--- linux-2.6.orig/include/asm-ia64/sn/sn_cpuid.h	2005-03-21 09:33:28.503877981 -0600
+++ linux-2.6/include/asm-ia64/sn/sn_cpuid.h	2005-03-21 10:06:22.752760405 -0600
@@ -4,7 +4,7 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * Copyright (C) 2000-2004 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 2000-2005 Silicon Graphics, Inc. All rights reserved.
  */
 
 
@@ -123,11 +123,8 @@
 
 /*
  * cnodeid_to_nasid - convert a cnodeid to a NASID
- *	Macro relies on pg_data for a node being on the node itself.
- *	Just extract the NASID from the pointer.
- *
  */
-#define cnodeid_to_nasid(cnodeid)	pda->cnodeid_to_nasid_table[cnodeid]
+#define cnodeid_to_nasid(cnodeid)	(sn_cnodeid_to_nasid[cnodeid])
  
 /*
  * nasid_to_cnodeid - convert a NASID to a cnodeid
Index: linux-2.6/include/asm-ia64/sn/arch.h
=================================--- linux-2.6.orig/include/asm-ia64/sn/arch.h	2005-03-21 09:33:28.540986971 -0600
+++ linux-2.6/include/asm-ia64/sn/arch.h	2005-03-21 10:55:47.249275028 -0600
@@ -5,7 +5,7 @@
  *
  * SGI specific setup.
  *
- * Copyright (C) 1995-1997,1999,2001-2004 Silicon Graphics, Inc.  All rights reserved.
+ * Copyright (C) 1995-1997,1999,2001-2005 Silicon Graphics, Inc.  All rights reserved.
  * Copyright (C) 1999 Ralf Baechle (ralf@gnu.org)
  */
 #ifndef _ASM_IA64_SN_ARCH_H
@@ -47,6 +47,15 @@
 #define MAX_COMPACT_NODES	2048
 #define CPUS_PER_NODE		4
 
+
+/*
+ * Compact node ID to nasid mappings kept in the per-cpu data areas of each
+ * cpu.
+ */
+DECLARE_PER_CPU(short, __sn_cnodeid_to_nasid[MAX_NUMNODES]);
+#define sn_cnodeid_to_nasid	(&__get_cpu_var(__sn_cnodeid_to_nasid[0]))
+
+
 extern void sn_flush_all_caches(long addr, long bytes);
 
 #endif /* _ASM_IA64_SN_ARCH_H */
Index: linux-2.6/arch/ia64/sn/kernel/setup.c
=================================--- linux-2.6.orig/arch/ia64/sn/kernel/setup.c	2005-03-21 09:33:28.548799390 -0600
+++ linux-2.6/arch/ia64/sn/kernel/setup.c	2005-03-21 11:02:47.545462823 -0600
@@ -3,7 +3,7 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * Copyright (C) 1999,2001-2004 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 1999,2001-2005 Silicon Graphics, Inc. All rights reserved.
  */
 
 #include <linux/config.h>
@@ -72,6 +72,9 @@
 DEFINE_PER_CPU(struct sn_hub_info_s, __sn_hub_info);
 EXPORT_PER_CPU_SYMBOL(__sn_hub_info);
 
+DEFINE_PER_CPU(short, __sn_cnodeid_to_nasid[MAX_NUMNODES]);
+EXPORT_PER_CPU_SYMBOL(__sn_cnodeid_to_nasid);
+
 partid_t sn_partid = -1;
 EXPORT_SYMBOL(sn_partid);
 char sn_system_serial_number_string[128];
@@ -364,11 +367,11 @@
 {
 	cnodeid_t cnode;
 
-	memset(pda->cnodeid_to_nasid_table, -1,
-	       sizeof(pda->cnodeid_to_nasid_table));
+	memset(sn_cnodeid_to_nasid, -1,
+			sizeof(__ia64_per_cpu_var(__sn_cnodeid_to_nasid)));
 	for_each_online_node(cnode)
-		pda->cnodeid_to_nasid_table[cnode] -		    pxm_to_nasid(nid_to_pxm_map[cnode]);
+		sn_cnodeid_to_nasid[cnode] +				pxm_to_nasid(nid_to_pxm_map[cnode]);
 
 	numionodes = num_online_nodes();
 	scan_for_ionodes();
@@ -477,15 +480,18 @@
 	pda->idle_flag = 0;
 
 	if (cpuid != 0) {
-		memcpy(pda->cnodeid_to_nasid_table,
-		       pdacpu(0)->cnodeid_to_nasid_table,
-		       sizeof(pda->cnodeid_to_nasid_table));
+		/* copy cpu 0's sn_cnodeid_to_nasid table to this cpu's */
+		memcpy(sn_cnodeid_to_nasid,
+		       (&per_cpu(__sn_cnodeid_to_nasid, 0)),
+		       sizeof(__ia64_per_cpu_var(__sn_cnodeid_to_nasid)));
 	}
 
 	/*
 	 * Check for WARs.
 	 * Only needs to be done once, on BSP.
-	 * Has to be done after loop above, because it uses pda.cnodeid_to_nasid_table[i].
+	 * Has to be done after loop above, because it uses this cpu's
+	 * sn_cnodeid_to_nasid table which was just initialized if this
+	 * isn't cpu 0.
 	 * Has to be done before assignment below.
 	 */
 	if (!wars_have_been_checked) {
@@ -571,8 +577,7 @@
 		brd = find_lboard_any(brd, KLTYPE_SNIA);
 
 		while (brd) {
-			pda->cnodeid_to_nasid_table[numionodes] -			    brd->brd_nasid;
+			sn_cnodeid_to_nasid[numionodes] = brd->brd_nasid;
 			physical_node_map[brd->brd_nasid] = numionodes;
 			root_lboard[numionodes] = brd;
 			numionodes++;
@@ -593,8 +598,7 @@
 				      root_lboard[nasid_to_cnodeid(nasid)],
 				      KLTYPE_TIO);
 		while (brd) {
-			pda->cnodeid_to_nasid_table[numionodes] -			    brd->brd_nasid;
+			sn_cnodeid_to_nasid[numionodes] = brd->brd_nasid;
 			physical_node_map[brd->brd_nasid] = numionodes;
 			root_lboard[numionodes] = brd;
 			numionodes++;
@@ -605,7 +609,6 @@
 			brd = find_lboard_any(brd, KLTYPE_TIO);
 		}
 	}
-
 }
 
 int

Re: [PATCH] move cnodeid_to_nasid_table out of pda

[Christoph Lameter]

On Mon, 21 Mar 2005, Dean Nelson wrote:

> Another step in the effort to eliminate the SN pda structure.
> This patch moves the cnodeid_to_nasid_table field out of the pda,
> making it a standalone per-cpu data item, and exports it so it can
> be accessed by kernel modules.

I have verified that Dean's patch also fixes the problem that the mmtimer
driver (clock driver for SN2 RTC) cannot be build as a module.

Re: [PATCH] move cnodeid_to_nasid_table out of pda

[Christoph Lameter]

> On Mon, 21 Mar 2005, Dean Nelson wrote:
>
> > Another step in the effort to eliminate the SN pda structure.
> > This patch moves the cnodeid_to_nasid_table field out of the pda,
> > making it a standalone per-cpu data item, and exports it so it can
> > be accessed by kernel modules.

Yuck something went wrong with the test. Turns out that I need a lot more
definitions to make mmtimer a module since it uses the SHUB access macros.
The following patch applies on top of Dean's patch:

Make key components of the pda_s structure cpu local and export them so that
the low level access macros to the SN2 SHub work. This is necessary to make
mmtimer work as a module.

Also removes the unused p_subnodepda in pda_s.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.11/include/asm-ia64/sn/pda.h
=================================--- linux-2.6.11.orig/include/asm-ia64/sn/pda.h	2005-03-21 18:54:47.000000000 -0800
+++ linux-2.6.11/include/asm-ia64/sn/pda.h	2005-03-21 20:31:19.000000000 -0800
@@ -25,23 +25,10 @@

 typedef struct pda_s {

-	/* Having a pointer in the begining of PDA tends to increase
-	 * the chance of having this pointer in cache. (Yes something
-	 * else gets pushed out). Doing this reduces the number of memory
-	 * access to all nodepda variables to be one
-	 */
-	struct nodepda_s *p_nodepda;		/* Pointer to Per node PDA */
-	struct subnodepda_s *p_subnodepda;	/* Pointer to CPU  subnode PDA */
-
 	/*
 	 * Support for SN LEDs
 	 */
 	volatile short	*led_address;
-	u16		nasid_bitmask;
-	u8		shub2;
-	u8		nasid_shift;
-	u8		as_shift;
-	u8		shub_1_1_found;
 	u8		led_state;
 	u8		hb_state;	/* supports blinking heartbeat leds */
 	unsigned int	hb_count;
@@ -83,12 +70,17 @@ DECLARE_PER_CPU(struct pda_s, pda_percpu

 #define pdacpu(cpu)	(&per_cpu(pda_percpu, cpu))

+DECLARE_PER_CPU(u64, __sn_shub2);
+#define sn_shub2	(__get_cpu_var(__sn_shub2))
+DECLARE_PER_CPU(u64, __sn_shub_1_1_found);
+#define sn_shub_1_1_found	(__get_cpu_var(__sn_shub_1_1_found))
+
 /*
  * Use this macro to test if shub 1.1 wars should be enabled
  */
-#define enable_shub_wars_1_1()	(pda->shub_1_1_found)
+#define enable_shub_wars_1_1()	(sn_shub_1_1_found)

-#define is_shub2()	(pda->shub2)
-#define is_shub1()	(pda->shub2 = 0)
+#define is_shub2()	(sn_shub2)
+#define is_shub1()	(sn_shub2 = 0)

 #endif /* _ASM_IA64_SN_PDA_H */
Index: linux-2.6.11/include/asm-ia64/sn/arch.h
=================================--- linux-2.6.11.orig/include/asm-ia64/sn/arch.h	2005-03-21 18:54:47.000000000 -0800
+++ linux-2.6.11/include/asm-ia64/sn/arch.h	2005-03-21 20:01:47.000000000 -0800
@@ -30,7 +30,12 @@
  */
 DECLARE_PER_CPU(short, __sn_cnodeid_to_nasid[MAX_NUMNODES]);
 #define sn_cnodeid_to_nasid	(&__get_cpu_var(__sn_cnodeid_to_nasid[0]))
-
+DECLARE_PER_CPU(u64, __sn_nasid_bitmask);
+#define sn_nasid_bitmask	(__get_cpu_var(__sn_nasid_bitmask))
+DECLARE_PER_CPU(u64, __sn_nasid_shift);
+#define sn_nasid_shift	(__get_cpu_var(__sn_nasid_shift))
+DECLARE_PER_CPU(u64, __sn_as_shift);
+#define sn_as_shift	(__get_cpu_var(__sn_as_shift))

 extern void sn_flush_all_caches(long addr, long bytes);

Index: linux-2.6.11/arch/ia64/sn/kernel/setup.c
=================================--- linux-2.6.11.orig/arch/ia64/sn/kernel/setup.c	2005-03-21 18:54:47.000000000 -0800
+++ linux-2.6.11/arch/ia64/sn/kernel/setup.c	2005-03-21 20:30:13.000000000 -0800
@@ -72,6 +72,20 @@ EXPORT_SYMBOL(sn_rtc_cycles_per_second);

 DEFINE_PER_CPU(short, __sn_cnodeid_to_nasid[MAX_NUMNODES]);
 EXPORT_PER_CPU_SYMBOL(__sn_cnodeid_to_nasid);
+DEFINE_PER_CPU(u64, __sn_nasid_bitmask);
+EXPORT_PER_CPU_SYMBOL(__sn_nasid_bitmask);
+DEFINE_PER_CPU(u64, __sn_nasid_shift);
+EXPORT_PER_CPU_SYMBOL(__sn_nasid_shift);
+DEFINE_PER_CPU(u64, __sn_as_shift);
+EXPORT_PER_CPU_SYMBOL(__sn_as_shift);
+
+DEFINE_PER_CPU(u64, __sn_shub2);
+EXPORT_PER_CPU_SYMBOL(__sn_shub2);
+DEFINE_PER_CPU(u64, __sn_shub_1_1_found);
+EXPORT_PER_CPU_SYMBOL(__sn_shub_1_1_found);
+
+DEFINE_PER_CPU(struct nodepda_s *, __sn_nodepda);
+EXPORT_PER_CPU_SYMBOL(__sn_nodepda);

 partid_t sn_partid = -1;
 EXPORT_SYMBOL(sn_partid);
@@ -433,10 +447,10 @@ void __init sn_cpu_init(void)
 	memset(pda, 0, sizeof(pda));
 	if (ia64_sn_get_hub_info(0, &shubtype, &nasid_bitmask, &nasid_shift))
 		BUG();
-	pda->shub2 = (u8)shubtype;
-	pda->nasid_bitmask = (u16)nasid_bitmask;
-	pda->nasid_shift = (u8)nasid_shift;
-	pda->as_shift = pda->nasid_shift - 2;
+	sn_shub2 = shubtype;
+	sn_nasid_bitmask = nasid_bitmask;
+	sn_nasid_shift = nasid_shift;
+	sn_as_shift = nasid_shift - 2;

 	/*
 	 * The boot cpu makes this call again after platform initialization is
@@ -461,7 +475,7 @@ void __init sn_cpu_init(void)

 	cnode = nasid_to_cnodeid(nasid);

-	pda->p_nodepda = nodepdaindr[cnode];
+	nodepda = nodepdaindr[cnode];
 	pda->led_address  	    (typeof(pda->led_address)) (LED0 + (slice << LED_CPU_SHIFT));
 	pda->led_state = LED_ALWAYS_SET;
@@ -488,7 +502,7 @@ void __init sn_cpu_init(void)
 		sn_check_for_wars();
 		wars_have_been_checked = 1;
 	}
-	pda->shub_1_1_found = shub_1_1_found;
+	sn_shub_1_1_found = shub_1_1_found;

 	/*
 	 * Set up addresses of PIO/MEM write status registers.
Index: linux-2.6.11/include/asm-ia64/sn/addrs.h
=================================--- linux-2.6.11.orig/include/asm-ia64/sn/addrs.h	2005-03-01 23:38:38.000000000 -0800
+++ linux-2.6.11/include/asm-ia64/sn/addrs.h	2005-03-21 19:21:36.000000000 -0800
@@ -57,9 +57,9 @@
 /*
  * Define basic shift & mask constants for manipulating NASIDs and AS values.
  */
-#define NASID_BITMASK		(pda->nasid_bitmask)
-#define NASID_SHIFT		(pda->nasid_shift)
-#define AS_SHIFT		(pda->as_shift)
+#define NASID_BITMASK		(sn_nasid_bitmask)
+#define NASID_SHIFT		(sn_nasid_shift)
+#define AS_SHIFT		(sn_as_shift)
 #define AS_BITMASK		0x3UL

 #define NASID_MASK              ((u64)NASID_BITMASK << NASID_SHIFT)
Index: linux-2.6.11/include/asm-ia64/sn/nodepda.h
=================================--- linux-2.6.11.orig/include/asm-ia64/sn/nodepda.h	2005-03-01 23:38:37.000000000 -0800
+++ linux-2.6.11/include/asm-ia64/sn/nodepda.h	2005-03-21 20:12:31.000000000 -0800
@@ -74,7 +74,9 @@ typedef struct nodepda_s nodepda_t;
  *	SUBNODEPDA(cnode,sn)	-> to access subnode PDA for cnodeid/subnode
  */

-#define	nodepda		pda->p_nodepda		/* Ptr to this node's PDA */
+DECLARE_PER_CPU(struct nodepda_s *, __sn_nodepda);
+#define nodepda (__get_cpu_var(__sn_nodepda))
+
 #define	NODEPDA(cnode)		(nodepda->pernode_pdaindr[cnode])

 /*

Re: [PATCH] move cnodeid_to_nasid_table out of pda

[Dean Nelson]

On Mon, Mar 21, 2005 at 08:42:58PM -0800, Christoph Lameter wrote:
> > On Mon, 21 Mar 2005, Dean Nelson wrote:
> >
> > > Another step in the effort to eliminate the SN pda structure.
> > > This patch moves the cnodeid_to_nasid_table field out of the pda,
> > > making it a standalone per-cpu data item, and exports it so it can
> > > be accessed by kernel modules.
> 
> Yuck something went wrong with the test. Turns out that I need a lot more
> definitions to make mmtimer a module since it uses the SHUB access macros.
> The following patch applies on top of Dean's patch:

You're missing a patch from Jack Steiner that basically does what you
are trying to do in this patch. Jack's patch can be found at:

	http://lia64.bkbits.net:8080/linux-ia64-test-2.6.12

and is about 14 days old entitled:

	[IA64-SGI] [PATCH 2/2] - New chipset support for SN platform

In this patch he moved the following fields out of the pda and into their
own per-cpu data structure called sn_hub_info_s, the pointer to which is
exported.

-       u16             nasid_bitmask;
-       u8              shub2;
-       u8              nasid_shift;
-       u8              as_shift;
-       u8              shub_1_1_found;

I suggest you get that patch.

I'm also in the process of creating a patch to separate the nodepda from
the pda and export the phys_cpuid array of structures. This will allow
modules to use cpuid_to_nasid(cpuid).