[Linux-ia64] [PATCH] move mmu_gathers into local_cpu_data

[Linux-ia64] [PATCH] move mmu_gathers into local_cpu_data

[Martin Hicks]

Hi,

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA.
I'm not exactly sure where to push this patch, because it does touch
include/asm-generic/tlb.h although it has no effect on non ia64 arches.

The main reason for making this change is that the mmu_gathers array
gets very large as the number of processors increases.

mh

-- 
Wild Open Source Inc.                  mort@wildopensource.com


diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/arch/ia64/kernel/setup.c linux-2.4.21-pre5/arch/ia64/kernel/setup.c
--- linux-2.4.21-pre5.pristine/arch/ia64/kernel/setup.c	Sun Mar 30 17:31:07 2003
+++ linux-2.4.21-pre5/arch/ia64/kernel/setup.c	Tue Apr 22 13:22:48 2003
@@ -566,6 +566,8 @@
 		for (cpu = 1; cpu < NR_CPUS; ++cpu)
 			memcpy(my_cpu_data->cpu_data[cpu]->cpu_data,
 			       my_cpu_data->cpu_data, sizeof(my_cpu_data->cpu_data));
+		my_cpu_data->mmu_gathers = alloc_bootmem_pages_node(BOOT_NODE_DATA(boot_get_local_cnodeid()),
+								    sizeof(mmu_gather_t));
 	} else {
 		order = get_order(sizeof(struct cpuinfo_ia64));
 		my_cpu_data = page_address(alloc_pages_node(numa_node_id(), GFP_KERNEL, order));
@@ -575,6 +577,10 @@
 			     order);
 		for (cpu = 0; cpu < NR_CPUS; ++cpu)
 			boot_cpu_data->cpu_data[cpu]->cpu_data[smp_processor_id()] = my_cpu_data;
+
+		my_cpu_data->mmu_gathers = page_address(boot_alloc_pages_node(boot_get_local_cnodeid(),
+									      GFP_KERNEL,
+									      get_order(sizeof(mmu_gather_t)));
 	}
 #else
 	my_cpu_data = cpu_data(smp_processor_id());
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/arch/ia64/mm/init.c linux-2.4.21-pre5/arch/ia64/mm/init.c
--- linux-2.4.21-pre5.pristine/arch/ia64/mm/init.c	Sun Mar 30 17:31:07 2003
+++ linux-2.4.21-pre5/arch/ia64/mm/init.c	Tue Apr 22 13:22:48 2003
@@ -28,7 +28,9 @@
 #include <asm/uaccess.h>
 #include <asm/tlb.h>
 
+#ifndef CONFIG_NUMA
 mmu_gather_t mmu_gathers[NR_CPUS];
+#endif
 
 /* References to section boundaries: */
 extern char _stext, _etext, _edata, __init_begin, __init_end;
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/include/asm-generic/tlb.h linux-2.4.21-pre5/include/asm-generic/tlb.h
--- linux-2.4.21-pre5.pristine/include/asm-generic/tlb.h	Fri Aug  2 18:39:45 2002
+++ linux-2.4.21-pre5/include/asm-generic/tlb.h	Tue Apr 22 13:24:36 2003
@@ -31,15 +31,18 @@
 	pte_t	ptes[FREE_PTE_NR];
 } mmu_gather_t;
 
+#ifndef local_mmu_gathers
 /* Users of the generic TLB shootdown code must declare this storage space. */
 extern mmu_gather_t	mmu_gathers[NR_CPUS];
+#define local_mmu_gathers	&mmu_gathers[smp_processor_id()]
+#endif
 
 /* tlb_gather_mmu
  *	Return a pointer to an initialized mmu_gather_t.
  */
 static inline mmu_gather_t *tlb_gather_mmu(struct mm_struct *mm)
 {
-	mmu_gather_t *tlb = &mmu_gathers[smp_processor_id()];
+	mmu_gather_t *tlb = local_mmu_gathers;
 
 	tlb->mm = mm;
 	/* Use fast mode if there is only one user of this mm (this process) */
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/include/asm-ia64/processor.h linux-2.4.21-pre5/include/asm-ia64/processor.h
--- linux-2.4.21-pre5.pristine/include/asm-ia64/processor.h	Wed Apr 16 17:41:04 2003
+++ linux-2.4.21-pre5/include/asm-ia64/processor.h	Tue Apr 22 13:38:58 2003
@@ -189,6 +189,7 @@
 	void *node_directory;
 	int numa_node_id;
 	struct cpuinfo_ia64 *cpu_data[NR_CPUS];
+        void *mmu_gathers;
 #endif
 	/* Platform specific word.  MUST BE LAST IN STRUCT */
 	__u64 platform_specific;
diff -X /home/mort/diffex -uNr linux-2.4.21-pre5.pristine/include/asm-ia64/tlb.h linux-2.4.21-pre5/include/asm-ia64/tlb.h
--- linux-2.4.21-pre5.pristine/include/asm-ia64/tlb.h	Fri Apr 11 17:56:37 2003
+++ linux-2.4.21-pre5/include/asm-ia64/tlb.h	Tue Apr 22 13:38:59 2003
@@ -1 +1,6 @@
+
+#ifdef CONFIG_NUMA
+#define local_mmu_gathers local_cpu_data->mmu_gathers
+#endif
+
 #include <asm-generic/tlb.h>

Re: [Linux-ia64] [PATCH] move mmu_gathers into local_cpu_data

[Christoph Hellwig]

On Tue, Apr 22, 2003 at 02:23:31PM -0400, Martin Hicks wrote:
> 
> Hi,
> 
> This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA.
> I'm not exactly sure where to push this patch, because it does touch
> include/asm-generic/tlb.h although it has no effect on non ia64 arches.
> 
> The main reason for making this change is that the mmu_gathers array
> gets very large as the number of processors increases.

For 2.5 please just make mmu_gathers per-cpu data for all arches.

Re: [Linux-ia64] [PATCH] move mmu_gathers into local_cpu_data

[Martin Hicks]

On Tue, Apr 22, 2003 at 09:39:38PM +0100, Christoph Hellwig wrote:
> On Tue, Apr 22, 2003 at 02:23:31PM -0400, Martin Hicks wrote:
> > 
> > Hi,
> > 
> > This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA.
> > I'm not exactly sure where to push this patch, because it does touch
> > include/asm-generic/tlb.h although it has no effect on non ia64 arches.
> > 
> > The main reason for making this change is that the mmu_gathers array
> > gets very large as the number of processors increases.
> 
> For 2.5 please just make mmu_gathers per-cpu data for all arches.
> 

That was my intention.  For 2.4 my goal was "touch as little as
possible, and hopefully it will get accepted".

mh

-- 
Wild Open Source Inc.                  mort@wildopensource.com

Re: [Linux-ia64] [PATCH] move mmu_gathers into local_cpu_data

[Bjorn Helgaas]

On Tuesday 22 April 2003 12:23 pm, Martin Hicks wrote:
> This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA.
> I'm not exactly sure where to push this patch, because it does touch
> include/asm-generic/tlb.h although it has no effect on non ia64 arches.

I applied this for 2.4, with a couple minor changes:
	- moved the static mmu_gathers[] declaration to setup.c,
	  so all the mmu_gather stuff is together
	- moved the cpuinfo change outside CONFIG_NUMA,
	  so we can just use the same local_mmu_gathers
	  #define all the time

I'd actually like to see the dynamic mmu_gather allocation done
in the non-NUMA case as well.  Non-NUMA SMP kernels are statically
allocating 128K (NR_CPUS = 32, 4K per mmu_gather_t), but typical
machines will only use 8K or 16K of that.

Here's the actual patch I applied.

#### AUTHOR mort@wildopensource.com
#### COMMENT START
### Comments for ChangeSet
Move mmu_gathers[] to local_cpu_data on ia64.

This is a patch that moves mmu_gathers into local_cpu_data on IA64 NUMA.
I'm not exactly sure where to push this patch, because it does touch
include/asm-generic/tlb.h although it has no effect on non ia64 arches.

The main reason for making this change is that the mmu_gathers array
gets very large as the number of processors increases.
### Comments for include/asm-generic/tlb.h
(local_mmu_gathers): Use if defined.
### Comments for include/asm-ia64/tlb.h
(local_mmu_gathers): Define.
### Comments for arch/ia64/kernel/setup.c
(mmu_gathers[]): Allocate statically for non-NUMA, per present CPU for NUMA.
### Comments for arch/ia64/mm/init.c
(mmu_gathers[]): move to kernel/setup.c alongside NUMA allocation.
### Comments for include/asm-ia64/processor.h
(mmu_gathers): Add to cpuinfo_ia64.
#### COMMENT END

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher.
# This patch includes the following deltas:
#	           ChangeSet	1.1145  -> 1.1146 
#	include/asm-generic/tlb.h	1.2     -> 1.3    
#	arch/ia64/kernel/setup.c	1.11    -> 1.12   
#	include/asm-ia64/processor.h	1.16    -> 1.17   
#	include/asm-ia64/tlb.h	1.1     -> 1.2    
#	 arch/ia64/mm/init.c	1.11    -> 1.12   
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 03/05/15	bjorn_helgaas@hp.com	1.1146
# Move mmu_gathers into per cpu area.
# --------------------------------------------
#
diff -Nru a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
--- a/arch/ia64/kernel/setup.c	Thu May 15 10:44:50 2003
+++ b/arch/ia64/kernel/setup.c	Thu May 15 10:44:50 2003
@@ -40,6 +40,7 @@
 #include <asm/system.h>
 #include <asm/mca.h>
 #include <asm/smp.h>
+#include <asm/tlb.h>
 
 #ifdef CONFIG_BLK_DEV_RAM
 # include <linux/blk.h>
@@ -58,6 +59,7 @@
  struct cpuinfo_ia64 *boot_cpu_data;
 #else
  struct cpuinfo_ia64 _cpu_data[NR_CPUS] __attribute__ ((section ("__special_page_section")));
+ mmu_gather_t mmu_gathers[NR_CPUS];
 #endif
 
 unsigned long ia64_cycles_per_usec;
@@ -567,6 +569,8 @@
 		for (cpu = 1; cpu < NR_CPUS; ++cpu)
 			memcpy(my_cpu_data->cpu_data[cpu]->cpu_data,
 			       my_cpu_data->cpu_data, sizeof(my_cpu_data->cpu_data));
+		my_cpu_data->mmu_gathers = alloc_bootmem_pages_node(BOOT_NODE_DATA(boot_get_local_cnodeid()),
+								    sizeof(mmu_gather_t));
 	} else {
 		order = get_order(sizeof(struct cpuinfo_ia64));
 		my_cpu_data = page_address(alloc_pages_node(numa_node_id(), GFP_KERNEL, order));
@@ -576,9 +580,14 @@
 			     order);
 		for (cpu = 0; cpu < NR_CPUS; ++cpu)
 			boot_cpu_data->cpu_data[cpu]->cpu_data[smp_processor_id()] = my_cpu_data;
+
+		my_cpu_data->mmu_gathers = page_address(boot_alloc_pages_node(boot_get_local_cnodeid(),
+									      GFP_KERNEL,
+									      get_order(sizeof(mmu_gather_t)));
 	}
 #else
 	my_cpu_data = cpu_data(smp_processor_id());
+	my_cpu_data->mmu_gathers = &mmu_gathers[smp_processor_id()];
 #endif
 
 	/*
diff -Nru a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
--- a/arch/ia64/mm/init.c	Thu May 15 10:44:50 2003
+++ b/arch/ia64/mm/init.c	Thu May 15 10:44:50 2003
@@ -26,9 +26,6 @@
 #include <asm/sal.h>
 #include <asm/system.h>
 #include <asm/uaccess.h>
-#include <asm/tlb.h>
-
-mmu_gather_t mmu_gathers[NR_CPUS];
 
 /* References to section boundaries: */
 extern char _stext, _etext, _edata, __init_begin, __init_end;
diff -Nru a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
--- a/include/asm-generic/tlb.h	Thu May 15 10:44:50 2003
+++ b/include/asm-generic/tlb.h	Thu May 15 10:44:50 2003
@@ -31,15 +31,18 @@
 	pte_t	ptes[FREE_PTE_NR];
 } mmu_gather_t;
 
+#ifndef local_mmu_gathers
 /* Users of the generic TLB shootdown code must declare this storage space. */
 extern mmu_gather_t	mmu_gathers[NR_CPUS];
+#define local_mmu_gathers	&mmu_gathers[smp_processor_id()]
+#endif
 
 /* tlb_gather_mmu
  *	Return a pointer to an initialized mmu_gather_t.
  */
 static inline mmu_gather_t *tlb_gather_mmu(struct mm_struct *mm)
 {
-	mmu_gather_t *tlb = &mmu_gathers[smp_processor_id()];
+	mmu_gather_t *tlb = local_mmu_gathers;
 
 	tlb->mm = mm;
 	/* Use fast mode if there is only one user of this mm (this process) */
diff -Nru a/include/asm-ia64/processor.h b/include/asm-ia64/processor.h
--- a/include/asm-ia64/processor.h	Thu May 15 10:44:50 2003
+++ b/include/asm-ia64/processor.h	Thu May 15 10:44:50 2003
@@ -168,6 +168,7 @@
 	__u32 ptce_count[2];
 	__u32 ptce_stride[2];
 	struct task_struct *ksoftirqd;	/* kernel softirq daemon for this CPU */
+	void *mmu_gathers;
 # ifdef CONFIG_PERFMON
 	unsigned long pfm_syst_info;
 # endif
diff -Nru a/include/asm-ia64/tlb.h b/include/asm-ia64/tlb.h
--- a/include/asm-ia64/tlb.h	Thu May 15 10:44:50 2003
+++ b/include/asm-ia64/tlb.h	Thu May 15 10:44:50 2003
@@ -1 +1,3 @@
+#define local_mmu_gathers local_cpu_data->mmu_gathers
+
 #include <asm-generic/tlb.h>