[PATCH 0/2] Align MIPS swapper_pg_dir for faster code.

[PATCH 0/2] Align MIPS swapper_pg_dir for faster code.

[David Daney]

From: David Daney <david.daney@cavium.com>

The MIPS swapper_pg_dir needs 64K alignment for faster TLB refills in
kernel mappings.  There are two parts to the patch set:

1) Modify generic vmlinux.lds.h to allow architectures to place
   additional sections at the start of .bss.  This allows alignment
   constraints to be met with minimal holes added for padding.
   Putting this in common code should reduce the risk of future
   changes to the linker scripts not being propagated to MIPS (or any
   other architecture that needs something like this).

2) Align the MIPS swapper_pg_dir.

Since the initial use of the code is for MIPS, perhaps both parts
could be merged by Ralf's tree (after collecting any Acked-bys).

David Daney (2):
  vmlinux.lds.h: Allow architectures to add sections to the front of
    .bss
  MIPS: Align swapper_pg_dir to 64K for better TLB Refill code.

 arch/mips/kernel/vmlinux.lds.S    |   21 +++++++++++++++++++--
 arch/mips/mm/init.c               |   17 +++++++++--------
 include/asm-generic/vmlinux.lds.h |    9 +++++++++
 3 files changed, 37 insertions(+), 10 deletions(-)

-- 
1.7.2.3

[PATCH 1/2] vmlinux.lds.h: Allow architectures to add sections to the front of .bss

[David Daney]

From: David Daney <david.daney@cavium.com>

Follow-on MIPS patch will put an object here that needs 64K alignment
to minimize padding.

For those architectures that don't define BSS_FIRST_SECTIONS, there is
no change.

Signed-off-by: David Daney <david.daney@cavium.com>
---
 include/asm-generic/vmlinux.lds.h |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 4e2e1cc..d1ea7ce 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -530,9 +530,18 @@
 		*(.scommon)						\
 	}
 
+/*
+ * Allow archectures to redefine BSS_FIRST_SECTIONS to add extra
+ * sections to the front of bss.
+ */
+#ifndef BSS_FIRST_SECTIONS
+#define BSS_FIRST_SECTIONS
+#endif
+
 #define BSS(bss_align)							\
 	. = ALIGN(bss_align);						\
 	.bss : AT(ADDR(.bss) - LOAD_OFFSET) {				\
+		BSS_FIRST_SECTIONS					\
 		*(.bss..page_aligned)					\
 		*(.dynbss)						\
 		*(.bss)							\
-- 
1.7.2.3

[PATCH 1/2] vmlinux.lds.h: Allow architectures to add sections to the front of .bss

[David Daney]

From: David Daney <david.daney@cavium.com>

Follow-on MIPS patch will put an object here that needs 64K alignment
to minimize padding.

For those architectures that don't define BSS_FIRST_SECTIONS, there is
no change.

Signed-off-by: David Daney <david.daney@cavium.com>
---
 include/asm-generic/vmlinux.lds.h |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 4e2e1cc..d1ea7ce 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -530,9 +530,18 @@
 		*(.scommon)						\
 	}
 
+/*
+ * Allow archectures to redefine BSS_FIRST_SECTIONS to add extra
+ * sections to the front of bss.
+ */
+#ifndef BSS_FIRST_SECTIONS
+#define BSS_FIRST_SECTIONS
+#endif
+
 #define BSS(bss_align)							\
 	. = ALIGN(bss_align);						\
 	.bss : AT(ADDR(.bss) - LOAD_OFFSET) {				\
+		BSS_FIRST_SECTIONS					\
 		*(.bss..page_aligned)					\
 		*(.dynbss)						\
 		*(.bss)							\
-- 
1.7.2.3

[PATCH 2/2] MIPS: Align swapper_pg_dir to 64K for better TLB Refill code.

[David Daney]

From: David Daney <david.daney@cavium.com>

We can save an instruction in the TLB Refill path for kernel mappings
by aligning swapper_pg_dir on a 64K boundary.  The address of
swapper_pg_dir can be generated with a single LUI instead of
LUI/{D}ADDUI.

The alignment of __init_end is bumped up to 64K so there are no holes
between it and swapper_pg_dir, which is placed at the very beginning
of .bss.

The alignment of invalid_pmd_table and invalid_pte_table can be
relaxed to PAGE_SIZE.  We do this by using __page_aligned_bss, which
has the added benefit of eliminating alignment holes in .bss.

Signed-off-by: David Daney <david.daney@cavium.com>
---
 arch/mips/kernel/vmlinux.lds.S |   21 +++++++++++++++++++--
 arch/mips/mm/init.c            |   17 +++++++++--------
 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index df243a6..007ccbe 100644
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -1,6 +1,13 @@
 #include <asm/asm-offsets.h>
 #include <asm/page.h>
 #include <asm/thread_info.h>
+
+/*
+ * Put .bss..swapper_pg_dir as the first thing in .bss. This will
+ * ensure that it has .bss alignment (64K).
+ */
+#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir)
+
 #include <asm-generic/vmlinux.lds.h>
 
 #undef mips
@@ -119,11 +126,21 @@ SECTIONS
 	}
 
 	PERCPU_SECTION(1 << CONFIG_MIPS_L1_CACHE_SHIFT)
-	. = ALIGN(PAGE_SIZE);
+	/*
+	 * Align to 64K in attempt to eliminate holes before the
+	 * .bss..swapper_pg_dir section at the start of .bss.  This
+	 * also satisfies PAGE_SIZE alignment as the largest page size
+	 * allowed is 64K.
+	 */
+	. = ALIGN(0x10000);
 	__init_end = .;
 	/* freed after init ends here */
 
-	BSS_SECTION(0, 0, 0)
+	/*
+	 * Force .bss to 64K alignment so that .bss..swapper_pg_dir
+	 * gets that alignment.  .sbss should be empty, so there will be
+	 * no holes after __init_end. */
+	BSS_SECTION(0, 0x10000, 0)
 
 	_end = . ;
 
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 1a85ba9..be9acb2 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -469,19 +469,20 @@ void __init_refok free_initmem(void)
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 unsigned long pgd_current[NR_CPUS];
 #endif
-/*
- * On 64-bit we've got three-level pagetables with a slightly
- * different layout ...
- */
-#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))
 
 /*
  * gcc 3.3 and older have trouble determining that PTRS_PER_PGD and PGD_ORDER
  * are constants.  So we use the variants from asm-offset.h until that gcc
  * will officially be retired.
+ *
+ * Align swapper_pg_dir in to 64K, allows its address to be loaded
+ * with a single LUI instruction in the TLB handlers.  If we used
+ * __aligned(64K), its size would get rounded up to the alignment
+ * size, and waste space.  So we place it in its own section and align
+ * it in the linker script.
  */
-pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);
+pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(.bss..swapper_pg_dir);
 #ifndef __PAGETABLE_PMD_FOLDED
-pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);
+pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
 #endif
-pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);
+pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
-- 
1.7.2.3

[PATCH 2/2] MIPS: Align swapper_pg_dir to 64K for better TLB Refill code.

[David Daney]

From: David Daney <david.daney@cavium.com>

We can save an instruction in the TLB Refill path for kernel mappings
by aligning swapper_pg_dir on a 64K boundary.  The address of
swapper_pg_dir can be generated with a single LUI instead of
LUI/{D}ADDUI.

The alignment of __init_end is bumped up to 64K so there are no holes
between it and swapper_pg_dir, which is placed at the very beginning
of .bss.

The alignment of invalid_pmd_table and invalid_pte_table can be
relaxed to PAGE_SIZE.  We do this by using __page_aligned_bss, which
has the added benefit of eliminating alignment holes in .bss.

Signed-off-by: David Daney <david.daney@cavium.com>
---
 arch/mips/kernel/vmlinux.lds.S |   21 +++++++++++++++++++--
 arch/mips/mm/init.c            |   17 +++++++++--------
 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index df243a6..007ccbe 100644
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -1,6 +1,13 @@
 #include <asm/asm-offsets.h>
 #include <asm/page.h>
 #include <asm/thread_info.h>
+
+/*
+ * Put .bss..swapper_pg_dir as the first thing in .bss. This will
+ * ensure that it has .bss alignment (64K).
+ */
+#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir)
+
 #include <asm-generic/vmlinux.lds.h>
 
 #undef mips
@@ -119,11 +126,21 @@ SECTIONS
 	}
 
 	PERCPU_SECTION(1 << CONFIG_MIPS_L1_CACHE_SHIFT)
-	. = ALIGN(PAGE_SIZE);
+	/*
+	 * Align to 64K in attempt to eliminate holes before the
+	 * .bss..swapper_pg_dir section at the start of .bss.  This
+	 * also satisfies PAGE_SIZE alignment as the largest page size
+	 * allowed is 64K.
+	 */
+	. = ALIGN(0x10000);
 	__init_end = .;
 	/* freed after init ends here */
 
-	BSS_SECTION(0, 0, 0)
+	/*
+	 * Force .bss to 64K alignment so that .bss..swapper_pg_dir
+	 * gets that alignment.  .sbss should be empty, so there will be
+	 * no holes after __init_end. */
+	BSS_SECTION(0, 0x10000, 0)
 
 	_end = . ;
 
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 1a85ba9..be9acb2 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -469,19 +469,20 @@ void __init_refok free_initmem(void)
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 unsigned long pgd_current[NR_CPUS];
 #endif
-/*
- * On 64-bit we've got three-level pagetables with a slightly
- * different layout ...
- */
-#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))
 
 /*
  * gcc 3.3 and older have trouble determining that PTRS_PER_PGD and PGD_ORDER
  * are constants.  So we use the variants from asm-offset.h until that gcc
  * will officially be retired.
+ *
+ * Align swapper_pg_dir in to 64K, allows its address to be loaded
+ * with a single LUI instruction in the TLB handlers.  If we used
+ * __aligned(64K), its size would get rounded up to the alignment
+ * size, and waste space.  So we place it in its own section and align
+ * it in the linker script.
  */
-pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);
+pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(.bss..swapper_pg_dir);
 #ifndef __PAGETABLE_PMD_FOLDED
-pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);
+pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
 #endif
-pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);
+pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
-- 
1.7.2.3

Re: [PATCH 0/2] Align MIPS swapper_pg_dir for faster code.

[Arnd Bergmann]

On Tuesday 14 August 2012, David Daney wrote:
> Since the initial use of the code is for MIPS, perhaps both parts
> could be merged by Ralf's tree (after collecting any Acked-bys).

Acked-by: Arnd Bergmann <arnd@arndb.de>

Re: [PATCH 0/2] Align MIPS swapper_pg_dir for faster code.

[Ralf Baechle]

On Tue, Aug 14, 2012 at 11:07:59AM -0700, David Daney wrote:

> From: David Daney <david.daney@cavium.com>
> 
> The MIPS swapper_pg_dir needs 64K alignment for faster TLB refills in
> kernel mappings.  There are two parts to the patch set:
> 
> 1) Modify generic vmlinux.lds.h to allow architectures to place
>    additional sections at the start of .bss.  This allows alignment
>    constraints to be met with minimal holes added for padding.
>    Putting this in common code should reduce the risk of future
>    changes to the linker scripts not being propagated to MIPS (or any
>    other architecture that needs something like this).
> 
> 2) Align the MIPS swapper_pg_dir.
> 
> Since the initial use of the code is for MIPS, perhaps both parts
> could be merged by Ralf's tree (after collecting any Acked-bys).

Looks good to me but will wait a bit longer for comments and (N)Acks
before merging.

  Ralf