[PATCH v2 0/4] arm64: simplify and optimize kernel mapping

[PATCH v2 0/4] arm64: simplify and optimize kernel mapping

[Ard Biesheuvel]

This series makes a couple of minor changes that should result in the
kernel being mapped in a more efficient manner.

First of all, it introduces 'segment' to describe the various chunks that
make up the kernel runtime mapping, and replaces some instances of chunk
with segment (#1)

Then, it merges the .head.text with the .text section (patch #3) after moving
everything except the kernel and EFI header into the __init section (patch #2)

Then, it standardizes the segment alignment to 64 KB for all page sizes.
(patch #4). In the example below (4 KB granule, with Jeremy's PTE_CONT
patch applied), we lose 80 KB in total to padding (some of which is covered
by .init), but the resulting mappings do look somewhat better.

Before:

0xffff000008082000-0xffff000008090000    56K  ro x  SHD AF         UXN MEM
0xffff000008090000-0xffff000008200000  1472K  ro x  SHD AF CON     UXN MEM
0xffff000008200000-0xffff000008600000     4M  ro x  SHD AF     BLK UXN MEM
0xffff000008600000-0xffff000008660000   384K  ro x  SHD AF CON     UXN MEM
0xffff000008660000-0xffff00000866c000    48K  ro x  SHD AF         UXN MEM
0xffff00000866c000-0xffff000008670000    16K  ro NX SHD AF         UXN MEM
0xffff000008670000-0xffff000008900000  2624K  ro NX SHD AF CON     UXN MEM
0xffff000008900000-0xffff000008909000    36K  ro NX SHD AF         UXN MEM
0xffff000008c39000-0xffff000008c40000    28K  RW NX SHD AF         UXN MEM
0xffff000008c40000-0xffff000008d50000  1088K  RW NX SHD AF CON     UXN MEM
0xffff000008d50000-0xffff000008d57000    28K  RW NX SHD AF         UXN MEM

After:

0xffff000008080000-0xffff000008200000  1536K  ro x  SHD AF CON     UXN MEM
0xffff000008200000-0xffff000008600000     4M  ro x  SHD AF     BLK UXN MEM
0xffff000008600000-0xffff000008670000   448K  ro x  SHD AF CON     UXN MEM
0xffff000008670000-0xffff000008910000  2688K  ro NX SHD AF CON     UXN MEM
0xffff000008c50000-0xffff000008d60000  1088K  RW NX SHD AF CON     UXN MEM
0xffff000008d60000-0xffff000008d6b000    44K  RW NX SHD AF         UXN MEM

Changes since v2:
- add patch #1 to introduce 'segment' to describe the parts of the static kernel
  Image that are mapped with different permissions and/or lifetimes
- drop dependency on 4 KB pages for CONFIG_DEBUG_ALIGN_RODATA (#4)

Ard Biesheuvel (4):
  arm64: use 'segment' rather than 'chunk' to describe mapped kernel
    regions
  arm64: move early boot code to the .init segment
  arm64: cover the .head.text section in the .text segment mapping
  arm64: simplify kernel segment mapping granularity

 arch/arm64/Kconfig.debug        |  2 +-
 arch/arm64/kernel/efi-entry.S   |  2 +-
 arch/arm64/kernel/head.S        | 32 +++++++++-----------
 arch/arm64/kernel/image.h       |  4 +++
 arch/arm64/kernel/vmlinux.lds.S | 26 +++++++++-------
 arch/arm64/mm/mmu.c             | 22 +++++++-------
 6 files changed, 47 insertions(+), 41 deletions(-)

-- 
2.5.0

[PATCH v2 1/4] arm64: use 'segment' rather than 'chunk' to describe mapped kernel regions

[Ard Biesheuvel]

Replace the poorly defined term chunk with segment, which is a term that is
already used by the ELF spec to describe contiguous mappings with the same
permission attributes of statically allocated ranges of an executable.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
 arch/arm64/mm/mmu.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index f3e5c74233f3..9be2065f8ddb 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -471,8 +471,8 @@ void fixup_init(void)
 	unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin));
 }
 
-static void __init map_kernel_chunk(pgd_t *pgd, void *va_start, void *va_end,
-				    pgprot_t prot, struct vm_struct *vma)
+static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
+				      pgprot_t prot, struct vm_struct *vma)
 {
 	phys_addr_t pa_start = __pa(va_start);
 	unsigned long size = va_end - va_start;
@@ -499,11 +499,11 @@ static void __init map_kernel(pgd_t *pgd)
 {
 	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
 
-	map_kernel_chunk(pgd, _stext, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text);
-	map_kernel_chunk(pgd, __start_rodata, _etext, PAGE_KERNEL, &vmlinux_rodata);
-	map_kernel_chunk(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
-			 &vmlinux_init);
-	map_kernel_chunk(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
+	map_kernel_segment(pgd, _stext, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text);
+	map_kernel_segment(pgd, __start_rodata, _etext, PAGE_KERNEL, &vmlinux_rodata);
+	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
+			   &vmlinux_init);
+	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
 
 	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
 		/*
-- 
2.5.0

[PATCH v2 1/4] arm64: use 'segment' rather than 'chunk' to describe mapped kernel regions

[Mark Rutland]

On Wed, Mar 30, 2016 at 05:43:06PM +0200, Ard Biesheuvel wrote:
> Replace the poorly defined term chunk with segment, which is a term that is
> already used by the ELF spec to describe contiguous mappings with the same
> permission attributes of statically allocated ranges of an executable.
> 
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>

Sounds better to me.

Acked-by: Mark Rutland <mark.rutland@arm.com>

Mark.

> ---
>  arch/arm64/mm/mmu.c | 14 +++++++-------
>  1 file changed, 7 insertions(+), 7 deletions(-)
> 
> diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
> index f3e5c74233f3..9be2065f8ddb 100644
> --- a/arch/arm64/mm/mmu.c
> +++ b/arch/arm64/mm/mmu.c
> @@ -471,8 +471,8 @@ void fixup_init(void)
>  	unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin));
>  }
>  
> -static void __init map_kernel_chunk(pgd_t *pgd, void *va_start, void *va_end,
> -				    pgprot_t prot, struct vm_struct *vma)
> +static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
> +				      pgprot_t prot, struct vm_struct *vma)
>  {
>  	phys_addr_t pa_start = __pa(va_start);
>  	unsigned long size = va_end - va_start;
> @@ -499,11 +499,11 @@ static void __init map_kernel(pgd_t *pgd)
>  {
>  	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
>  
> -	map_kernel_chunk(pgd, _stext, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text);
> -	map_kernel_chunk(pgd, __start_rodata, _etext, PAGE_KERNEL, &vmlinux_rodata);
> -	map_kernel_chunk(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
> -			 &vmlinux_init);
> -	map_kernel_chunk(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
> +	map_kernel_segment(pgd, _stext, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text);
> +	map_kernel_segment(pgd, __start_rodata, _etext, PAGE_KERNEL, &vmlinux_rodata);
> +	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
> +			   &vmlinux_init);
> +	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data);
>  
>  	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
>  		/*
> -- 
> 2.5.0
> 

[PATCH v2 2/4] arm64: move early boot code to the .init segment

[Ard Biesheuvel]

Apart from the arm64/linux and EFI header data structures, there is nothing
in the .head.text section that must reside at the beginning of the Image.
So let's move it to the .init section where it belongs.

Note that this involves some minor tweaking of the EFI header, primarily
because the address of 'stext' no longer coincides with the start of the
.text section. It also requires a couple of relocated symbol references
to be slightly rewritten or their definition moved to the linker script.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
 arch/arm64/kernel/efi-entry.S |  2 +-
 arch/arm64/kernel/head.S      | 32 +++++++++-----------
 arch/arm64/kernel/image.h     |  4 +++
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S
index cae3112f7791..e88c064b845c 100644
--- a/arch/arm64/kernel/efi-entry.S
+++ b/arch/arm64/kernel/efi-entry.S
@@ -62,7 +62,7 @@ ENTRY(entry)
 	 */
 	mov	x20, x0		// DTB address
 	ldr	x0, [sp, #16]	// relocated _text address
-	movz	x21, #:abs_g0:stext_offset
+	ldr	w21, =stext_offset
 	add	x21, x0, x21
 
 	/*
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 4203d5f257bc..b43417618847 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -102,8 +102,6 @@ _head:
 #endif
 
 #ifdef CONFIG_EFI
-	.globl	__efistub_stext_offset
-	.set	__efistub_stext_offset, stext - _head
 	.align 3
 pe_header:
 	.ascii	"PE"
@@ -123,11 +121,11 @@ optional_header:
 	.short	0x20b				// PE32+ format
 	.byte	0x02				// MajorLinkerVersion
 	.byte	0x14				// MinorLinkerVersion
-	.long	_end - stext			// SizeOfCode
+	.long	_end - efi_header_end		// SizeOfCode
 	.long	0				// SizeOfInitializedData
 	.long	0				// SizeOfUninitializedData
 	.long	__efistub_entry - _head		// AddressOfEntryPoint
-	.long	__efistub_stext_offset		// BaseOfCode
+	.long	efi_header_end - _head		// BaseOfCode
 
 extra_header_fields:
 	.quad	0				// ImageBase
@@ -144,7 +142,7 @@ extra_header_fields:
 	.long	_end - _head			// SizeOfImage
 
 	// Everything before the kernel image is considered part of the header
-	.long	__efistub_stext_offset		// SizeOfHeaders
+	.long	efi_header_end - _head		// SizeOfHeaders
 	.long	0				// CheckSum
 	.short	0xa				// Subsystem (EFI application)
 	.short	0				// DllCharacteristics
@@ -188,10 +186,10 @@ section_table:
 	.byte	0
 	.byte	0
 	.byte	0        		// end of 0 padding of section name
-	.long	_end - stext		// VirtualSize
-	.long	__efistub_stext_offset	// VirtualAddress
-	.long	_edata - stext		// SizeOfRawData
-	.long	__efistub_stext_offset	// PointerToRawData
+	.long	_end - efi_header_end	// VirtualSize
+	.long	efi_header_end - _head	// VirtualAddress
+	.long	_edata - efi_header_end	// SizeOfRawData
+	.long	efi_header_end - _head	// PointerToRawData
 
 	.long	0		// PointerToRelocations (0 for executables)
 	.long	0		// PointerToLineNumbers (0 for executables)
@@ -200,15 +198,18 @@ section_table:
 	.long	0xe0500020	// Characteristics (section flags)
 
 	/*
-	 * EFI will load stext onwards at the 4k section alignment
+	 * EFI will load .text onwards at the 4k section alignment
 	 * described in the PE/COFF header. To ensure that instruction
 	 * sequences using an adrp and a :lo12: immediate will function
-	 * correctly at this alignment, we must ensure that stext is
+	 * correctly at this alignment, we must ensure that .text is
 	 * placed at a 4k boundary in the Image to begin with.
 	 */
 	.align 12
+efi_header_end:
 #endif
 
+	__INIT
+
 ENTRY(stext)
 	bl	preserve_boot_args
 	bl	el2_setup			// Drop to EL1, w20=cpu_boot_mode
@@ -223,12 +224,12 @@ ENTRY(stext)
 	 * the TCR will have been set.
 	 */
 	ldr	x27, 0f				// address to jump to after
-						// MMU has been enabled
+	neg	x27, x27			// MMU has been enabled
 	adr_l	lr, __enable_mmu		// return (PIC) address
 	b	__cpu_setup			// initialise processor
 ENDPROC(stext)
 	.align	3
-0:	.quad	__mmap_switched - (_head - TEXT_OFFSET) + KIMAGE_VADDR
+0:	.quad	(_text - TEXT_OFFSET) - __mmap_switched - KIMAGE_VADDR
 
 /*
  * Preserve the arguments passed by the bootloader in x0 .. x3
@@ -397,7 +398,7 @@ __create_page_tables:
 	ldr	x5, =KIMAGE_VADDR
 	add	x5, x5, x23			// add KASLR displacement
 	create_pgd_entry x0, x5, x3, x6
-	ldr	w6, kernel_img_size
+	ldr	w6, =kernel_img_size
 	add	x6, x6, x5
 	mov	x3, x24				// phys offset
 	create_block_map x0, x7, x3, x5, x6
@@ -414,9 +415,6 @@ __create_page_tables:
 
 	ret	x28
 ENDPROC(__create_page_tables)
-
-kernel_img_size:
-	.long	_end - (_head - TEXT_OFFSET)
 	.ltorg
 
 /*
diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
index 5e360ce88f10..4fd72da646a3 100644
--- a/arch/arm64/kernel/image.h
+++ b/arch/arm64/kernel/image.h
@@ -71,8 +71,12 @@
 	DEFINE_IMAGE_LE64(_kernel_offset_le, TEXT_OFFSET);	\
 	DEFINE_IMAGE_LE64(_kernel_flags_le, __HEAD_FLAGS);
 
+kernel_img_size = _end - (_text - TEXT_OFFSET);
+
 #ifdef CONFIG_EFI
 
+__efistub_stext_offset = stext - _text;
+
 /*
  * Prevent the symbol aliases below from being emitted into the kallsyms
  * table, by forcing them to be absolute symbols (which are conveniently
-- 
2.5.0

[PATCH v2 2/4] arm64: move early boot code to the .init segment

[Will Deacon]

Hi Ard,

On Wed, Mar 30, 2016 at 05:43:07PM +0200, Ard Biesheuvel wrote:
> Apart from the arm64/linux and EFI header data structures, there is nothing
> in the .head.text section that must reside at the beginning of the Image.
> So let's move it to the .init section where it belongs.
> 
> Note that this involves some minor tweaking of the EFI header, primarily
> because the address of 'stext' no longer coincides with the start of the
> .text section. It also requires a couple of relocated symbol references
> to be slightly rewritten or their definition moved to the linker script.

[...]

>  ENTRY(stext)
>  	bl	preserve_boot_args
>  	bl	el2_setup			// Drop to EL1, w20=cpu_boot_mode
> @@ -223,12 +224,12 @@ ENTRY(stext)
>  	 * the TCR will have been set.
>  	 */
>  	ldr	x27, 0f				// address to jump to after
> -						// MMU has been enabled
> +	neg	x27, x27			// MMU has been enabled
>  	adr_l	lr, __enable_mmu		// return (PIC) address
>  	b	__cpu_setup			// initialise processor
>  ENDPROC(stext)
>  	.align	3
> -0:	.quad	__mmap_switched - (_head - TEXT_OFFSET) + KIMAGE_VADDR
> +0:	.quad	(_text - TEXT_OFFSET) - __mmap_switched - KIMAGE_VADDR

I'm struggling to understand why you need to change this. Furthermore,
it looks like the gas docs for expressions require that addition/subtraction
can only be performed on arguments that are in the same section, so
this feels pretty rickety.

What's the problem you're solving here?

Will

[PATCH v2 2/4] arm64: move early boot code to the .init segment

[Ard Biesheuvel]

On 14 April 2016 at 18:32, Will Deacon <will.deacon@arm.com> wrote:
> Hi Ard,
>
> On Wed, Mar 30, 2016 at 05:43:07PM +0200, Ard Biesheuvel wrote:
>> Apart from the arm64/linux and EFI header data structures, there is nothing
>> in the .head.text section that must reside at the beginning of the Image.
>> So let's move it to the .init section where it belongs.
>>
>> Note that this involves some minor tweaking of the EFI header, primarily
>> because the address of 'stext' no longer coincides with the start of the
>> .text section. It also requires a couple of relocated symbol references
>> to be slightly rewritten or their definition moved to the linker script.
>
> [...]
>
>>  ENTRY(stext)
>>       bl      preserve_boot_args
>>       bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
>> @@ -223,12 +224,12 @@ ENTRY(stext)
>>        * the TCR will have been set.
>>        */
>>       ldr     x27, 0f                         // address to jump to after
>> -                                             // MMU has been enabled
>> +     neg     x27, x27                        // MMU has been enabled
>>       adr_l   lr, __enable_mmu                // return (PIC) address
>>       b       __cpu_setup                     // initialise processor
>>  ENDPROC(stext)
>>       .align  3
>> -0:   .quad   __mmap_switched - (_head - TEXT_OFFSET) + KIMAGE_VADDR
>> +0:   .quad   (_text - TEXT_OFFSET) - __mmap_switched - KIMAGE_VADDR
>
> I'm struggling to understand why you need to change this. Furthermore,
> it looks like the gas docs for expressions require that addition/subtraction
> can only be performed on arguments that are in the same section, so
> this feels pretty rickety.
>
> What's the problem you're solving here?
>

The problem is that this function is no longer in the same section as
_head aka _text, so the latter can only appear in the reloc expression
as a positive term, and the remaining ones will all be combined into
the addend.

[PATCH v2 2/4] arm64: move early boot code to the .init segment

[Ard Biesheuvel]

On 04/14/2016 06:39 PM, Ard Biesheuvel wrote:
> On 14 April 2016 at 18:32, Will Deacon <will.deacon@arm.com> wrote:
>> Hi Ard,
>>
>> On Wed, Mar 30, 2016 at 05:43:07PM +0200, Ard Biesheuvel wrote:
>>> Apart from the arm64/linux and EFI header data structures, there is nothing
>>> in the .head.text section that must reside at the beginning of the Image.
>>> So let's move it to the .init section where it belongs.
>>>
>>> Note that this involves some minor tweaking of the EFI header, primarily
>>> because the address of 'stext' no longer coincides with the start of the
>>> .text section. It also requires a couple of relocated symbol references
>>> to be slightly rewritten or their definition moved to the linker script.
>>
>> [...]
>>
>>>  ENTRY(stext)
>>>       bl      preserve_boot_args
>>>       bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
>>> @@ -223,12 +224,12 @@ ENTRY(stext)
>>>        * the TCR will have been set.
>>>        */
>>>       ldr     x27, 0f                         // address to jump to after
>>> -                                             // MMU has been enabled
>>> +     neg     x27, x27                        // MMU has been enabled
>>>       adr_l   lr, __enable_mmu                // return (PIC) address
>>>       b       __cpu_setup                     // initialise processor
>>>  ENDPROC(stext)
>>>       .align  3
>>> -0:   .quad   __mmap_switched - (_head - TEXT_OFFSET) + KIMAGE_VADDR
>>> +0:   .quad   (_text - TEXT_OFFSET) - __mmap_switched - KIMAGE_VADDR
>>
>> I'm struggling to understand why you need to change this. Furthermore,
>> it looks like the gas docs for expressions require that addition/subtraction
>> can only be performed on arguments that are in the same section, so
>> this feels pretty rickety.
>>
>> What's the problem you're solving here?
>>
> 
> The problem is that this function is no longer in the same section as
> _head aka _text, so the latter can only appear in the reloc expression
> as a positive term, and the remaining ones will all be combined into
> the addend.
> 

Alternatively, if you prefer, we could move the relocation processing
out of __mmap_switched() and run the code from the ID map, while
accessing the relocations via the virtual mapping. That way, all the
dodgy arithmetic (which actually doesn't look as dodgy anymore) is
confined to #ifdef CONFIG_RELOCATABLE sections. Something like

------8<--------
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 4203d5f257bc..f9eeb1936c38 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -222,13 +222,11 @@ ENTRY(stext)
 	 * On return, the CPU will be ready for the MMU to be turned on and
 	 * the TCR will have been set.
 	 */
-	ldr	x27, 0f				// address to jump to after
+	adr_l	x27, __relocate_kernel		// address to jump to after
 						// MMU has been enabled
 	adr_l	lr, __enable_mmu		// return (PIC) address
 	b	__cpu_setup			// initialise processor
 ENDPROC(stext)
-	.align	3
-0:	.quad	__mmap_switched - (_head - TEXT_OFFSET) + KIMAGE_VADDR
 
 /*
  * Preserve the arguments passed by the bootloader in x0 .. x3
@@ -437,44 +435,6 @@ __mmap_switched:
 	bl	__pi_memset
 	dsb	ishst				// Make zero page visible to PTW
 
-#ifdef CONFIG_RELOCATABLE
-
-	/*
-	 * Iterate over each entry in the relocation table, and apply the
-	 * relocations in place.
-	 */
-	adr_l	x8, __dynsym_start		// start of symbol table
-	adr_l	x9, __reloc_start		// start of reloc table
-	adr_l	x10, __reloc_end		// end of reloc table
-
-0:	cmp	x9, x10
-	b.hs	2f
-	ldp	x11, x12, [x9], #24
-	ldr	x13, [x9, #-8]
-	cmp	w12, #R_AARCH64_RELATIVE
-	b.ne	1f
-	add	x13, x13, x23			// relocate
-	str	x13, [x11, x23]
-	b	0b
-
-1:	cmp	w12, #R_AARCH64_ABS64
-	b.ne	0b
-	add	x12, x12, x12, lsl #1		// symtab offset: 24x top word
-	add	x12, x8, x12, lsr #(32 - 3)	// ... shifted into bottom word
-	ldrsh	w14, [x12, #6]			// Elf64_Sym::st_shndx
-	ldr	x15, [x12, #8]			// Elf64_Sym::st_value
-	cmp	w14, #-0xf			// SHN_ABS (0xfff1) ?
-	add	x14, x15, x23			// relocate
-	csel	x15, x14, x15, ne
-	add	x15, x13, x15
-	str	x15, [x11, x23]
-	b	0b
-
-2:	adr_l	x8, kimage_vaddr		// make relocated kimage_vaddr
-	dc	cvac, x8			// value visible to secondaries
-	dsb	sy				// with MMU off
-#endif
-
 	adr_l	sp, initial_sp, x4
 	mov	x4, sp
 	and	x4, x4, #~(THREAD_SIZE - 1)
@@ -795,7 +755,6 @@ __enable_mmu:
 	ic	iallu				// flush instructions fetched
 	dsb	nsh				// via old mapping
 	isb
-	add	x27, x27, x23			// relocated __mmap_switched
 #endif
 	br	x27
 ENDPROC(__enable_mmu)
@@ -808,3 +767,50 @@ __no_granule_support:
 	wfi
 	b 1b
 ENDPROC(__no_granule_support)
+
+__relocate_kernel:
+#ifdef CONFIG_RELOCATABLE
+	/*
+	 * Iterate over each entry in the relocation table, and apply the
+	 * relocations in place.
+	 */
+	ldr	w8, =__dynsym_start_offset	// offset to symbol table
+	ldr	w9, =__reloc_start_offset	// offset to reloc table
+	ldr	w10, =__reloc_end_offset	// offset to end of reloc table
+
+	ldr	x11, =KIMAGE_VADDR		// default virtual offset
+	add	x11, x11, x23			// actual virtual offset
+	add	x8, x8, x11			// __va(.dynsym)
+	add	x9, x9, x11			// __va(.reloc)
+	add	x10, x10, x11			// __va(.reloc) + sizeof(.reloc)
+
+0:	cmp	x9, x10
+	b.hs	2f
+	ldp	x11, x12, [x9], #24
+	ldr	x13, [x9, #-8]
+	cmp	w12, #R_AARCH64_RELATIVE
+	b.ne	1f
+	add	x13, x13, x23			// relocate
+	str	x13, [x11, x23]
+	b	0b
+
+1:	cmp	w12, #R_AARCH64_ABS64
+	b.ne	0b
+	add	x12, x12, x12, lsl #1		// symtab offset: 24x top word
+	add	x12, x8, x12, lsr #(32 - 3)	// ... shifted into bottom word
+	ldrsh	w14, [x12, #6]			// Elf64_Sym::st_shndx
+	ldr	x15, [x12, #8]			// Elf64_Sym::st_value
+	cmp	w14, #-0xf			// SHN_ABS (0xfff1) ?
+	add	x14, x15, x23			// relocate
+	csel	x15, x14, x15, ne
+	add	x15, x13, x15
+	str	x15, [x11, x23]
+	b	0b
+
+2:	ldr	x8, =kimage_vaddr		// make relocated kimage_vaddr
+	dc	cvac, x8			// value visible to secondaries
+	dsb	sy				// with MMU off
+#endif
+	ldr	x8, =__mmap_switched
+	br	x8
+ENDPROC(__relocate_kernel)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 5a1939a74ff3..e9278013d5d2 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -154,12 +154,12 @@ SECTIONS
 		*(.altinstr_replacement)
 	}
 	.rela : ALIGN(8) {
-		__reloc_start = .;
+		__reloc_start_offset = ABSOLUTE(. - KIMAGE_VADDR);
 		*(.rela .rela*)
-		__reloc_end = .;
+		__reloc_end_offset = ABSOLUTE(. - KIMAGE_VADDR);
 	}
 	.dynsym : ALIGN(8) {
-		__dynsym_start = .;
+		__dynsym_start_offset = ABSOLUTE(. - KIMAGE_VADDR);
 		*(.dynsym)
 	}
 	.dynstr : {

[PATCH v2 3/4] arm64: cover the .head.text section in the .text segment mapping

[Ard Biesheuvel]

Keeping .head.text out of the .text mapping buys us very little: its actual
payload is only 4 KB, most of which is padding, but the page alignment may
add up to 2 MB (in case of CONFIG_DEBUG_ALIGN_RODATA=y) of additional
padding to the uncompressed kernel Image.

Also, on 4 KB granule kernels, the 4 KB misalignment of .text forces us to
map the adjacent 56 KB of code without the PTE_CONT attribute, and since
this region contains things like the vector table and the GIC interrupt
handling entry point, this region is likely to benefit from the reduced TLB
pressure that results from PTE_CONT mappings.

So remove the alignment between the .head.text and .text sections, and use
the [_text, _etext) rather than the [_stext, _etext) interval for mapping
the .text segment.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
 arch/arm64/kernel/vmlinux.lds.S |  1 -
 arch/arm64/mm/mmu.c             | 10 +++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 5a1939a74ff3..61a1075b9c30 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -96,7 +96,6 @@ SECTIONS
 		_text = .;
 		HEAD_TEXT
 	}
-	ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
 	.text : {			/* Real text segment		*/
 		_stext = .;		/* Text and read-only data	*/
 			__exception_text_start = .;
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9be2065f8ddb..dbad533076d1 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -385,7 +385,7 @@ static void create_mapping_late(phys_addr_t phys, unsigned long virt,
 
 static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end)
 {
-	unsigned long kernel_start = __pa(_stext);
+	unsigned long kernel_start = __pa(_text);
 	unsigned long kernel_end = __pa(_etext);
 
 	/*
@@ -417,7 +417,7 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end
 				     early_pgtable_alloc);
 
 	/*
-	 * Map the linear alias of the [_stext, _etext) interval as
+	 * Map the linear alias of the [_text, _etext) interval as
 	 * read-only/non-executable. This makes the contents of the
 	 * region accessible to subsystems such as hibernate, but
 	 * protects it from inadvertent modification or execution.
@@ -449,8 +449,8 @@ void mark_rodata_ro(void)
 {
 	unsigned long section_size;
 
-	section_size = (unsigned long)__start_rodata - (unsigned long)_stext;
-	create_mapping_late(__pa(_stext), (unsigned long)_stext,
+	section_size = (unsigned long)__start_rodata - (unsigned long)_text;
+	create_mapping_late(__pa(_text), (unsigned long)_text,
 			    section_size, PAGE_KERNEL_ROX);
 	/*
 	 * mark .rodata as read only. Use _etext rather than __end_rodata to
@@ -499,7 +499,7 @@ static void __init map_kernel(pgd_t *pgd)
 {
 	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data;
 
-	map_kernel_segment(pgd, _stext, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text);
+	map_kernel_segment(pgd, _text, __start_rodata, PAGE_KERNEL_EXEC, &vmlinux_text);
 	map_kernel_segment(pgd, __start_rodata, _etext, PAGE_KERNEL, &vmlinux_rodata);
 	map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC,
 			   &vmlinux_init);
-- 
2.5.0

[PATCH v2 4/4] arm64: simplify kernel segment mapping granularity

[Ard Biesheuvel]

The mapping of the kernel consist of four segments, each of which is mapped
with different permission attributes and/or lifetimes. To optimize the TLB
and translation table footprint, we define various opaque constants in the
linker script that resolve to different aligment values depending on the
page size and whether CONFIG_DEBUG_ALIGN_RODATA is set.

Considering that
- a 4 KB granule kernel benefits from a 64 KB segment alignment (due to
  the fact that it allows the use of the contiguous bit),
- the minimum alignment of the .data segment is THREAD_SIZE already, not
  PAGE_SIZE (i.e., we already have padding between _data and the start of
  the .data payload in many cases),
- 2 MB is a suitable alignment value on all granule sizes, either for
  mapping directly (level 2 on 4 KB), or via the contiguous bit (level 3 on
  16 KB and 64 KB),
- anything beyond 2 MB exceeds the minimum alignment mandated by the boot
  protocol, and can only be mapped efficiently if the physical alignment
  happens to be the same,

we can simplify this by standardizing on 64 KB (or 2 MB) explicitly, i.e.,
regardless of granule size, all segments are aligned either to 64 KB, or to
2 MB if CONFIG_DEBUG_ALIGN_RODATA=y. This also means we can drop the Kconfig
dependency of CONFIG_DEBUG_ALIGN_RODATA on CONFIG_ARM64_4K_PAGES.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
 arch/arm64/Kconfig.debug        |  2 +-
 arch/arm64/kernel/vmlinux.lds.S | 25 ++++++++++++--------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug
index 7e76845a0434..710fde4ad0f0 100644
--- a/arch/arm64/Kconfig.debug
+++ b/arch/arm64/Kconfig.debug
@@ -59,7 +59,7 @@ config DEBUG_RODATA
 	  If in doubt, say Y
 
 config DEBUG_ALIGN_RODATA
-	depends on DEBUG_RODATA && ARM64_4K_PAGES
+	depends on DEBUG_RODATA
 	bool "Align linker sections up to SECTION_SIZE"
 	help
 	  If this option is enabled, sections that may potentially be marked as
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 61a1075b9c30..77d86c976abd 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -63,14 +63,19 @@ PECOFF_FILE_ALIGNMENT = 0x200;
 #endif
 
 #if defined(CONFIG_DEBUG_ALIGN_RODATA)
-#define ALIGN_DEBUG_RO			. = ALIGN(1<<SECTION_SHIFT);
-#define ALIGN_DEBUG_RO_MIN(min)		ALIGN_DEBUG_RO
-#elif defined(CONFIG_DEBUG_RODATA)
-#define ALIGN_DEBUG_RO			. = ALIGN(1<<PAGE_SHIFT);
-#define ALIGN_DEBUG_RO_MIN(min)		ALIGN_DEBUG_RO
+/*
+ *  4 KB granule:   1 level 2 entry
+ * 16 KB granule: 128 level 3 entries, with contiguous bit
+ * 64 KB granule:  32 level 3 entries, with contiguous bit
+ */
+#define SEGMENT_ALIGN			SZ_2M
 #else
-#define ALIGN_DEBUG_RO
-#define ALIGN_DEBUG_RO_MIN(min)		. = ALIGN(min);
+/*
+ *  4 KB granule:  16 level 3 entries, with contiguous bit
+ * 16 KB granule:   4 level 3 entries, without contiguous bit
+ * 64 KB granule:   1 level 3 entry
+ */
+#define SEGMENT_ALIGN			SZ_64K
 #endif
 
 SECTIONS
@@ -114,12 +119,12 @@ SECTIONS
 		*(.got)			/* Global offset table		*/
 	}
 
-	ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
+	. = ALIGN(SEGMENT_ALIGN);
 	RO_DATA(PAGE_SIZE)		/* everything from this point to */
 	EXCEPTION_TABLE(8)		/* _etext will be marked RO NX   */
 	NOTES
 
-	ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
+	. = ALIGN(SEGMENT_ALIGN);
 	_etext = .;			/* End of text and rodata section */
 	__init_begin = .;
 
@@ -168,7 +173,7 @@ SECTIONS
 		*(.hash)
 	}
 
-	. = ALIGN(PAGE_SIZE);
+	. = ALIGN(SEGMENT_ALIGN);
 	__init_end = .;
 
 	_data = .;
-- 
2.5.0