[PATCH 0/5] x86/boot: Early boot cleanups

[PATCH 0/5] x86/boot: Early boot cleanups

[Brian Gerst]

This series moves clearing BSS and copying boot parameters and the
command line to the early boot code.  Clearing BSS early prevents using
unitialized data making the code more robust.  Copying the boot
parameters early simplifies the code by not needing to pass around the
orignal source pointer and does not need special handling when
encryption is enabled.

Note: The memory encryption parts have only been compile tested due to
lack of access to capable hardware.

Brian Gerst (5):
  x86/sme: Clear decrypted BSS separately
  x86/boot/64: Clear BSS as early as possible
  x86/boot: Remove hardcoded boot_param constants
  x86/boot/64: Remove copy_bootdata() call
  x86/boot/64: Copy boot parameters and command line earlier

 arch/x86/boot/startup/map_kernel.c      | 10 ++--
 arch/x86/boot/startup/sme.c             |  6 ++-
 arch/x86/include/asm/mem_encrypt.h      | 14 ++----
 arch/x86/include/asm/setup.h            | 11 ++---
 arch/x86/kernel/asm-offsets.c           |  3 ++
 arch/x86/kernel/head64.c                | 61 ++---------------------
 arch/x86/kernel/head_32.S               |  4 +-
 arch/x86/kernel/head_64.S               | 47 ++++++++++++++----
 arch/x86/kernel/setup.c                 |  1 +
 arch/x86/kernel/smpboot.c               |  2 +-
 arch/x86/kernel/vmlinux.lds.S           |  2 +-
 arch/x86/mm/mem_encrypt_amd.c           | 64 ++-----------------------
 arch/x86/xen/enlighten_pv.c             |  4 +-
 arch/x86/xen/xen-head.S                 |  5 +-
 drivers/firmware/efi/libstub/x86-stub.c |  6 +--
 15 files changed, 76 insertions(+), 164 deletions(-)


base-commit: 1b57632fa6f9ad161c2633a6df4c65e973790218
-- 
2.55.0

[PATCH 1/5] x86/sme: Clear decrypted BSS separately

[Brian Gerst]

The decrypted BSS section needs to be cleared after it is remapped as
decrypted memory.  Separate it so that the normal BSS section can be
cleared earlier.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
---
 arch/x86/kernel/vmlinux.lds.S | 2 +-
 arch/x86/mm/mem_encrypt_amd.c | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 2438b89a4620..e64c797e06c7 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -365,9 +365,9 @@ SECTIONS
 		*(.bss..page_aligned)
 		. = ALIGN(PAGE_SIZE);
 		*(BSS_MAIN)
-		BSS_DECRYPTED
 		. = ALIGN(PAGE_SIZE);
 		__bss_stop = .;
+		BSS_DECRYPTED
 	}
 
 	/*
diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
index 2f8c32173972..d39e1e29bcb9 100644
--- a/arch/x86/mm/mem_encrypt_amd.c
+++ b/arch/x86/mm/mem_encrypt_amd.c
@@ -479,6 +479,9 @@ void __init sme_early_init(void)
 	if (!sme_me_mask)
 		return;
 
+	memset(__start_bss_decrypted, 0,
+	       (unsigned long) __end_bss_decrypted - (unsigned long) __start_bss_decrypted);
+
 	early_pmd_flags = __sme_set(early_pmd_flags);
 
 	__supported_pte_mask = __sme_set(__supported_pte_mask);
-- 
2.55.0

[PATCH 2/5] x86/boot/64: Clear BSS as early as possible

[Brian Gerst]

Currently, the BSS section is not cleared until x86_64_start_kernel().
Using unitialized BSS data before that point leads to difficult to debug
problems.  Fix this by moving clear_bss() to as early as possible.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
---
 arch/x86/include/asm/setup.h |  2 --
 arch/x86/kernel/head64.c     | 12 ------------
 arch/x86/kernel/head_64.S    | 20 ++++++++++++++++++++
 arch/x86/xen/enlighten_pv.c  |  2 --
 arch/x86/xen/xen-head.S      |  5 ++++-
 5 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 914eb32581c7..5534f4aaf73b 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -126,8 +126,6 @@ void *extend_brk(size_t size, size_t align);
 
 extern void probe_roms(void);
 
-void clear_bss(void);
-
 #ifdef __i386__
 
 asmlinkage void __init __noreturn i386_start_kernel(void);
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index fd28b53dbac5..9f19635c6390 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -172,16 +172,6 @@ void __init do_early_exception(struct pt_regs *regs, int trapnr)
 	early_fixup_exception(regs, trapnr);
 }
 
-/* Don't add a printk in there. printk relies on the PDA which is not initialized 
-   yet. */
-void __init clear_bss(void)
-{
-	memset(__bss_start, 0,
-	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
-	memset(__brk_base, 0,
-	       (unsigned long) __brk_limit - (unsigned long) __brk_base);
-}
-
 static unsigned long get_cmd_line_ptr(void)
 {
 	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
@@ -246,8 +236,6 @@ asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode
 		vmemmap_base		= __VMEMMAP_BASE_L5;
 	}
 
-	clear_bss();
-
 	/*
 	 * This needs to happen *before* kasan_early_init() because latter maps stuff
 	 * into that page.
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 7ed5520dd52e..15ef5ea52b9a 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -37,6 +37,8 @@
 	.code64
 SYM_CODE_START_NOALIGN(startup_64)
 	UNWIND_HINT_END_OF_STACK
+	cld
+
 	/*
 	 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
 	 * and someone has loaded an identity mapped page table
@@ -61,6 +63,8 @@ SYM_CODE_START_NOALIGN(startup_64)
 	/* Set up the stack for verify_cpu() */
 	leaq	__top_init_kernel_stack(%rip), %rsp
 
+	call	clear_bss
+
 	/*
 	 * Set up GSBASE.
 	 * Note that on SMP the boot CPU uses the init data section until
@@ -140,6 +144,22 @@ SYM_CODE_START_NOALIGN(startup_64)
 	jmp	*.Lcommon_startup_64(%rip)
 SYM_CODE_END(startup_64)
 
+SYM_FUNC_START(clear_bss)
+	xorl	%eax, %eax
+
+	leaq	__bss_start(%rip), %rdi
+	leaq	__bss_stop(%rip), %rcx
+	subq	%rdi, %rcx
+	rep stosb
+
+	leaq	__brk_base(%rip), %rdi
+	leaq	__brk_limit(%rip), %rcx
+	subq	%rdi, %rcx
+	rep stosb
+
+	RET
+SYM_FUNC_END(clear_bss)
+
 	__INITRODATA
 SYM_DATA_LOCAL(.Lcommon_startup_64, .quad common_startup_64)
 
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 2c64b388f616..6aa13d19c0a4 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1334,8 +1334,6 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
 	if (!si)
 		return;
 
-	clear_bss();
-
 	xen_start_info = si;
 
 	__text_gen_insn(&early_xen_iret_patch,
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 5dad6c51cdc3..9b56659246fe 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -31,6 +31,9 @@ SYM_CODE_START(startup_xen)
 
 	leaq	__top_init_kernel_stack(%rip), %rsp
 
+	movq	%rsi, %r15
+	call	clear_bss
+
 	/*
 	 * Set up GSBASE.
 	 * Note that, on SMP, the boot cpu uses init data section until
@@ -41,7 +44,7 @@ SYM_CODE_START(startup_xen)
 	xorl	%edx, %edx
 	wrmsr
 
-	mov	%rsi, %rdi
+	mov	%r15, %rdi
 	call xen_start_kernel
 SYM_CODE_END(startup_xen)
 	__FINIT
-- 
2.55.0

Re: [PATCH 2/5] x86/boot/64: Clear BSS as early as possible

[Nikolay Borisov]

On 7/24/26 06:02, Brian Gerst wrote:
> Currently, the BSS section is not cleared until x86_64_start_kernel().
> Using unitialized BSS data before that point leads to difficult to debug
> problems.  Fix this by moving clear_bss() to as early as possible.
> 
> Signed-off-by: Brian Gerst <brgerst@gmail.com>

<snip>


> diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
> index 7ed5520dd52e..15ef5ea52b9a 100644
> --- a/arch/x86/kernel/head_64.S
> +++ b/arch/x86/kernel/head_64.S
> @@ -37,6 +37,8 @@
>   	.code64
>   SYM_CODE_START_NOALIGN(startup_64)
>   	UNWIND_HINT_END_OF_STACK
> +	cld
> +
>   	/*
>   	 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
>   	 * and someone has loaded an identity mapped page table
> @@ -61,6 +63,8 @@ SYM_CODE_START_NOALIGN(startup_64)
>   	/* Set up the stack for verify_cpu() */
>   	leaq	__top_init_kernel_stack(%rip), %rsp
>   
> +	call	clear_bss
> +
>   	/*
>   	 * Set up GSBASE.
>   	 * Note that on SMP the boot CPU uses the init data section until
> @@ -140,6 +144,22 @@ SYM_CODE_START_NOALIGN(startup_64)
>   	jmp	*.Lcommon_startup_64(%rip)
>   SYM_CODE_END(startup_64)
>   
> +SYM_FUNC_START(clear_bss)
> +	xorl	%eax, %eax
> +
> +	leaq	__bss_start(%rip), %rdi
> +	leaq	__bss_stop(%rip), %rcx
> +	subq	%rdi, %rcx
> +	rep stosb

Why not stosq (with appropriate adjustment to xor %rax, %rax as well)? 
The length is guaranteed to be multiple of page_size ergo 8 byte as well ?

> +
> +	leaq	__brk_base(%rip), %rdi
> +	leaq	__brk_limit(%rip), %rcx
> +	subq	%rdi, %rcx
> +	rep stosb
> +
> +	RET
> +SYM_FUNC_END(clear_bss)
> +
>   	__INITRODATA
>   SYM_DATA_LOCAL(.Lcommon_startup_64, .quad common_startup_64)
>   
> diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
> index 2c64b388f616..6aa13d19c0a4 100644
> --- a/arch/x86/xen/enlighten_pv.c
> +++ b/arch/x86/xen/enlighten_pv.c
> @@ -1334,8 +1334,6 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
>   	if (!si)
>   		return;
>   
> -	clear_bss();
> -
>   	xen_start_info = si;
>   
>   	__text_gen_insn(&early_xen_iret_patch,
> diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
> index 5dad6c51cdc3..9b56659246fe 100644
> --- a/arch/x86/xen/xen-head.S
> +++ b/arch/x86/xen/xen-head.S
> @@ -31,6 +31,9 @@ SYM_CODE_START(startup_xen)
>   
>   	leaq	__top_init_kernel_stack(%rip), %rsp
>   
> +	movq	%rsi, %r15

Why is this change necessary, rsi is ont used in clear_bss ?
> +	call	clear_bss
> +
>   	/*
>   	 * Set up GSBASE.
>   	 * Note that, on SMP, the boot cpu uses init data section until
> @@ -41,7 +44,7 @@ SYM_CODE_START(startup_xen)
>   	xorl	%edx, %edx
>   	wrmsr
>   
> -	mov	%rsi, %rdi
> +	mov	%r15, %rdi
>   	call xen_start_kernel
>   SYM_CODE_END(startup_xen)
>   	__FINIT

Re: [PATCH 2/5] x86/boot/64: Clear BSS as early as possible

[Brian Gerst]

On Fri, Jul 24, 2026 at 7:04 AM Nikolay Borisov <nik.borisov@suse.com> wrote:
>
>
>
> On 7/24/26 06:02, Brian Gerst wrote:
> > Currently, the BSS section is not cleared until x86_64_start_kernel().
> > Using unitialized BSS data before that point leads to difficult to debug
> > problems.  Fix this by moving clear_bss() to as early as possible.
> >
> > Signed-off-by: Brian Gerst <brgerst@gmail.com>
>
> <snip>
>
>
> > diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
> > index 7ed5520dd52e..15ef5ea52b9a 100644
> > --- a/arch/x86/kernel/head_64.S
> > +++ b/arch/x86/kernel/head_64.S
> > @@ -37,6 +37,8 @@
> >       .code64
> >   SYM_CODE_START_NOALIGN(startup_64)
> >       UNWIND_HINT_END_OF_STACK
> > +     cld
> > +
> >       /*
> >        * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
> >        * and someone has loaded an identity mapped page table
> > @@ -61,6 +63,8 @@ SYM_CODE_START_NOALIGN(startup_64)
> >       /* Set up the stack for verify_cpu() */
> >       leaq    __top_init_kernel_stack(%rip), %rsp
> >
> > +     call    clear_bss
> > +
> >       /*
> >        * Set up GSBASE.
> >        * Note that on SMP the boot CPU uses the init data section until
> > @@ -140,6 +144,22 @@ SYM_CODE_START_NOALIGN(startup_64)
> >       jmp     *.Lcommon_startup_64(%rip)
> >   SYM_CODE_END(startup_64)
> >
> > +SYM_FUNC_START(clear_bss)
> > +     xorl    %eax, %eax
> > +
> > +     leaq    __bss_start(%rip), %rdi
> > +     leaq    __bss_stop(%rip), %rcx
> > +     subq    %rdi, %rcx
> > +     rep stosb
>
> Why not stosq (with appropriate adjustment to xor %rax, %rax as well)?
> The length is guaranteed to be multiple of page_size ergo 8 byte as well ?

I used code similar to what GCC compiled the original memset() to.
Using a larger width store would require adding a shift instruction
for RCX.  This is one-time boot code so performance isn't critical,
and some modern processors favor REP STOSB anyways (X86_FEATURE_ERMS).

> > +
> > +     leaq    __brk_base(%rip), %rdi
> > +     leaq    __brk_limit(%rip), %rcx
> > +     subq    %rdi, %rcx
> > +     rep stosb
> > +
> > +     RET
> > +SYM_FUNC_END(clear_bss)
> > +
> >       __INITRODATA
> >   SYM_DATA_LOCAL(.Lcommon_startup_64, .quad common_startup_64)
> >
> > diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
> > index 2c64b388f616..6aa13d19c0a4 100644
> > --- a/arch/x86/xen/enlighten_pv.c
> > +++ b/arch/x86/xen/enlighten_pv.c
> > @@ -1334,8 +1334,6 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
> >       if (!si)
> >               return;
> >
> > -     clear_bss();
> > -
> >       xen_start_info = si;
> >
> >       __text_gen_insn(&early_xen_iret_patch,
> > diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
> > index 5dad6c51cdc3..9b56659246fe 100644
> > --- a/arch/x86/xen/xen-head.S
> > +++ b/arch/x86/xen/xen-head.S
> > @@ -31,6 +31,9 @@ SYM_CODE_START(startup_xen)
> >
> >       leaq    __top_init_kernel_stack(%rip), %rsp
> >
> > +     movq    %rsi, %r15
>
> Why is this change necessary, rsi is ont used in clear_bss ?

It contains the start_info parameter that is passed to
xen_start_kernel(), which needs to be preserved over the call to
clear_bss().

> > +     call    clear_bss
> > +
> >       /*
> >        * Set up GSBASE.
> >        * Note that, on SMP, the boot cpu uses init data section until
> > @@ -41,7 +44,7 @@ SYM_CODE_START(startup_xen)
> >       xorl    %edx, %edx
> >       wrmsr
> >
> > -     mov     %rsi, %rdi
> > +     mov     %r15, %rdi
> >       call xen_start_kernel
> >   SYM_CODE_END(startup_xen)
> >       __FINIT
>

[PATCH 3/5] x86/boot: Remove hardcoded boot_param constants

[Brian Gerst]

Use generated constants or sizeof() instead of hardcoded values.

No functional change.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
---
 arch/x86/include/asm/setup.h            | 3 ---
 arch/x86/kernel/asm-offsets.c           | 2 ++
 arch/x86/kernel/head_32.S               | 4 ++--
 drivers/firmware/efi/libstub/x86-stub.c | 6 +++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 5534f4aaf73b..d23f4272d930 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -21,11 +21,8 @@
 
 #endif /* __i386__ */
 
-#define PARAM_SIZE 4096		/* sizeof(struct boot_params) */
-
 #define OLD_CL_MAGIC		0xA33F
 #define OLD_CL_ADDRESS		0x020	/* Relative to real mode data */
-#define NEW_CL_POINTER		0x228	/* Relative to real mode data */
 
 #ifndef __ASSEMBLER__
 #include <linux/cache.h>
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 081816888f7a..0c49bb50188d 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -105,6 +105,8 @@ static void __used common(void)
 	OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment);
 	OFFSET(BP_init_size, boot_params, hdr.init_size);
 	OFFSET(BP_pref_address, boot_params, hdr.pref_address);
+	OFFSET(BP_cmd_line_ptr, boot_params, hdr.cmd_line_ptr);
+	DEFINE(SIZEOF_boot_params, sizeof(struct boot_params));
 
 	BLANK();
 	DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 5171cb746444..1f66ccc36fe8 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -96,10 +96,10 @@ SYM_CODE_START(startup_32)
  * page tables.
  */
 	movl $pa(boot_params),%edi
-	movl $(PARAM_SIZE/4),%ecx
+	movl $(SIZEOF_boot_params/4),%ecx
 	cld
 	rep movsl
-	movl pa(boot_params) + NEW_CL_POINTER,%esi
+	movl pa(boot_params) + BP_cmd_line_ptr,%esi
 	andl %esi,%esi
 	jz 1f			# No command line
 	movl $pa(boot_command_line),%edi
diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c
index cef32e2c82d8..c3522d417344 100644
--- a/drivers/firmware/efi/libstub/x86-stub.c
+++ b/drivers/firmware/efi/libstub/x86-stub.c
@@ -520,11 +520,11 @@ static efi_status_t efi_allocate_bootparams(efi_handle_t handle,
 		return status;
 	}
 
-	status = efi_allocate_pages(PARAM_SIZE, &alloc, ULONG_MAX);
+	status = efi_allocate_pages(sizeof(struct boot_params), &alloc, ULONG_MAX);
 	if (status != EFI_SUCCESS)
 		return status;
 
-	boot_params = memset((void *)alloc, 0x0, PARAM_SIZE);
+	boot_params = memset((void *)alloc, 0x0, sizeof(struct boot_params));
 	hdr	    = &boot_params->hdr;
 
 	/* Assign the setup_header fields that the kernel actually cares about */
@@ -537,7 +537,7 @@ static efi_status_t efi_allocate_bootparams(efi_handle_t handle,
 	/* Convert unicode cmdline to ascii */
 	cmdline_ptr = efi_convert_cmdline(image);
 	if (!cmdline_ptr) {
-		efi_free(PARAM_SIZE, alloc);
+		efi_free(sizeof(struct boot_params), alloc);
 		return EFI_OUT_OF_RESOURCES;
 	}
 
-- 
2.55.0

[PATCH 4/5] x86/boot/64: Remove copy_bootdata() call

[Brian Gerst]

On a normal boot, copy_bootdata() is first called in
x86_64_start_kernel().  The second call in x86_64_start_reservations()
is effectively a no-op.

Xen PV directly initializes boot_params, so it does not need to call
copy_bootdata() at all.

Remove the unnecessary call to copy_bootdata() from
x86_64_start_reservations().

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
---
 arch/x86/include/asm/setup.h | 2 +-
 arch/x86/kernel/head64.c     | 8 ++------
 arch/x86/xen/enlighten_pv.c  | 2 +-
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index d23f4272d930..18a4d3826af7 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -130,7 +130,7 @@ void __init mk_early_pgtbl_32(void);
 
 #else
 asmlinkage void __init __noreturn x86_64_start_kernel(char *real_mode);
-asmlinkage void __init __noreturn x86_64_start_reservations(char *real_mode_data);
+asmlinkage void __init __noreturn x86_64_start_reservations(void);
 
 #endif /* __i386__ */
 #endif /* _SETUP */
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 9f19635c6390..d895376e9346 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -276,15 +276,11 @@ asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode
 	/* set init_top_pgt kernel high mapping*/
 	init_top_pgt[511] = early_top_pgt[511];
 
-	x86_64_start_reservations(real_mode_data);
+	x86_64_start_reservations();
 }
 
-void __init __noreturn x86_64_start_reservations(char *real_mode_data)
+void __init __noreturn x86_64_start_reservations(void)
 {
-	/* version is always not zero if it is copied */
-	if (!boot_params.hdr.version)
-		copy_bootdata(__va(real_mode_data));
-
 	x86_early_init_platform_quirks();
 
 	switch (boot_params.hdr.hardware_subarch) {
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 6aa13d19c0a4..e6e2666da5a1 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1556,7 +1556,7 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
 
 	/* Start the world */
 	cr4_init_shadow(); /* 32b kernel does this in i386_start_kernel() */
-	x86_64_start_reservations((char *)__pa_symbol(&boot_params));
+	x86_64_start_reservations();
 }
 
 static int xen_cpu_up_prepare_pv(unsigned int cpu)
-- 
2.55.0

[PATCH 5/5] x86/boot/64: Copy boot parameters and command line earlier

[Brian Gerst]

Move the copy of the boot parameters and command line to the early
setup code.  This eliminates the need to pass the original pointer to
several other functions.

By copying the data into kernel memory before encryption is enabled,
the original location no longer needs to be remapped as decrypted.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
---
 arch/x86/boot/startup/map_kernel.c | 10 ++---
 arch/x86/boot/startup/sme.c        |  6 ++-
 arch/x86/include/asm/mem_encrypt.h | 14 ++-----
 arch/x86/include/asm/setup.h       |  4 +-
 arch/x86/kernel/asm-offsets.c      |  1 +
 arch/x86/kernel/head64.c           | 41 +-------------------
 arch/x86/kernel/head_64.S          | 27 ++++++++-----
 arch/x86/kernel/setup.c            |  1 +
 arch/x86/kernel/smpboot.c          |  2 +-
 arch/x86/mm/mem_encrypt_amd.c      | 61 ------------------------------
 10 files changed, 37 insertions(+), 130 deletions(-)

diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
index 83ba98d61572..874c91f1b57b 100644
--- a/arch/x86/boot/startup/map_kernel.c
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -30,15 +30,14 @@ static inline bool check_la57_support(void)
 	return true;
 }
 
-static unsigned long __init sme_postprocess_startup(struct boot_params *bp,
-						    pmdval_t *pmd,
+static unsigned long __init sme_postprocess_startup(pmdval_t *pmd,
 						    unsigned long p2v_offset)
 {
 	unsigned long paddr, paddr_end;
 	int i;
 
 	/* Encrypt the kernel and related (if SME is active) */
-	sme_encrypt_kernel(bp);
+	sme_encrypt_kernel();
 
 	/*
 	 * Clear the memory encryption mask from the .bss..decrypted section.
@@ -84,8 +83,7 @@ static unsigned long __init sme_postprocess_startup(struct boot_params *bp,
  * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
  * subtracting p2v_offset from the RIP-relative address.
  */
-unsigned long __init __startup_64(unsigned long p2v_offset,
-				  struct boot_params *bp)
+unsigned long __init __startup_64(unsigned long p2v_offset)
 {
 	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
 	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
@@ -213,5 +211,5 @@ unsigned long __init __startup_64(unsigned long p2v_offset,
 	for (; i < PTRS_PER_PMD; i++)
 		pmd[i] &= ~_PAGE_PRESENT;
 
-	return sme_postprocess_startup(bp, pmd, p2v_offset);
+	return sme_postprocess_startup(pmd, p2v_offset);
 }
diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index c07a2c381ed1..2a6208254fa3 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -283,8 +283,9 @@ static unsigned long __init sme_pgtable_calc(unsigned long len)
 	return entries + tables;
 }
 
-void __init sme_encrypt_kernel(struct boot_params *bp)
+void __init sme_encrypt_kernel(void)
 {
+	struct boot_params *bp = rip_rel_ptr(&boot_params);
 	unsigned long workarea_start, workarea_end, workarea_len;
 	unsigned long execute_start, execute_end, execute_len;
 	unsigned long kernel_start, kernel_end, kernel_len;
@@ -487,8 +488,9 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
 	native_write_cr3(__native_read_cr3());
 }
 
-void __init sme_enable(struct boot_params *bp)
+void __init sme_enable(void)
 {
+	struct boot_params *bp = rip_rel_ptr(&boot_params);
 	unsigned int eax, ebx, ecx, edx;
 	unsigned long feature_mask;
 	unsigned long me_mask;
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index ea6494628cb0..72b63b4a0dcb 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -42,13 +42,10 @@ void __init sme_early_encrypt(resource_size_t paddr,
 void __init sme_early_decrypt(resource_size_t paddr,
 			      unsigned long size);
 
-void __init sme_map_bootdata(char *real_mode_data);
-void __init sme_unmap_bootdata(char *real_mode_data);
-
 void __init sme_early_init(void);
 
-void sme_encrypt_kernel(struct boot_params *bp);
-void sme_enable(struct boot_params *bp);
+void sme_encrypt_kernel(void);
+void sme_enable(void);
 
 int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size);
 int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
@@ -76,13 +73,10 @@ static inline void __init sme_early_encrypt(resource_size_t paddr,
 static inline void __init sme_early_decrypt(resource_size_t paddr,
 					    unsigned long size) { }
 
-static inline void __init sme_map_bootdata(char *real_mode_data) { }
-static inline void __init sme_unmap_bootdata(char *real_mode_data) { }
-
 static inline void __init sme_early_init(void) { }
 
-static inline void sme_encrypt_kernel(struct boot_params *bp) { }
-static inline void sme_enable(struct boot_params *bp) { }
+static inline void sme_encrypt_kernel(void) { }
+static inline void sme_enable(void) { }
 
 static inline void sev_es_init_vc_handling(void) { }
 
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 18a4d3826af7..0e867910b29b 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -47,7 +47,7 @@ extern unsigned long acpi_realmode_flags;
 
 extern void reserve_standard_io_resources(void);
 extern void i386_reserve_resources(void);
-extern unsigned long __startup_64(unsigned long p2v_offset, struct boot_params *bp);
+extern unsigned long __startup_64(unsigned long p2v_offset);
 extern void startup_64_setup_gdt_idt(void);
 extern void startup_64_load_idt(void *vc_handler);
 extern void __pi_startup_64_load_idt(void *vc_handler);
@@ -129,7 +129,7 @@ asmlinkage void __init __noreturn i386_start_kernel(void);
 void __init mk_early_pgtbl_32(void);
 
 #else
-asmlinkage void __init __noreturn x86_64_start_kernel(char *real_mode);
+asmlinkage void __init __noreturn x86_64_start_kernel(void);
 asmlinkage void __init __noreturn x86_64_start_reservations(void);
 
 #endif /* __i386__ */
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 0c49bb50188d..c7a2c210aaac 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -106,6 +106,7 @@ static void __used common(void)
 	OFFSET(BP_init_size, boot_params, hdr.init_size);
 	OFFSET(BP_pref_address, boot_params, hdr.pref_address);
 	OFFSET(BP_cmd_line_ptr, boot_params, hdr.cmd_line_ptr);
+	OFFSET(BP_ext_cmd_line_ptr, boot_params, ext_cmd_line_ptr);
 	DEFINE(SIZEOF_boot_params, sizeof(struct boot_params));
 
 	BLANK();
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index d895376e9346..b4d5c7b7dc58 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -172,44 +172,7 @@ void __init do_early_exception(struct pt_regs *regs, int trapnr)
 	early_fixup_exception(regs, trapnr);
 }
 
-static unsigned long get_cmd_line_ptr(void)
-{
-	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
-
-	cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;
-
-	return cmd_line_ptr;
-}
-
-static void __init copy_bootdata(char *real_mode_data)
-{
-	char * command_line;
-	unsigned long cmd_line_ptr;
-
-	/*
-	 * If SME is active, this will create decrypted mappings of the
-	 * boot data in advance of the copy operations.
-	 */
-	sme_map_bootdata(real_mode_data);
-
-	memcpy(&boot_params, real_mode_data, sizeof(boot_params));
-	sanitize_boot_params(&boot_params);
-	cmd_line_ptr = get_cmd_line_ptr();
-	if (cmd_line_ptr) {
-		command_line = __va(cmd_line_ptr);
-		memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
-	}
-
-	/*
-	 * The old boot data is no longer needed and won't be reserved,
-	 * freeing up that memory for use by the system. If SME is active,
-	 * we need to remove the mappings that were created so that the
-	 * memory doesn't remain mapped as decrypted.
-	 */
-	sme_unmap_bootdata(real_mode_data);
-}
-
-asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode_data)
+asmlinkage __visible void __init __noreturn x86_64_start_kernel(void)
 {
 	/*
 	 * Build-time sanity checks on the kernel image and module
@@ -266,7 +229,7 @@ asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode
 	/* Needed before cc_platform_has() can be used for TDX */
 	tdx_early_init();
 
-	copy_bootdata(__va(real_mode_data));
+	sanitize_boot_params(&boot_params);
 
 	/*
 	 * Load microcode early on BSP.
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 15ef5ea52b9a..58b5a2654c2b 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -27,6 +27,7 @@
 #include <asm/fixmap.h>
 #include <asm/smp.h>
 #include <asm/thread_info.h>
+#include <asm/setup.h>
 
 /*
  * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
@@ -65,6 +66,22 @@ SYM_CODE_START_NOALIGN(startup_64)
 
 	call	clear_bss
 
+	/* Copy the boot parameters and command line into kernel memory. */
+	movq	%r15, %rsi
+	leaq	boot_params(%rip), %rdi
+	movl	$(SIZEOF_boot_params / 4), %ecx
+	rep movsl
+
+	movl	boot_params + BP_cmd_line_ptr(%rip), %esi
+	movl	boot_params + BP_ext_cmd_line_ptr(%rip), %eax
+	shlq	$32, %rax
+	orq	%rax, %rsi
+	jz	.Lno_command_line
+	leaq	boot_command_line(%rip), %rdi
+	movl	$(COMMAND_LINE_SIZE / 4), %ecx
+	rep movsl
+.Lno_command_line:
+
 	/*
 	 * Set up GSBASE.
 	 * Note that on SMP the boot CPU uses the init data section until
@@ -92,9 +109,8 @@ SYM_CODE_START_NOALIGN(startup_64)
 	 * Activate SEV/SME memory encryption if supported/enabled. This needs to
 	 * be done now, since this also includes setup of the SEV-SNP CPUID table,
 	 * which needs to be done before any CPUID instructions are executed in
-	 * subsequent code. Pass the boot_params pointer as the first argument.
+	 * subsequent code.
 	 */
-	movq	%r15, %rdi
 	call	__pi_sme_enable
 #endif
 
@@ -114,7 +130,6 @@ SYM_CODE_START_NOALIGN(startup_64)
 	 * is active) to be added to the initial pgdir entry that will be
 	 * programmed into CR3.
 	 */
-	movq	%r15, %rsi
 	call	__pi___startup_64
 
 	/* Form the CR3 value being sure to include the CR3 modifier */
@@ -195,9 +210,6 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 	UNWIND_HINT_END_OF_STACK
 	ANNOTATE_NOENDBR
 
-	/* Clear %R15 which holds the boot_params pointer on the boot CPU */
-	xorl	%r15d, %r15d
-
 	/* Derive the runtime physical address of init_top_pgt[] */
 	movq	phys_base(%rip), %rax
 	addq	$(init_top_pgt - __START_KERNEL_map), %rax
@@ -429,9 +441,6 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
 	pushq $0
 	popfq
 
-	/* Pass the boot_params pointer as first argument */
-	movq	%r15, %rdi
-
 .Ljump_to_C_code:
 	xorl	%ebp, %ebp	# clear frame pointer
 	ANNOTATE_RETPOLINE_SAFE
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 46882ce79c3a..bd9d42767c8c 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -77,6 +77,7 @@ unsigned long _brk_start = (unsigned long)__brk_base;
 unsigned long _brk_end   = (unsigned long)__brk_base;
 
 struct boot_params boot_params;
+SYM_PIC_ALIAS(boot_params);
 
 /*
  * These are the four main kernel memory regions, we put them into
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index ba01a9e919b7..06287eed5cbc 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -226,7 +226,7 @@ static void ap_calibrate_delay(void)
 /*
  * Activate a secondary processor.
  */
-static void notrace __noendbr start_secondary(void *unused)
+static void notrace __noendbr start_secondary(void)
 {
 	/*
 	 * Don't put *anything* except direct CPU state initialization
diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
index d39e1e29bcb9..877d4d85c949 100644
--- a/arch/x86/mm/mem_encrypt_amd.c
+++ b/arch/x86/mm/mem_encrypt_amd.c
@@ -153,67 +153,6 @@ void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
 	__sme_early_enc_dec(paddr, size, false);
 }
 
-static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
-					     bool map)
-{
-	unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
-	pmdval_t pmd_flags, pmd;
-
-	/* Use early_pmd_flags but remove the encryption mask */
-	pmd_flags = __sme_clr(early_pmd_flags);
-
-	do {
-		pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
-		__early_make_pgtable((unsigned long)vaddr, pmd);
-
-		vaddr += PMD_SIZE;
-		paddr += PMD_SIZE;
-		size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
-	} while (size);
-
-	flush_tlb_local();
-}
-
-void __init sme_unmap_bootdata(char *real_mode_data)
-{
-	struct boot_params *boot_data;
-	unsigned long cmdline_paddr;
-
-	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
-		return;
-
-	/* Get the command line address before unmapping the real_mode_data */
-	boot_data = (struct boot_params *)real_mode_data;
-	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
-
-	__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
-
-	if (!cmdline_paddr)
-		return;
-
-	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
-}
-
-void __init sme_map_bootdata(char *real_mode_data)
-{
-	struct boot_params *boot_data;
-	unsigned long cmdline_paddr;
-
-	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
-		return;
-
-	__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
-
-	/* Get the command line address after mapping the real_mode_data */
-	boot_data = (struct boot_params *)real_mode_data;
-	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
-
-	if (!cmdline_paddr)
-		return;
-
-	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
-}
-
 static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot)
 {
 	unsigned long pfn = 0;
-- 
2.55.0