[PATCH 5/5] x86/boot/64: Copy boot parameters and command line earlier

[Brian Gerst <brgerst@gmail.com>]

Move the copy of the boot parameters and command line to the early
setup code.  This eliminates the need to pass the original pointer to
several other functions.

By copying the data into kernel memory before encryption is enabled,
the original location no longer needs to be remapped as decrypted.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
---
 arch/x86/boot/startup/map_kernel.c | 10 ++---
 arch/x86/boot/startup/sme.c        |  6 ++-
 arch/x86/include/asm/mem_encrypt.h | 14 ++-----
 arch/x86/include/asm/setup.h       |  4 +-
 arch/x86/kernel/asm-offsets.c      |  1 +
 arch/x86/kernel/head64.c           | 41 +-------------------
 arch/x86/kernel/head_64.S          | 27 ++++++++-----
 arch/x86/kernel/setup.c            |  1 +
 arch/x86/kernel/smpboot.c          |  2 +-
 arch/x86/mm/mem_encrypt_amd.c      | 61 ------------------------------
 10 files changed, 37 insertions(+), 130 deletions(-)

diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
index 83ba98d61572..874c91f1b57b 100644
--- a/arch/x86/boot/startup/map_kernel.c
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -30,15 +30,14 @@ static inline bool check_la57_support(void)
 	return true;
 }
 
-static unsigned long __init sme_postprocess_startup(struct boot_params *bp,
-						    pmdval_t *pmd,
+static unsigned long __init sme_postprocess_startup(pmdval_t *pmd,
 						    unsigned long p2v_offset)
 {
 	unsigned long paddr, paddr_end;
 	int i;
 
 	/* Encrypt the kernel and related (if SME is active) */
-	sme_encrypt_kernel(bp);
+	sme_encrypt_kernel();
 
 	/*
 	 * Clear the memory encryption mask from the .bss..decrypted section.
@@ -84,8 +83,7 @@ static unsigned long __init sme_postprocess_startup(struct boot_params *bp,
  * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
  * subtracting p2v_offset from the RIP-relative address.
  */
-unsigned long __init __startup_64(unsigned long p2v_offset,
-				  struct boot_params *bp)
+unsigned long __init __startup_64(unsigned long p2v_offset)
 {
 	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
 	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
@@ -213,5 +211,5 @@ unsigned long __init __startup_64(unsigned long p2v_offset,
 	for (; i < PTRS_PER_PMD; i++)
 		pmd[i] &= ~_PAGE_PRESENT;
 
-	return sme_postprocess_startup(bp, pmd, p2v_offset);
+	return sme_postprocess_startup(pmd, p2v_offset);
 }
diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index c07a2c381ed1..2a6208254fa3 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -283,8 +283,9 @@ static unsigned long __init sme_pgtable_calc(unsigned long len)
 	return entries + tables;
 }
 
-void __init sme_encrypt_kernel(struct boot_params *bp)
+void __init sme_encrypt_kernel(void)
 {
+	struct boot_params *bp = rip_rel_ptr(&boot_params);
 	unsigned long workarea_start, workarea_end, workarea_len;
 	unsigned long execute_start, execute_end, execute_len;
 	unsigned long kernel_start, kernel_end, kernel_len;
@@ -487,8 +488,9 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
 	native_write_cr3(__native_read_cr3());
 }
 
-void __init sme_enable(struct boot_params *bp)
+void __init sme_enable(void)
 {
+	struct boot_params *bp = rip_rel_ptr(&boot_params);
 	unsigned int eax, ebx, ecx, edx;
 	unsigned long feature_mask;
 	unsigned long me_mask;
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index ea6494628cb0..72b63b4a0dcb 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -42,13 +42,10 @@ void __init sme_early_encrypt(resource_size_t paddr,
 void __init sme_early_decrypt(resource_size_t paddr,
 			      unsigned long size);
 
-void __init sme_map_bootdata(char *real_mode_data);
-void __init sme_unmap_bootdata(char *real_mode_data);
-
 void __init sme_early_init(void);
 
-void sme_encrypt_kernel(struct boot_params *bp);
-void sme_enable(struct boot_params *bp);
+void sme_encrypt_kernel(void);
+void sme_enable(void);
 
 int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size);
 int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
@@ -76,13 +73,10 @@ static inline void __init sme_early_encrypt(resource_size_t paddr,
 static inline void __init sme_early_decrypt(resource_size_t paddr,
 					    unsigned long size) { }
 
-static inline void __init sme_map_bootdata(char *real_mode_data) { }
-static inline void __init sme_unmap_bootdata(char *real_mode_data) { }
-
 static inline void __init sme_early_init(void) { }
 
-static inline void sme_encrypt_kernel(struct boot_params *bp) { }
-static inline void sme_enable(struct boot_params *bp) { }
+static inline void sme_encrypt_kernel(void) { }
+static inline void sme_enable(void) { }
 
 static inline void sev_es_init_vc_handling(void) { }
 
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 18a4d3826af7..0e867910b29b 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -47,7 +47,7 @@ extern unsigned long acpi_realmode_flags;
 
 extern void reserve_standard_io_resources(void);
 extern void i386_reserve_resources(void);
-extern unsigned long __startup_64(unsigned long p2v_offset, struct boot_params *bp);
+extern unsigned long __startup_64(unsigned long p2v_offset);
 extern void startup_64_setup_gdt_idt(void);
 extern void startup_64_load_idt(void *vc_handler);
 extern void __pi_startup_64_load_idt(void *vc_handler);
@@ -129,7 +129,7 @@ asmlinkage void __init __noreturn i386_start_kernel(void);
 void __init mk_early_pgtbl_32(void);
 
 #else
-asmlinkage void __init __noreturn x86_64_start_kernel(char *real_mode);
+asmlinkage void __init __noreturn x86_64_start_kernel(void);
 asmlinkage void __init __noreturn x86_64_start_reservations(void);
 
 #endif /* __i386__ */
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 0c49bb50188d..c7a2c210aaac 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -106,6 +106,7 @@ static void __used common(void)
 	OFFSET(BP_init_size, boot_params, hdr.init_size);
 	OFFSET(BP_pref_address, boot_params, hdr.pref_address);
 	OFFSET(BP_cmd_line_ptr, boot_params, hdr.cmd_line_ptr);
+	OFFSET(BP_ext_cmd_line_ptr, boot_params, ext_cmd_line_ptr);
 	DEFINE(SIZEOF_boot_params, sizeof(struct boot_params));
 
 	BLANK();
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index d895376e9346..b4d5c7b7dc58 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -172,44 +172,7 @@ void __init do_early_exception(struct pt_regs *regs, int trapnr)
 	early_fixup_exception(regs, trapnr);
 }
 
-static unsigned long get_cmd_line_ptr(void)
-{
-	unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
-
-	cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;
-
-	return cmd_line_ptr;
-}
-
-static void __init copy_bootdata(char *real_mode_data)
-{
-	char * command_line;
-	unsigned long cmd_line_ptr;
-
-	/*
-	 * If SME is active, this will create decrypted mappings of the
-	 * boot data in advance of the copy operations.
-	 */
-	sme_map_bootdata(real_mode_data);
-
-	memcpy(&boot_params, real_mode_data, sizeof(boot_params));
-	sanitize_boot_params(&boot_params);
-	cmd_line_ptr = get_cmd_line_ptr();
-	if (cmd_line_ptr) {
-		command_line = __va(cmd_line_ptr);
-		memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
-	}
-
-	/*
-	 * The old boot data is no longer needed and won't be reserved,
-	 * freeing up that memory for use by the system. If SME is active,
-	 * we need to remove the mappings that were created so that the
-	 * memory doesn't remain mapped as decrypted.
-	 */
-	sme_unmap_bootdata(real_mode_data);
-}
-
-asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode_data)
+asmlinkage __visible void __init __noreturn x86_64_start_kernel(void)
 {
 	/*
 	 * Build-time sanity checks on the kernel image and module
@@ -266,7 +229,7 @@ asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode
 	/* Needed before cc_platform_has() can be used for TDX */
 	tdx_early_init();
 
-	copy_bootdata(__va(real_mode_data));
+	sanitize_boot_params(&boot_params);
 
 	/*
 	 * Load microcode early on BSP.
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 15ef5ea52b9a..58b5a2654c2b 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -27,6 +27,7 @@
 #include <asm/fixmap.h>
 #include <asm/smp.h>
 #include <asm/thread_info.h>
+#include <asm/setup.h>
 
 /*
  * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
@@ -65,6 +66,22 @@ SYM_CODE_START_NOALIGN(startup_64)
 
 	call	clear_bss
 
+	/* Copy the boot parameters and command line into kernel memory. */
+	movq	%r15, %rsi
+	leaq	boot_params(%rip), %rdi
+	movl	$(SIZEOF_boot_params / 4), %ecx
+	rep movsl
+
+	movl	boot_params + BP_cmd_line_ptr(%rip), %esi
+	movl	boot_params + BP_ext_cmd_line_ptr(%rip), %eax
+	shlq	$32, %rax
+	orq	%rax, %rsi
+	jz	.Lno_command_line
+	leaq	boot_command_line(%rip), %rdi
+	movl	$(COMMAND_LINE_SIZE / 4), %ecx
+	rep movsl
+.Lno_command_line:
+
 	/*
 	 * Set up GSBASE.
 	 * Note that on SMP the boot CPU uses the init data section until
@@ -92,9 +109,8 @@ SYM_CODE_START_NOALIGN(startup_64)
 	 * Activate SEV/SME memory encryption if supported/enabled. This needs to
 	 * be done now, since this also includes setup of the SEV-SNP CPUID table,
 	 * which needs to be done before any CPUID instructions are executed in
-	 * subsequent code. Pass the boot_params pointer as the first argument.
+	 * subsequent code.
 	 */
-	movq	%r15, %rdi
 	call	__pi_sme_enable
 #endif
 
@@ -114,7 +130,6 @@ SYM_CODE_START_NOALIGN(startup_64)
 	 * is active) to be added to the initial pgdir entry that will be
 	 * programmed into CR3.
 	 */
-	movq	%r15, %rsi
 	call	__pi___startup_64
 
 	/* Form the CR3 value being sure to include the CR3 modifier */
@@ -195,9 +210,6 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 	UNWIND_HINT_END_OF_STACK
 	ANNOTATE_NOENDBR
 
-	/* Clear %R15 which holds the boot_params pointer on the boot CPU */
-	xorl	%r15d, %r15d
-
 	/* Derive the runtime physical address of init_top_pgt[] */
 	movq	phys_base(%rip), %rax
 	addq	$(init_top_pgt - __START_KERNEL_map), %rax
@@ -429,9 +441,6 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
 	pushq $0
 	popfq
 
-	/* Pass the boot_params pointer as first argument */
-	movq	%r15, %rdi
-
 .Ljump_to_C_code:
 	xorl	%ebp, %ebp	# clear frame pointer
 	ANNOTATE_RETPOLINE_SAFE
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 46882ce79c3a..bd9d42767c8c 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -77,6 +77,7 @@ unsigned long _brk_start = (unsigned long)__brk_base;
 unsigned long _brk_end   = (unsigned long)__brk_base;
 
 struct boot_params boot_params;
+SYM_PIC_ALIAS(boot_params);
 
 /*
  * These are the four main kernel memory regions, we put them into
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index ba01a9e919b7..06287eed5cbc 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -226,7 +226,7 @@ static void ap_calibrate_delay(void)
 /*
  * Activate a secondary processor.
  */
-static void notrace __noendbr start_secondary(void *unused)
+static void notrace __noendbr start_secondary(void)
 {
 	/*
 	 * Don't put *anything* except direct CPU state initialization
diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
index d39e1e29bcb9..877d4d85c949 100644
--- a/arch/x86/mm/mem_encrypt_amd.c
+++ b/arch/x86/mm/mem_encrypt_amd.c
@@ -153,67 +153,6 @@ void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
 	__sme_early_enc_dec(paddr, size, false);
 }
 
-static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
-					     bool map)
-{
-	unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
-	pmdval_t pmd_flags, pmd;
-
-	/* Use early_pmd_flags but remove the encryption mask */
-	pmd_flags = __sme_clr(early_pmd_flags);
-
-	do {
-		pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
-		__early_make_pgtable((unsigned long)vaddr, pmd);
-
-		vaddr += PMD_SIZE;
-		paddr += PMD_SIZE;
-		size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
-	} while (size);
-
-	flush_tlb_local();
-}
-
-void __init sme_unmap_bootdata(char *real_mode_data)
-{
-	struct boot_params *boot_data;
-	unsigned long cmdline_paddr;
-
-	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
-		return;
-
-	/* Get the command line address before unmapping the real_mode_data */
-	boot_data = (struct boot_params *)real_mode_data;
-	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
-
-	__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
-
-	if (!cmdline_paddr)
-		return;
-
-	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
-}
-
-void __init sme_map_bootdata(char *real_mode_data)
-{
-	struct boot_params *boot_data;
-	unsigned long cmdline_paddr;
-
-	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
-		return;
-
-	__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
-
-	/* Get the command line address after mapping the real_mode_data */
-	boot_data = (struct boot_params *)real_mode_data;
-	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
-
-	if (!cmdline_paddr)
-		return;
-
-	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
-}
-
 static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot)
 {
 	unsigned long pfn = 0;
-- 
2.55.0