Tentative fix for dom0 boot problem

Tentative fix for dom0 boot problem

[Juergen Gross]

[-- Attachment #1.1.1: Type: text/plain, Size: 69 bytes --]

Julien,

could you please test the attached patches?


Juergen

[-- Attachment #1.1.2: 0001-x86-xen-use-clear_bss-for-Xen-PV-guests.patch --]
[-- Type: text/x-patch, Size: 2994 bytes --]

From 7d6734e517b65c3702a279fc2c1fff5ba7927ef9 Mon Sep 17 00:00:00 2001
From: Juergen Gross <jgross@suse.com>
Date: Wed, 22 Jun 2022 12:19:55 +0200
Subject: [PATCH 1/2] x86/xen: use clear_bss() for Xen PV guests

Instead of clearing the bss area in assembly code, use the clear_bss()
function.

This requires to pass the start_info address as parameter to
xen_start_kernel() in order to avoid the xen_start_info being zeroed
again.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/x86/include/asm/setup.h |  3 +++
 arch/x86/kernel/head64.c     |  2 +-
 arch/x86/xen/enlighten_pv.c  |  8 ++++++--
 arch/x86/xen/xen-head.S      | 10 +---------
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index f8b9ee97a891..f37cbff7354c 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -120,6 +120,9 @@ void *extend_brk(size_t size, size_t align);
 	static char __brk_##name[size]
 
 extern void probe_roms(void);
+
+void clear_bss(void);
+
 #ifdef __i386__
 
 asmlinkage void __init i386_start_kernel(void);
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index bd4a34100ed0..e7e233209a8c 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -426,7 +426,7 @@ void __init do_early_exception(struct pt_regs *regs, int trapnr)
 
 /* Don't add a printk in there. printk relies on the PDA which is not initialized 
    yet. */
-static void __init clear_bss(void)
+void __init clear_bss(void)
 {
 	memset(__bss_start, 0,
 	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index e3297b15701c..70fb2ea85e90 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1183,15 +1183,19 @@ static void __init xen_domu_set_legacy_features(void)
 extern void early_xen_iret_patch(void);
 
 /* First C function to be called on Xen boot */
-asmlinkage __visible void __init xen_start_kernel(void)
+asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
 {
 	struct physdev_set_iopl set_iopl;
 	unsigned long initrd_start = 0;
 	int rc;
 
-	if (!xen_start_info)
+	if (!si)
 		return;
 
+	clear_bss();
+
+	xen_start_info = si;
+
 	__text_gen_insn(&early_xen_iret_patch,
 			JMP32_INSN_OPCODE, &early_xen_iret_patch, &xen_iret,
 			JMP32_INSN_SIZE);
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 3a2cd93bf059..13af6fe453e3 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -48,15 +48,6 @@ SYM_CODE_START(startup_xen)
 	ANNOTATE_NOENDBR
 	cld
 
-	/* Clear .bss */
-	xor %eax,%eax
-	mov $__bss_start, %rdi
-	mov $__bss_stop, %rcx
-	sub %rdi, %rcx
-	shr $3, %rcx
-	rep stosq
-
-	mov %rsi, xen_start_info
 	mov initial_stack(%rip), %rsp
 
 	/* Set up %gs.
@@ -71,6 +62,7 @@ SYM_CODE_START(startup_xen)
 	cdq
 	wrmsr
 
+	mov	%rsi, %rdi
 	call xen_start_kernel
 SYM_CODE_END(startup_xen)
 	__FINIT
-- 
2.35.3


[-- Attachment #1.1.3: 0002-x86-fix-setup-of-brk-area.patch --]
[-- Type: text/x-patch, Size: 1174 bytes --]

From ff35bc33c5ee184868d41bf76dcc7322191f9fd3 Mon Sep 17 00:00:00 2001
From: Juergen Gross <jgross@suse.com>
Date: Wed, 22 Jun 2022 12:17:47 +0200
Subject: [PATCH 2/2] x86: fix setup of brk area

Commit e32683c6f7d2 ("x86/mm: Fix RESERVE_BRK() for older binutils")
put the brk area into the .bss segment, causing it not to be cleared
initially. As the brk area is used to allocate early page tables, these
might contain garbage in not explicitly written entries.

Fix that by letting clear_bss() clear the brk area, too.

Fixes: e32683c6f7d2 ("x86/mm: Fix RESERVE_BRK() for older binutils")
Signed-off-by: Juergen Gross <jgross@suse.com>
---
 arch/x86/kernel/head64.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index e7e233209a8c..6a3cfaf6b72a 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -430,6 +430,8 @@ void __init clear_bss(void)
 {
 	memset(__bss_start, 0,
 	       (unsigned long) __bss_stop - (unsigned long) __bss_start);
+	memset(__brk_base, 0,
+	       (unsigned long) __brk_limit - (unsigned long) __brk_base);
 }
 
 static unsigned long get_cmd_line_ptr(void)
-- 
2.35.3


[-- Attachment #1.1.4: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 3149 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: Tentative fix for dom0 boot problem

[Julien Grall]

On 22/06/2022 11:45, Juergen Gross wrote:
> Julien,

Hi Juergen,

> could you please test the attached patches?

I am getting the following error:

(XEN) d0v0 Unhandled: vec 14, #PF[0003]
(XEN) Pagetable walk from ffffffff84001000:
(XEN)  L4[0x1ff] = 000000046c004067 0000000000004004
(XEN)  L3[0x1fe] = 000000046c003067 0000000000004003
(XEN)  L2[0x020] = 000000046c024067 0000000000004024
(XEN)  L1[0x001] = 001000046c001025 0000000000004001
(XEN) domain_crash_sync called from entry.S: fault at ffff82d040325906 
x86_64/entry.S#create_bounce_frame+0x15d/0x177
(XEN) Domain 0 (vcpu#0) crashed on cpu#1:
(XEN) ----[ Xen-4.17-unstable  x86_64  debug=y  Tainted:   C    ]----
(XEN) CPU:    1
(XEN) RIP:    e033:[<ffffffff832a3481>]
(XEN) RFLAGS: 0000000000000206   EM: 1   CONTEXT: pv guest (d0v0)
(XEN) rax: 0000000000000000   rbx: ffffffff84000000   rcx: 000000000002b000
(XEN) rdx: ffffffff84000000   rsi: ffffffff84000000   rdi: ffffffff84001000
(XEN) rbp: 0000000000000000   rsp: ffffffff82a03e60   r8:  0000000000000000
(XEN) r9:  0000000000000000   r10: 0000000000000000   r11: 0000000000000000
(XEN) r12: 0000000000000000   r13: 0000000000000000   r14: 0000000000000000
(XEN) r15: 0000000000000000   cr0: 0000000080050033   cr4: 00000000003426e0
(XEN) cr3: 000000046c001000   cr2: ffffffff84001000
(XEN) fsb: 0000000000000000   gsb: ffffffff83271000   gss: 0000000000000000
(XEN) ds: 0000   es: 0000   fs: 0000   gs: 0000   ss: e02b   cs: e033
(XEN) Guest stack trace from rsp=ffffffff82a03e60:
(XEN)    000000000002b000 0000000000000000 0000000000000003 ffffffff832a3481
(XEN)    000000010000e030 0000000000010006 ffffffff82a03ea8 000000000000e02b
(XEN)    0000000000000000 ffffffff832ae884 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 ffffffff832a317f 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
(XEN) Hardware Dom0 crashed: rebooting machine in 5 seconds.

Cheers,

-- 
Julien Grall

Re: Tentative fix for dom0 boot problem

[Juergen Gross]

[-- Attachment #1.1.1: Type: text/plain, Size: 2754 bytes --]

On 22.06.22 12:50, Julien Grall wrote:
> 
> 
> On 22/06/2022 11:45, Juergen Gross wrote:
>> Julien,
> 
> Hi Juergen,
> 
>> could you please test the attached patches?
> 
> I am getting the following error:
> 
> (XEN) d0v0 Unhandled: vec 14, #PF[0003]
> (XEN) Pagetable walk from ffffffff84001000:
> (XEN)  L4[0x1ff] = 000000046c004067 0000000000004004
> (XEN)  L3[0x1fe] = 000000046c003067 0000000000004003
> (XEN)  L2[0x020] = 000000046c024067 0000000000004024
> (XEN)  L1[0x001] = 001000046c001025 0000000000004001

Hmm, from this data I guess this was a write to a page table.

> (XEN) domain_crash_sync called from entry.S: fault at ffff82d040325906 
> x86_64/entry.S#create_bounce_frame+0x15d/0x177
> (XEN) Domain 0 (vcpu#0) crashed on cpu#1:
> (XEN) ----[ Xen-4.17-unstable  x86_64  debug=y  Tainted:   C    ]----
> (XEN) CPU:    1
> (XEN) RIP:    e033:[<ffffffff832a3481>]

Can you please find out the associated statement?

> (XEN) RFLAGS: 0000000000000206   EM: 1   CONTEXT: pv guest (d0v0)
> (XEN) rax: 0000000000000000   rbx: ffffffff84000000   rcx: 000000000002b000
> (XEN) rdx: ffffffff84000000   rsi: ffffffff84000000   rdi: ffffffff84001000
> (XEN) rbp: 0000000000000000   rsp: ffffffff82a03e60   r8:  0000000000000000
> (XEN) r9:  0000000000000000   r10: 0000000000000000   r11: 0000000000000000
> (XEN) r12: 0000000000000000   r13: 0000000000000000   r14: 0000000000000000
> (XEN) r15: 0000000000000000   cr0: 0000000080050033   cr4: 00000000003426e0
> (XEN) cr3: 000000046c001000   cr2: ffffffff84001000
> (XEN) fsb: 0000000000000000   gsb: ffffffff83271000   gss: 0000000000000000
> (XEN) ds: 0000   es: 0000   fs: 0000   gs: 0000   ss: e02b   cs: e033
> (XEN) Guest stack trace from rsp=ffffffff82a03e60:
> (XEN)    000000000002b000 0000000000000000 0000000000000003 ffffffff832a3481
> (XEN)    000000010000e030 0000000000010006 ffffffff82a03ea8 000000000000e02b
> (XEN)    0000000000000000 ffffffff832ae884 0000000000000000 0000000000000000
> (XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
> (XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
> (XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
> (XEN)    0000000000000000 0000000000000000 0000000000000000 0000000000000000
> (XEN)    0000000000000000 0000000000000000 ffffffff832a317f 0000000000000000

Further analysis might be easier if you can supply function + displacement for
any text segment addresses on the stack.

BTW, I could boot the kernel with my patches as Dom0 without any problem. OTOH
it booted even without the patches. :-)


Juergen

[-- Attachment #1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 3149 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: Tentative fix for dom0 boot problem

[Julien Grall]

Hi Juergen,

On 22/06/2022 13:13, Juergen Gross wrote:
> On 22.06.22 12:50, Julien Grall wrote:
>>
>>
>> On 22/06/2022 11:45, Juergen Gross wrote:
>>> Julien,
>>
>> Hi Juergen,
>>
>>> could you please test the attached patches?
>>
>> I am getting the following error:
>>
>> (XEN) d0v0 Unhandled: vec 14, #PF[0003]
>> (XEN) Pagetable walk from ffffffff84001000:
>> (XEN)  L4[0x1ff] = 000000046c004067 0000000000004004
>> (XEN)  L3[0x1fe] = 000000046c003067 0000000000004003
>> (XEN)  L2[0x020] = 000000046c024067 0000000000004024
>> (XEN)  L1[0x001] = 001000046c001025 0000000000004001
> 
> Hmm, from this data I guess this was a write to a page table.
> 
>> (XEN) domain_crash_sync called from entry.S: fault at ffff82d040325906 
>> x86_64/entry.S#create_bounce_frame+0x15d/0x177
>> (XEN) Domain 0 (vcpu#0) crashed on cpu#1:
>> (XEN) ----[ Xen-4.17-unstable  x86_64  debug=y  Tainted:   C    ]----
>> (XEN) CPU:    1
>> (XEN) RIP:    e033:[<ffffffff832a3481>]
> 
> Can you please find out the associated statement?

arch/x86/kernel/head64.c:433

This is the memset() for __brk_base.

> 
>> (XEN) RFLAGS: 0000000000000206   EM: 1   CONTEXT: pv guest (d0v0)
>> (XEN) rax: 0000000000000000   rbx: ffffffff84000000   rcx: 
>> 000000000002b000
>> (XEN) rdx: ffffffff84000000   rsi: ffffffff84000000   rdi: 
>> ffffffff84001000
>> (XEN) rbp: 0000000000000000   rsp: ffffffff82a03e60   r8:  
>> 0000000000000000
>> (XEN) r9:  0000000000000000   r10: 0000000000000000   r11: 
>> 0000000000000000
>> (XEN) r12: 0000000000000000   r13: 0000000000000000   r14: 
>> 0000000000000000
>> (XEN) r15: 0000000000000000   cr0: 0000000080050033   cr4: 
>> 00000000003426e0
>> (XEN) cr3: 000000046c001000   cr2: ffffffff84001000
>> (XEN) fsb: 0000000000000000   gsb: ffffffff83271000   gss: 
>> 0000000000000000
>> (XEN) ds: 0000   es: 0000   fs: 0000   gs: 0000   ss: e02b   cs: e033
>> (XEN) Guest stack trace from rsp=ffffffff82a03e60:
>> (XEN)    000000000002b000 0000000000000000 0000000000000003 
>> ffffffff832a3481
>> (XEN)    000000010000e030 0000000000010006 ffffffff82a03ea8 
>> 000000000000e02b
>> (XEN)    0000000000000000 ffffffff832ae884 0000000000000000 
>> 0000000000000000
>> (XEN)    0000000000000000 0000000000000000 0000000000000000 
>> 0000000000000000
>> (XEN)    0000000000000000 0000000000000000 0000000000000000 
>> 0000000000000000
>> (XEN)    0000000000000000 0000000000000000 0000000000000000 
>> 0000000000000000
>> (XEN)    0000000000000000 0000000000000000 0000000000000000 
>> 0000000000000000
>> (XEN)    0000000000000000 0000000000000000 ffffffff832a317f 
>> 0000000000000000
> 
> Further analysis might be easier if you can supply function + 
> displacement for
> any text segment addresses on the stack.

ffffffff832ae884: arch/x86/include/asm/text-patching.h:112
ffffffff832a317f: arch/x86/kernel/head64.c:325

> 
> BTW, I could boot the kernel with my patches as Dom0 without any 
> problem. OTOH
> it booted even without the patches. :-)

So I have tried with two different compilers (GCC 7.3.1 and GCC 10.2.1) 
and hit the same error. This would suggest this is related to my 
.config. You can find it in [1] if you want to reproduce it yourself.

Cheers,

[1] https://pastebin.com/xityGDN9

-- 
Julien Grall

Re: Tentative fix for dom0 boot problem

[Juergen Gross]

[-- Attachment #1.1.1: Type: text/plain, Size: 2749 bytes --]

On 22.06.22 15:20, Julien Grall wrote:
> Hi Juergen,
> 
> On 22/06/2022 13:13, Juergen Gross wrote:
>> On 22.06.22 12:50, Julien Grall wrote:
>>>
>>>
>>> On 22/06/2022 11:45, Juergen Gross wrote:
>>>> Julien,
>>>
>>> Hi Juergen,
>>>
>>>> could you please test the attached patches?
>>>
>>> I am getting the following error:
>>>
>>> (XEN) d0v0 Unhandled: vec 14, #PF[0003]
>>> (XEN) Pagetable walk from ffffffff84001000:
>>> (XEN)  L4[0x1ff] = 000000046c004067 0000000000004004
>>> (XEN)  L3[0x1fe] = 000000046c003067 0000000000004003
>>> (XEN)  L2[0x020] = 000000046c024067 0000000000004024
>>> (XEN)  L1[0x001] = 001000046c001025 0000000000004001
>>
>> Hmm, from this data I guess this was a write to a page table.
>>
>>> (XEN) domain_crash_sync called from entry.S: fault at ffff82d040325906 
>>> x86_64/entry.S#create_bounce_frame+0x15d/0x177
>>> (XEN) Domain 0 (vcpu#0) crashed on cpu#1:
>>> (XEN) ----[ Xen-4.17-unstable  x86_64  debug=y  Tainted:   C    ]----
>>> (XEN) CPU:    1
>>> (XEN) RIP:    e033:[<ffffffff832a3481>]
>>
>> Can you please find out the associated statement?
> 
> arch/x86/kernel/head64.c:433
> 
> This is the memset() for __brk_base.

Very weird.

In the kernel's linker script we have:

         __end_of_kernel_reserve = .;

         . = ALIGN(PAGE_SIZE);
         .brk (NOLOAD) : AT(ADDR(.brk) - LOAD_OFFSET) {
                 __brk_base = .;
                 . += 64 * 1024;         /* 64k alignment slop space */
                 *(.bss..brk)            /* areas brk users have reserved */
                 __brk_limit = .;
         }

         . = ALIGN(PAGE_SIZE);           /* keep VO_INIT_SIZE page aligned */
         _end = .;

So the area to be zeroed should be larger than 64k.

> 
>>
>>> (XEN) RFLAGS: 0000000000000206   EM: 1   CONTEXT: pv guest (d0v0)
>>> (XEN) rax: 0000000000000000   rbx: ffffffff84000000   rcx: 000000000002b000
>>> (XEN) rdx: ffffffff84000000   rsi: ffffffff84000000   rdi: ffffffff84001000

Just guessing I'd say that memset started at ffffffff84000000 and there are
2b000 bytes left to be cleared. A disassembly of clear_bss() would help here.

Anyway it seems as if the memset would run right into the initial page tables
supplied by the hypervisor.

Can you please post the hypervisor's console messages regarding dom0 sizes
and locations?

Could it be that the brk area is the last section relevant for loading the
kernel? In contrast to your .config, I have CONFIG_AMD_MEM_ENCRYPT defined
and this adds the .init.scratch section after the brk area. Maybe the
hypervisor is not adjusting the page table location correctly due to the
NOLOAD attribute of brk?


Juergen

[-- Attachment #1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 3149 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]