[Qemu-devel] FreeBSD/amd64 guests with -kernel-kqemu, pagefault at mov %r10d, %gs

[Qemu-devel] FreeBSD/amd64 guests with -kernel-kqemu, pagefault at mov %r10d, %gs

[Juergen Lock]

..before that it does a mov %r10d,%fs which seems to work (%r10d is
_udatasel in both cases) so it can't be the segment itself that it
doesn't like, or can it?  Anyone have an idea what this might be?
(it works without -kernel-kqemu.)

 From the failed kernel log:

[...]
start_init: trying /sbin/init
kernel trap 12 with interrupts disabled


Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address   = 0xfff8
fault code              = supervisor read data, page not present
instruction pointer     = 0x8:0xffffffff806dc771
stack pointer           = 0x10:0xffffffff91f9f840
frame pointer           = 0x10:0xffffffff91f9f8a0
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = resume, IOPL = 0
current process         = 1 (init)
trap number             = 12
panic: page fault
cpuid = 0
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
panic() at panic+0x17d
trap_fatal() at trap_fatal+0x29b
trap_pfault() at trap_pfault+0x22d
trap() at trap+0x30c
calltrap() at calltrap+0x8
--- trap 0xc, rip = 0xffffffff806dc771, rsp = 0xffffffff91f9f840, rbp = 0xffffffff91f9f8a0 ---
exec_setregs() at exec_setregs+0x81
kern_execve() at kern_execve+0x78d
execve() at execve+0x3d
start_init() at start_init+0x232
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffffff91f9fd30, rbp = 0 ---
Uptime: 4s
Cannot dump. No dump device defined.
[...]

 And this is the dissassembly of the surrounding code:

(kgdb) disassemble exec_setregs
Dump of assembler code for function exec_setregs:
0xffffffff806dc6f0 <exec_setregs+0>:	push   %rbp
0xffffffff806dc6f1 <exec_setregs+1>:	mov    %rsp,%rbp
0xffffffff806dc6f4 <exec_setregs+4>:	sub    $0x40,%rsp
0xffffffff806dc6f8 <exec_setregs+8>:	mov    %rbx,0xffffffffffffffd8(%rbp)
0xffffffff806dc6fc <exec_setregs+12>:	mov    %r13,0xffffffffffffffe8(%rbp)
0xffffffff806dc700 <exec_setregs+16>:	mov    $0xc0000100,%ebx
0xffffffff806dc705 <exec_setregs+21>:	mov    %r14,0xfffffffffffffff0(%rbp)
0xffffffff806dc709 <exec_setregs+25>:	mov    %r12,0xffffffffffffffe0(%rbp)
0xffffffff806dc70d <exec_setregs+29>:	mov    %rdx,%r13
0xffffffff806dc710 <exec_setregs+32>:	mov    %r15,0xfffffffffffffff8(%rbp)
0xffffffff806dc714 <exec_setregs+36>:	mov    %rsi,0xffffffffffffffc0(%rbp)
0xffffffff806dc718 <exec_setregs+40>:	mov    %rdi,%r14
0xffffffff806dc71b <exec_setregs+43>:	mov    0x2a8(%rdi),%r12
0xffffffff806dc722 <exec_setregs+50>:	mov    0x250(%rdi),%r15
0xffffffff806dc729 <exec_setregs+57>:	callq  0xffffffff8049ad10 <critical_enter>
0xffffffff806dc72e <exec_setregs+62>:	xor    %eax,%eax
0xffffffff806dc730 <exec_setregs+64>:	mov    %ebx,%ecx
0xffffffff806dc732 <exec_setregs+66>:	mov    %eax,%edx
0xffffffff806dc734 <exec_setregs+68>:	wrmsr  
0xffffffff806dc736 <exec_setregs+70>:	mov    $0xc0000102,%ecx
0xffffffff806dc73b <exec_setregs+75>:	wrmsr  
---Type <return> to continue, or q <return> to quit---
0xffffffff806dc73d <exec_setregs+77>:	movq   $0x0,0x48(%r15)
0xffffffff806dc745 <exec_setregs+85>:	movq   $0x0,0x50(%r15)
0xffffffff806dc74d <exec_setregs+93>:	callq  0xffffffff8049ac00 <critical_exit>
0xffffffff806dc752 <exec_setregs+98>:	mov    4183943(%rip),%r10d        # 0xffffffff80ad9ee0 <_udatasel>
0xffffffff806dc759 <exec_setregs+105>:	mov    %r10d,%ds
0xffffffff806dc75c <exec_setregs+108>:	mov    %r10d,%es
0xffffffff806dc75f <exec_setregs+111>:	mov    %ebx,%ecx
0xffffffff806dc761 <exec_setregs+113>:	rdmsr  
0xffffffff806dc763 <exec_setregs+115>:	mov    %r10d,%fs
0xffffffff806dc766 <exec_setregs+118>:	wrmsr  
0xffffffff806dc768 <exec_setregs+120>:	mov    $0xc0000101,%ecx
0xffffffff806dc76d <exec_setregs+125>:	pushfq 
0xffffffff806dc76e <exec_setregs+126>:	cli    
0xffffffff806dc76f <exec_setregs+127>:	rdmsr  
0xffffffff806dc771 <exec_setregs+129>:	mov    %r10d,%gs
			   failed insn  ^^^^^^^^^^^^^^^^^^
0xffffffff806dc774 <exec_setregs+132>:	wrmsr  
0xffffffff806dc776 <exec_setregs+134>:	popfq  
0xffffffff806dc777 <exec_setregs+135>:	mov    %r10d,0x58(%r15)
0xffffffff806dc77b <exec_setregs+139>:	mov    4183902(%rip),%r9d        # 0xffffffff80ad9ee0 <_udatasel>
0xffffffff806dc782 <exec_setregs+146>:	mov    $0xc0,%esi
---Type <return> to continue, or q <return> to quit---
0xffffffff806dc787 <exec_setregs+151>:	lea    0xfffffffffffffff8(%r13),%rbx
0xffffffff806dc78b <exec_setregs+155>:	mov    %r9d,0x5c(%r15)
0xffffffff806dc78f <exec_setregs+159>:	mov    4183882(%rip),%r8d        # 0xffffffff80ad9ee0 <_udatasel>
0xffffffff806dc796 <exec_setregs+166>:	and    $0xfffffffffffffff0,%rbx
0xffffffff806dc79a <exec_setregs+170>:	add    $0x8,%rbx
0xffffffff806dc79e <exec_setregs+174>:	mov    %r8d,0x60(%r15)
0xffffffff806dc7a2 <exec_setregs+178>:	mov    4183864(%rip),%edi        # 0xffffffff80ad9ee0 <_udatasel>
0xffffffff806dc7a8 <exec_setregs+184>:	mov    %edi,0x64(%r15)
0xffffffff806dc7ac <exec_setregs+188>:	mov    %r12,%rdi
0xffffffff806dc7af <exec_setregs+191>:	callq  0xffffffff806eeb10 <bzero>
0xffffffff806dc7b4 <exec_setregs+196>:	mov    0xa8(%r12),%rcx
0xffffffff806dc7bc <exec_setregs+204>:	mov    0xffffffffffffffc0(%rbp),%rsi
0xffffffff806dc7c0 <exec_setregs+208>:	mov    %rbx,0xb0(%r12)
0xffffffff806dc7c8 <exec_setregs+216>:	mov    %r13,(%r12)
0xffffffff806dc7cc <exec_setregs+220>:	and    $0x100,%ecx
0xffffffff806dc7d2 <exec_setregs+226>:	mov    %rsi,0x98(%r12)
0xffffffff806dc7da <exec_setregs+234>:	or     $0x202,%rcx
0xffffffff806dc7e1 <exec_setregs+241>:	mov    %rcx,0xa8(%r12)
0xffffffff806dc7e9 <exec_setregs+249>:	movslq 4183792(%rip),%rdx        # 0xffffffff80ad9ee0 <_udatasel>
0xffffffff806dc7f0 <exec_setregs+256>:	mov    %rdx,0xb8(%r12)
---Type <return> to continue, or q <return> to quit---
0xffffffff806dc7f8 <exec_setregs+264>:	movslq 4183781(%rip),%rax        # 0xffffffff80ad9ee4 <_ucodesel>
0xffffffff806dc7ff <exec_setregs+271>:	mov    %rax,0xa0(%r12)
0xffffffff806dc807 <exec_setregs+279>:	testb  $0x2,0x2a0(%r15)
0xffffffff806dc80f <exec_setregs+287>:	je     0xffffffff806dc864 <exec_setregs+372>
0xffffffff806dc811 <exec_setregs+289>:	movq   $0x0,0x68(%r15)
0xffffffff806dc819 <exec_setregs+297>:	movq   $0x0,0x70(%r15)
0xffffffff806dc821 <exec_setregs+305>:	movq   $0x0,0x78(%r15)
0xffffffff806dc829 <exec_setregs+313>:	movq   $0x0,0x80(%r15)
0xffffffff806dc834 <exec_setregs+324>:	movq   $0x0,0x88(%r15)
0xffffffff806dc83f <exec_setregs+335>:	movq   $0x0,0x90(%r15)
0xffffffff806dc84a <exec_setregs+346>:	mov    %gs:0x20,%r11
0xffffffff806dc853 <exec_setregs+355>:	cmp    %r15,%r11
0xffffffff806dc856 <exec_setregs+358>:	mov    %r11,0xffffffffffffffd0(%rbp)
0xffffffff806dc85a <exec_setregs+362>:	je     0xffffffff806dc882 <exec_setregs+402>
0xffffffff806dc85c <exec_setregs+364>:	andq   $0xfffffffffffffffd,0x2a0(%r15)
0xffffffff806dc864 <exec_setregs+372>:	mov    %r14,%rdi
0xffffffff806dc867 <exec_setregs+375>:	callq  0xffffffff806dab90 <fpstate_drop>
0xffffffff806dc86c <exec_setregs+380>:	mov    0xffffffffffffffd8(%rbp),%rbx
0xffffffff806dc870 <exec_setregs+384>:	mov    0xffffffffffffffe0(%rbp),%r12
---Type <return> to continue, or q <return> to quit---
0xffffffff806dc874 <exec_setregs+388>:	mov    0xffffffffffffffe8(%rbp),%r13
0xffffffff806dc878 <exec_setregs+392>:	mov    0xfffffffffffffff0(%rbp),%r14
0xffffffff806dc87c <exec_setregs+396>:	mov    0xfffffffffffffff8(%rbp),%r15
0xffffffff806dc880 <exec_setregs+400>:	leaveq 
0xffffffff806dc881 <exec_setregs+401>:	retq   
0xffffffff806dc882 <exec_setregs+402>:	callq  0xffffffff806daa00 <reset_dbregs>
0xffffffff806dc887 <exec_setregs+407>:	jmp    0xffffffff806dc85c <exec_setregs+364>
End of assembler dump.

 Source is here:
	http://www.freebsd.org/cgi/cvsweb.cgi/src/sys/amd64/amd64/machdep.c
 (kernel used has 1.683:
	http://www.freebsd.org/cgi/cvsweb.cgi/src/sys/amd64/amd64/machdep.c?rev=1.683;content-type=text%2Fplain
but that function hasnt changed in the latest (HEAD) version i.e. 1.686)

 Thanx :)
	Juergen

Re: [Qemu-devel] FreeBSD/amd64 guests with -kernel-kqemu, pagefault at mov %r10d, %gs

[Mulyadi Santosa]

Hi...

On Wed, May 7, 2008 at 1:59 AM, Juergen Lock <nox@jelal.kn-bremen.de> wrote:
> ..before that it does a mov %r10d,%fs which seems to work (%r10d is
>  _udatasel in both cases) so it can't be the segment itself that it
>  doesn't like, or can it?  Anyone have an idea what this might be?
>  (it works without -kernel-kqemu.)

<..snip..>

>  0xffffffff806dc752 <exec_setregs+98>:   mov    4183943(%rip),%r10d        # 0xffffffff80ad9ee0 <_udatasel>
>  0xffffffff806dc759 <exec_setregs+105>:  mov    %r10d,%ds
>  0xffffffff806dc75c <exec_setregs+108>:  mov    %r10d,%es
>  0xffffffff806dc75f <exec_setregs+111>:  mov    %ebx,%ecx
>  0xffffffff806dc761 <exec_setregs+113>:  rdmsr
>  0xffffffff806dc763 <exec_setregs+115>:  mov    %r10d,%fs
>  0xffffffff806dc766 <exec_setregs+118>:  wrmsr
>  0xffffffff806dc768 <exec_setregs+120>:  mov    $0xc0000101,%ecx
>  0xffffffff806dc76d <exec_setregs+125>:  pushfq
>  0xffffffff806dc76e <exec_setregs+126>:  cli
>  0xffffffff806dc76f <exec_setregs+127>:  rdmsr
>  0xffffffff806dc771 <exec_setregs+129>:  mov    %r10d,%gs
>                            failed insn  ^^^^^^^^^^^^^^^^^^

I think I agree somehow accessing %gs is the quirk. let's just hope gs
points to valid entry in GDT or LDT...

But may I ask, what does the effect of "cli" in -kernel-kqemu on
FreeBSD's kqemu?

regards,

Mulyadi.

Re: [Qemu-devel] FreeBSD/amd64 guests with -kernel-kqemu, pagefault at mov %r10d, %gs

[Juergen Lock]

In article <f284c33d0805062005w1b66405w6485d85a06bfd084@mail.gmail.com> you write:
>Hi...
>
>On Wed, May 7, 2008 at 1:59 AM, Juergen Lock <nox@jelal.kn-bremen.de> wrote:
>> ..before that it does a mov %r10d,%fs which seems to work (%r10d is
>>  _udatasel in both cases) so it can't be the segment itself that it
>>  doesn't like, or can it?  Anyone have an idea what this might be?
>>  (it works without -kernel-kqemu.)
>
><..snip..>
>
>>  0xffffffff806dc752 <exec_setregs+98>:   mov    4183943(%rip),%r10d   
># 0xffffffff80ad9ee0 <_udatasel>
>>  0xffffffff806dc759 <exec_setregs+105>:  mov    %r10d,%ds
>>  0xffffffff806dc75c <exec_setregs+108>:  mov    %r10d,%es
>>  0xffffffff806dc75f <exec_setregs+111>:  mov    %ebx,%ecx
>>  0xffffffff806dc761 <exec_setregs+113>:  rdmsr
>>  0xffffffff806dc763 <exec_setregs+115>:  mov    %r10d,%fs
>>  0xffffffff806dc766 <exec_setregs+118>:  wrmsr
>>  0xffffffff806dc768 <exec_setregs+120>:  mov    $0xc0000101,%ecx
>>  0xffffffff806dc76d <exec_setregs+125>:  pushfq
>>  0xffffffff806dc76e <exec_setregs+126>:  cli
>>  0xffffffff806dc76f <exec_setregs+127>:  rdmsr
>>  0xffffffff806dc771 <exec_setregs+129>:  mov    %r10d,%gs
>>                            failed insn  ^^^^^^^^^^^^^^^^^^
>
>I think I agree somehow accessing %gs is the quirk. let's just hope gs
>points to valid entry in GDT or LDT...
>
As I said %r10d is _udatasel here and was just succesfully loaded into %fs.

>But may I ask, what does the effect of "cli" in -kernel-kqemu on
>FreeBSD's kqemu?
>
 cli seems to be handled in kqemu-1.3.0pre11/common/interp.c in
insn_interp(), look for
	LABEL(fa) /* cli */

 Or do you mean why cli is used here?  %gs is used for cpu-local
variables in the FreeBSD kernel (like curthread), and since the gsbase
msr apparently gets reset on a mov to %gs, it needs to be saved and
restored across that mov, and cli is then needed because an interrupt
that would be executed before the msr is restored would find cpu-local
variables wrong...

 Oh, and just in case this is not clear, this is code from the guest not
from kqemu itself. :)

	Juergen