[kvm-ppc-devel] guest panic fixed

[kvm-ppc-devel] guest panic fixed

[Hollis Blanchard]

Thanks to some excellent debugging by Christian, we found the cause of
our annoying guest panics. We had not implemented exception priorities
(my bad), so a userspace ITLB miss was being delivered immediately after
entry to the decrementer interrupt vector. Patch follows,
http://penguinppc.org/~hollisb/kvm/ updated.

NFS root with virtio-net still doesn't work for me, but it doesn't
crash. :)

diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -68,6 +68,44 @@ static const u32 interrupt_msr_mask[16] 
 	[BOOKE_INTERRUPT_DEBUG]         = MSR_ME,
 };
 
+const unsigned char exception_priority[] = {
+	[BOOKE_INTERRUPT_DATA_STORAGE] = 0,
+	[BOOKE_INTERRUPT_INST_STORAGE] = 1,
+	[BOOKE_INTERRUPT_ALIGNMENT] = 2,
+	[BOOKE_INTERRUPT_PROGRAM] = 3,
+	[BOOKE_INTERRUPT_FP_UNAVAIL] = 4,
+	[BOOKE_INTERRUPT_SYSCALL] = 5,
+	[BOOKE_INTERRUPT_AP_UNAVAIL] = 6,
+	[BOOKE_INTERRUPT_DTLB_MISS] = 7,
+	[BOOKE_INTERRUPT_ITLB_MISS] = 8,
+	[BOOKE_INTERRUPT_MACHINE_CHECK] = 9,
+	[BOOKE_INTERRUPT_DEBUG] = 10,
+	[BOOKE_INTERRUPT_CRITICAL] = 11,
+	[BOOKE_INTERRUPT_WATCHDOG] = 12,
+	[BOOKE_INTERRUPT_EXTERNAL] = 13,
+	[BOOKE_INTERRUPT_FIT] = 14,
+	[BOOKE_INTERRUPT_DECREMENTER] = 15,
+};
+
+const unsigned char priority_exception[] = {
+	BOOKE_INTERRUPT_DATA_STORAGE,
+	BOOKE_INTERRUPT_INST_STORAGE,
+	BOOKE_INTERRUPT_ALIGNMENT,
+	BOOKE_INTERRUPT_PROGRAM,
+	BOOKE_INTERRUPT_FP_UNAVAIL,
+	BOOKE_INTERRUPT_SYSCALL,
+	BOOKE_INTERRUPT_AP_UNAVAIL,
+	BOOKE_INTERRUPT_DTLB_MISS,
+	BOOKE_INTERRUPT_ITLB_MISS,
+	BOOKE_INTERRUPT_MACHINE_CHECK,
+	BOOKE_INTERRUPT_DEBUG,
+	BOOKE_INTERRUPT_CRITICAL,
+	BOOKE_INTERRUPT_WATCHDOG,
+	BOOKE_INTERRUPT_EXTERNAL,
+	BOOKE_INTERRUPT_FIT,
+	BOOKE_INTERRUPT_DECREMENTER,
+};
+
 
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
@@ -183,19 +221,21 @@ static void kvmppc_check_and_deliver_int
 static void kvmppc_check_and_deliver_interrupts(struct kvm_vcpu *vcpu)
 {
 	unsigned long *pending = &vcpu->arch.pending_exceptions;
-	int exception;
-
-	exception = find_first_bit(pending, BITS_PER_BYTE * sizeof(*pending));
-	while (exception < BOOKE_MAX_INTERRUPT) {
+	unsigned int exception;
+	unsigned int priority;
+
+	priority = find_first_bit(pending, BITS_PER_BYTE * sizeof(*pending));
+	while (priority <= BOOKE_MAX_INTERRUPT) {
+		exception = priority_exception[priority];
 		if (kvmppc_can_deliver_interrupt(vcpu, exception)) {
 			kvmppc_clear_exception(vcpu, exception);
 			kvmppc_deliver_interrupt(vcpu, exception);
 			break;
 		}
 
-		exception = find_next_bit(pending,
-		                          BITS_PER_BYTE * sizeof(*pending),
-		                          exception + 1);
+		priority = find_next_bit(pending,
+		                         BITS_PER_BYTE * sizeof(*pending),
+		                         priority + 1);
 	}
 }
 
diff --git a/include/asm-powerpc/kvm_ppc.h b/include/asm-powerpc/kvm_ppc.h
--- a/include/asm-powerpc/kvm_ppc.h
+++ b/include/asm-powerpc/kvm_ppc.h
@@ -41,6 +41,9 @@ enum emulation_result {
 	EMULATE_FAIL,         /* can't emulate this instruction */
 };
 
+extern const unsigned char exception_priority[];
+extern const unsigned char priority_exception[];
+
 extern int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
 extern char kvmppc_handlers_start[];
 extern unsigned long kvmppc_handler_len;
@@ -65,12 +68,14 @@ extern void kvmppc_44x_tlb_trace(int act
 
 static inline void kvmppc_queue_exception(struct kvm_vcpu *vcpu, int exception)
 {
-	set_bit(exception, &vcpu->arch.pending_exceptions);
+	unsigned int priority = exception_priority[exception];
+	set_bit(priority, &vcpu->arch.pending_exceptions);
 }
 
 static inline void kvmppc_clear_exception(struct kvm_vcpu *vcpu, int exception)
 {
-	clear_bit(exception, &vcpu->arch.pending_exceptions);
+	unsigned int priority = exception_priority[exception];
+	clear_bit(priority, &vcpu->arch.pending_exceptions);
 }
 
 static inline void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)


-- 
Hollis Blanchard
IBM Linux Technology Center


-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-ppc-devel mailing list
kvm-ppc-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/kvm-ppc-devel

Re: [kvm-ppc-devel] guest panic fixed

[Christian Ehrhardt]

Hollis Blanchard wrote:
> Thanks to some excellent debugging by Christian, we found the cause of
> our annoying guest panics. We had not implemented exception priorities
> (my bad), so a userspace ITLB miss was being delivered immediately after
> entry to the decrementer interrupt vector. Patch follows,
> http://penguinppc.org/~hollisb/kvm/ updated.
> 
> NFS root with virtio-net still doesn't work for me, but it doesn't
> crash. :)
> 
> @@ -65,12 +68,14 @@ extern void kvmppc_44x_tlb_trace(int act
> 
>  static inline void kvmppc_queue_exception(struct kvm_vcpu *vcpu, int exception)
>  {
> -	set_bit(exception, &vcpu->arch.pending_exceptions);
> +	unsigned int priority = exception_priority[exception];
> +	set_bit(priority, &vcpu->arch.pending_exceptions);

saves a line and a local var on irq path (well the compiler might optimize that anyway)
	set_bit(exception_priority[exception], &vcpu->arch.pending_exceptions);

>  }
> 
>  static inline void kvmppc_clear_exception(struct kvm_vcpu *vcpu, int exception)
>  {
> -	clear_bit(exception, &vcpu->arch.pending_exceptions);
> +	unsigned int priority = exception_priority[exception];
> +	clear_bit(priority, &vcpu->arch.pending_exceptions);

same as above

>  }
> 
>  static inline void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
 


-- 

Grüsse / regards, 
Christian Ehrhardt
IBM Linux Technology Center, Open Virtualization

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-ppc-devel mailing list
kvm-ppc-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/kvm-ppc-devel

Re: [kvm-ppc-devel] guest panic fixed

[Christian Ehrhardt]

Hollis Blanchard wrote:
> Thanks to some excellent debugging by Christian, we found the cause of
> our annoying guest panics. We had not implemented exception priorities
> (my bad), so a userspace ITLB miss was being delivered immediately after
> entry to the decrementer interrupt vector. Patch follows,
> http://penguinppc.org/~hollisb/kvm/ updated.
> 
> NFS root with virtio-net still doesn't work for me, but it doesn't
> crash. :)
> 

my network works fine (at least initially) see the attached log:

Sending DHCP requests .., OK
IP-Config: Got DHCP answer from 192.168.1.2, my address is 192.168.1.10
IP-Config: Complete:
     device=eth0, addr\x192.168.1.10, mask%5.255.255.0, gw\x192.168.1.2,
     host\x192.168.1.10, domain=local-devnet, nis-domain=(none),
     bootserver\x192.168.1.2, rootserver\x192.168.1.2, rootpathLooking up port of RPC 100003/2 on 192.168.1.2
Looking up port of RPC 100005/1 on 192.168.1.2
VFS: Mounted root (nfs filesystem).
Freeing unused kernel memory: 116k init
modprobe: FATAL: Could not load /lib/modules/2.6.25-rc3-dirty/modules.dep: No such file or directory
modprobe: FATAL: Could not load /lib/modules/2.6.25-rc3-dirty/modules.dep: No such file or directory
INIT: version 2.85 booting
nfs: server 192.168.1.2 not responding, still trying
nfs: server 192.168.1.2 not responding, still trying
nfs: server 192.168.1.2 not responding, still trying
INFO: task rc.sysinit:679 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.


After that I run into those lockup's for init:1 and rc.sysinit:679. 
The network is working fine for a while and then that I get these nfs timeouts you see above.
I think thats the track I will continue - find out whats wrong there e.g. virtio mangles the packets, too high latency, lost virtio interrupts, ... whatever.

@Hollis let me know if I can help you with your nfs root setup somehow?

-- 

Grüsse / regards, 
Christian Ehrhardt
IBM Linux Technology Center, Open Virtualization

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-ppc-devel mailing list
kvm-ppc-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/kvm-ppc-devel

Re: [kvm-ppc-devel] guest panic fixed

[Hollis Blanchard]

On Thu, 2008-03-20 at 12:08 +0100, Christian Ehrhardt wrote:
> Hollis Blanchard wrote:
> > Thanks to some excellent debugging by Christian, we found the cause of
> > our annoying guest panics. We had not implemented exception priorities
> > (my bad), so a userspace ITLB miss was being delivered immediately after
> > entry to the decrementer interrupt vector. Patch follows,
> > http://penguinppc.org/~hollisb/kvm/ updated.
> > 
> > NFS root with virtio-net still doesn't work for me, but it doesn't
> > crash. :)
> > 
> 
> my network works fine (at least initially) see the attached log:
> 
> Sending DHCP requests .., OK
> IP-Config: Got DHCP answer from 192.168.1.2, my address is 192.168.1.10
> IP-Config: Complete:
>      device=eth0, addr\x192.168.1.10, mask%5.255.255.0, gw\x192.168.1.2,
>      host\x192.168.1.10, domain=local-devnet, nis-domain=(none),
>      bootserver\x192.168.1.2, rootserver\x192.168.1.2, rootpath> Looking up port of RPC 100003/2 on 192.168.1.2
> Looking up port of RPC 100005/1 on 192.168.1.2
> VFS: Mounted root (nfs filesystem).
> Freeing unused kernel memory: 116k init
> modprobe: FATAL: Could not load /lib/modules/2.6.25-rc3-dirty/modules.dep: No such file or directory
> modprobe: FATAL: Could not load /lib/modules/2.6.25-rc3-dirty/modules.dep: No such file or directory
> INIT: version 2.85 booting
> nfs: server 192.168.1.2 not responding, still trying
> nfs: server 192.168.1.2 not responding, still trying
> nfs: server 192.168.1.2 not responding, still trying
> INFO: task rc.sysinit:679 blocked for more than 120 seconds.
> "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> 
> 
> After that I run into those lockup's for init:1 and rc.sysinit:679. 
> The network is working fine for a while and then that I get these nfs
> timeouts you see above.
> I think thats the track I will continue - find out whats wrong there
> e.g. virtio mangles the packets, too high latency, lost virtio
> interrupts, ... whatever.
> 
> @Hollis let me know if I can help you with your nfs root setup somehow?

No, that's pretty much exactly what I see, but my timeouts appear before
modprobe and the INIT message.

-- 
Hollis Blanchard
IBM Linux Technology Center


-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-ppc-devel mailing list
kvm-ppc-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/kvm-ppc-devel

Re: [kvm-ppc-devel] guest panic fixed

[Hollis Blanchard]

On Thu, 2008-03-20 at 09:32 +0100, Christian Ehrhardt wrote:
> Hollis Blanchard wrote:
> > Thanks to some excellent debugging by Christian, we found the cause of
> > our annoying guest panics. We had not implemented exception priorities
> > (my bad), so a userspace ITLB miss was being delivered immediately after
> > entry to the decrementer interrupt vector. Patch follows,
> > http://penguinppc.org/~hollisb/kvm/ updated.
> > 
> > NFS root with virtio-net still doesn't work for me, but it doesn't
> > crash. :)
> > 
> > @@ -65,12 +68,14 @@ extern void kvmppc_44x_tlb_trace(int act
> > 
> >  static inline void kvmppc_queue_exception(struct kvm_vcpu *vcpu, int exception)
> >  {
> > -	set_bit(exception, &vcpu->arch.pending_exceptions);
> > +	unsigned int priority = exception_priority[exception];
> > +	set_bit(priority, &vcpu->arch.pending_exceptions);
> 
> saves a line and a local var on irq path (well the compiler might optimize that anyway)
> 	set_bit(exception_priority[exception], &vcpu->arch.pending_exceptions);

The compiler should handle it; you can examine the disassembly if you're
worried.

-- 
Hollis Blanchard
IBM Linux Technology Center


-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft
Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/
_______________________________________________
kvm-ppc-devel mailing list
kvm-ppc-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/kvm-ppc-devel