PCI SERR NMI may cause Xen to deadlock

PCI SERR NMI may cause Xen to deadlock

[David Vrabel]

If a PCI System Error (SERR) is asserted it causes an NMI. If this NMI
occurs while the CPU is in printk() then Xen may deadlock as
pci_serr_error() calls console_force_unlock() which screws up the
console lock.

I don't think removing the console_force_unlock() is sufficient as it
looks like printk() is unsafe to call from NMI context.  We would like
keep the diagnostic.  Does the following (untested) patch to defer the
printk() to a softirq look like a valid approach?

I also considered passing the NMI to dom0 (like other NMIs are) but I
couldn't see how NMIs were handled in the current upstream pv-ops kernels.

diff -r e2722b24dc09 xen/arch/x86/traps.c
--- a/xen/arch/x86/traps.c	Thu Jan 26 17:43:31 2012 +0000
+++ b/xen/arch/x86/traps.c	Tue Jan 31 16:28:45 2012 +0000
@@ -3173,6 +3173,11 @@ static void nmi_mce_softirq(void)
     st->vcpu = NULL;
 }

+static void pci_serr_softirq(void)
+{
+    printk("\n\nNMI - PCI system error (SERR)\n");
+}
+
 void async_exception_cleanup(struct vcpu *curr)
 {
     int trap;
@@ -3259,10 +3264,11 @@ static void nmi_dom0_report(unsigned int

 static void pci_serr_error(struct cpu_user_regs *regs)
 {
-    console_force_unlock();
-    printk("\n\nNMI - PCI system error (SERR)\n");
-
     outb((inb(0x61) & 0x0f) | 0x04, 0x61); /* clear-and-disable the PCI
SERR error line. */
+
+    /* Would like to print a diagnostic here but can't call printk()
+       from NMI context -- raise a softirq instead. */
+    raise_softirq(PCI_SERR_SOFTIRQ);
 }

 static void io_check_error(struct cpu_user_regs *regs)
@@ -3563,6 +3569,7 @@ void __init trap_init(void)
     cpu_init();

     open_softirq(NMI_MCE_SOFTIRQ, nmi_mce_softirq);
+    open_softirq(PCI_SERR_SOFTIRQ, pci_serr_softirq);
 }

 long register_guest_nmi_callback(unsigned long address)
diff -r e2722b24dc09 xen/include/asm-x86/softirq.h
--- a/xen/include/asm-x86/softirq.h	Thu Jan 26 17:43:31 2012 +0000
+++ b/xen/include/asm-x86/softirq.h	Tue Jan 31 16:28:45 2012 +0000
@@ -6,6 +6,7 @@
 #define VCPU_KICK_SOFTIRQ      (NR_COMMON_SOFTIRQS + 2)

 #define MACHINE_CHECK_SOFTIRQ  (NR_COMMON_SOFTIRQS + 3)
-#define NR_ARCH_SOFTIRQS       4
+#define PCI_SERR_SOFTIRQ       (NR_COMMON_SOFTIRQS + 4)
+#define NR_ARCH_SOFTIRQS       5

 #endif /* __ASM_SOFTIRQ_H__ */

David

Re: PCI SERR NMI may cause Xen to deadlock

[Keir Fraser]

On 31/01/2012 16:37, "David Vrabel" <david.vrabel@citrix.com> wrote:

> If a PCI System Error (SERR) is asserted it causes an NMI. If this NMI
> occurs while the CPU is in printk() then Xen may deadlock as
> pci_serr_error() calls console_force_unlock() which screws up the
> console lock.
> 
> I don't think removing the console_force_unlock() is sufficient as it
> looks like printk() is unsafe to call from NMI context.  We would like
> keep the diagnostic.  Does the following (untested) patch to defer the
> printk() to a softirq look like a valid approach?

Yes, if the NMI is non-fatal then console_force_unlock() cannot be used. So
your only other option is to defer the printk.

 -- Keir

> I also considered passing the NMI to dom0 (like other NMIs are) but I
> couldn't see how NMIs were handled in the current upstream pv-ops kernels.
> 
> diff -r e2722b24dc09 xen/arch/x86/traps.c
> --- a/xen/arch/x86/traps.c Thu Jan 26 17:43:31 2012 +0000
> +++ b/xen/arch/x86/traps.c Tue Jan 31 16:28:45 2012 +0000
> @@ -3173,6 +3173,11 @@ static void nmi_mce_softirq(void)
>      st->vcpu = NULL;
>  }
> 
> +static void pci_serr_softirq(void)
> +{
> +    printk("\n\nNMI - PCI system error (SERR)\n");
> +}
> +
>  void async_exception_cleanup(struct vcpu *curr)
>  {
>      int trap;
> @@ -3259,10 +3264,11 @@ static void nmi_dom0_report(unsigned int
> 
>  static void pci_serr_error(struct cpu_user_regs *regs)
>  {
> -    console_force_unlock();
> -    printk("\n\nNMI - PCI system error (SERR)\n");
> -
>      outb((inb(0x61) & 0x0f) | 0x04, 0x61); /* clear-and-disable the PCI
> SERR error line. */
> +
> +    /* Would like to print a diagnostic here but can't call printk()
> +       from NMI context -- raise a softirq instead. */
> +    raise_softirq(PCI_SERR_SOFTIRQ);
>  }
> 
>  static void io_check_error(struct cpu_user_regs *regs)
> @@ -3563,6 +3569,7 @@ void __init trap_init(void)
>      cpu_init();
> 
>      open_softirq(NMI_MCE_SOFTIRQ, nmi_mce_softirq);
> +    open_softirq(PCI_SERR_SOFTIRQ, pci_serr_softirq);
>  }
> 
>  long register_guest_nmi_callback(unsigned long address)
> diff -r e2722b24dc09 xen/include/asm-x86/softirq.h
> --- a/xen/include/asm-x86/softirq.h Thu Jan 26 17:43:31 2012 +0000
> +++ b/xen/include/asm-x86/softirq.h Tue Jan 31 16:28:45 2012 +0000
> @@ -6,6 +6,7 @@
>  #define VCPU_KICK_SOFTIRQ      (NR_COMMON_SOFTIRQS + 2)
> 
>  #define MACHINE_CHECK_SOFTIRQ  (NR_COMMON_SOFTIRQS + 3)
> -#define NR_ARCH_SOFTIRQS       4
> +#define PCI_SERR_SOFTIRQ       (NR_COMMON_SOFTIRQS + 4)
> +#define NR_ARCH_SOFTIRQS       5
> 
>  #endif /* __ASM_SOFTIRQ_H__ */
> 
> David
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

Re: PCI SERR NMI may cause Xen to deadlock

[George Dunlap]

On Tue, Jan 31, 2012 at 4:37 PM, David Vrabel <david.vrabel@citrix.com> wrote:
> If a PCI System Error (SERR) is asserted it causes an NMI. If this NMI
> occurs while the CPU is in printk() then Xen may deadlock as
> pci_serr_error() calls console_force_unlock() which screws up the
> console lock.
>
> I don't think removing the console_force_unlock() is sufficient as it
> looks like printk() is unsafe to call from NMI context.  We would like
> keep the diagnostic.  Does the following (untested) patch to defer the
> printk() to a softirq look like a valid approach?
>
> I also considered passing the NMI to dom0 (like other NMIs are) but I
> couldn't see how NMIs were handled in the current upstream pv-ops kernels.

Thanks David.  Unfortunately reproducing the error condition isn't as
consistent as I remember.  In 6 manual invocations, I got one instance
of PCI SERR, in which the patch worked correctly (i.e., the error
message and did not crash the host).

 -George

>
> diff -r e2722b24dc09 xen/arch/x86/traps.c
> --- a/xen/arch/x86/traps.c      Thu Jan 26 17:43:31 2012 +0000
> +++ b/xen/arch/x86/traps.c      Tue Jan 31 16:28:45 2012 +0000
> @@ -3173,6 +3173,11 @@ static void nmi_mce_softirq(void)
>     st->vcpu = NULL;
>  }
>
> +static void pci_serr_softirq(void)
> +{
> +    printk("\n\nNMI - PCI system error (SERR)\n");
> +}
> +
>  void async_exception_cleanup(struct vcpu *curr)
>  {
>     int trap;
> @@ -3259,10 +3264,11 @@ static void nmi_dom0_report(unsigned int
>
>  static void pci_serr_error(struct cpu_user_regs *regs)
>  {
> -    console_force_unlock();
> -    printk("\n\nNMI - PCI system error (SERR)\n");
> -
>     outb((inb(0x61) & 0x0f) | 0x04, 0x61); /* clear-and-disable the PCI
> SERR error line. */
> +
> +    /* Would like to print a diagnostic here but can't call printk()
> +       from NMI context -- raise a softirq instead. */
> +    raise_softirq(PCI_SERR_SOFTIRQ);
>  }
>
>  static void io_check_error(struct cpu_user_regs *regs)
> @@ -3563,6 +3569,7 @@ void __init trap_init(void)
>     cpu_init();
>
>     open_softirq(NMI_MCE_SOFTIRQ, nmi_mce_softirq);
> +    open_softirq(PCI_SERR_SOFTIRQ, pci_serr_softirq);
>  }
>
>  long register_guest_nmi_callback(unsigned long address)
> diff -r e2722b24dc09 xen/include/asm-x86/softirq.h
> --- a/xen/include/asm-x86/softirq.h     Thu Jan 26 17:43:31 2012 +0000
> +++ b/xen/include/asm-x86/softirq.h     Tue Jan 31 16:28:45 2012 +0000
> @@ -6,6 +6,7 @@
>  #define VCPU_KICK_SOFTIRQ      (NR_COMMON_SOFTIRQS + 2)
>
>  #define MACHINE_CHECK_SOFTIRQ  (NR_COMMON_SOFTIRQS + 3)
> -#define NR_ARCH_SOFTIRQS       4
> +#define PCI_SERR_SOFTIRQ       (NR_COMMON_SOFTIRQS + 4)
> +#define NR_ARCH_SOFTIRQS       5
>
>  #endif /* __ASM_SOFTIRQ_H__ */
>
> David
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel