[prepatch] removal of oops->printk deadlocks

[prepatch] removal of oops->printk deadlocks

[Andrew Morton]

This patch is designed to remove the things which can cause loss
of oops information due to the SMP deadlocks which can occur during
the oops processing itself.  It's fairly x86-specific, but the
infrastructure is in place for other SMP-capable architectures
to use.

It also changes the x86 NMI oopser so that it no longer shuts the
console up after the first NMI oops.  Instead, each CPU is allowed
to print out NMI diagnostics a single time per reboot.

This patch only addresses console-on-vgacon.  If the console is
on a serial device then there may still be deadlocks.  The serial
drivers can individually test `oops_in_progress' to avoid this.

The call to poke_blanked_console() is resurrected, but it's no
longer deadlocky.  This is fairly important - without this change
a machine could be wedged with oops diagnostics on its screen,
but the screen would remain blanked.

The patch addresses oopses, die(), NMI oopses and BUG().  The changes
to BUG() result in 3 kbytes less .rodata.

There's a tiny kernel module at
http://www.uow.edu.au/~andrewm/linux/ddeadlock.c which may be
used to test the NMI oopser changes.

Comments are welcome!


--- linux-2.4.0-test11-pre4/include/linux/kernel.h	Sun Oct 15 01:27:46 2000
+++ linux-akpm/include/linux/kernel.h	Tue Nov 14 21:02:18 2000
@@ -62,6 +62,7 @@
 
 asmlinkage int printk(const char * fmt, ...)
 	__attribute__ ((format (printf, 1, 2)));
+extern int oops_in_progress;
 
 #if DEBUG
 #define pr_debug(fmt,arg...) \
--- linux-2.4.0-test11-pre4/include/asm-i386/page.h	Sat Nov  4 16:22:49 2000
+++ linux-akpm/include/asm-i386/page.h	Tue Nov 14 21:02:18 2000
@@ -88,8 +88,9 @@
  * Tell the user there is some problem. Beep too, so we can
  * see^H^H^Hhear bugs in early bootup as well!
  */
+extern void do_BUG(const char *file, int line);
 #define BUG() do { \
-	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	do_BUG(__FILE__, __LINE__); \
 	__asm__ __volatile__(".byte 0x0f,0x0b"); \
 } while (0)
 
--- linux-2.4.0-test11-pre4/kernel/printk.c	Sat Nov  4 16:22:49 2000
+++ linux-akpm/kernel/printk.c	Tue Nov 14 21:02:18 2000
@@ -51,6 +51,7 @@
 static unsigned long logged_chars;
 struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int preferred_console = -1;
+int oops_in_progress;
 
 /*
  *	Setup a list of consoles. Called from init/main.c
@@ -260,6 +261,8 @@
 	static signed char msg_level = -1;
 	long flags;
 
+	if (oops_in_progress)
+		spin_lock_init(&console_lock);
 	spin_lock_irqsave(&console_lock, flags);
 	va_start(args, fmt);
 	i = vsprintf(buf + 3, fmt, args); /* hopefully i < sizeof(buf)-4 */
@@ -308,7 +311,8 @@
 			msg_level = -1;
 	}
 	spin_unlock_irqrestore(&console_lock, flags);
-	wake_up_interruptible(&log_wait);
+	if (!oops_in_progress)
+		wake_up_interruptible(&log_wait);
 	return i;
 }
 
--- linux-2.4.0-test11-pre4/arch/i386/mm/fault.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/arch/i386/mm/fault.c	Tue Nov 14 21:02:18 2000
@@ -77,17 +77,40 @@
 	return 0;
 }
 
-extern spinlock_t console_lock, timerlist_lock;
+#ifdef CONFIG_SMP
+extern unsigned volatile int global_irq_lock;
+#endif
 
 /*
  * Unlock any spinlocks which will prevent us from getting the
- * message out (timerlist_lock is aquired through the
- * console unblank code)
+ * message out and tell the printk/console paths that an emergency
+ * message is coming through
+ */
+void bust_spinlocks(int yes)
+{
+	if (yes) {
+		oops_in_progress = 1;
+#ifdef CONFIG_SMP
+		global_irq_lock = 0;	/* Many serial drivers do __global_cli() */
+#endif
+	} else {
+		oops_in_progress = 0;
+		/*
+		 * OK, the message is on the console.  Now we call printk()
+		 * without oops_in_progress set so that printk will give syslogd
+		 * a poke.  Hold onto your hats...
+		 */
+		printk("");
+	}		
+}
+
+/*
+ * Called from the BUG() macro.
  */
-void bust_spinlocks(void)
+void do_BUG(const char *file, int line)
 {
-	spin_lock_init(&console_lock);
-	spin_lock_init(&timerlist_lock);
+	bust_spinlocks(1);
+	printk("kernel BUG at %s:%d!\n", file, line);
 }
 
 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);
@@ -264,8 +287,7 @@
  * terminate things with extreme prejudice.
  */
 
-	bust_spinlocks();
-
+	bust_spinlocks(1);
 	if (address < PAGE_SIZE)
 		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 	else
@@ -283,6 +305,7 @@
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
 	die("Oops", regs, error_code);
+	bust_spinlocks(0);
 	do_exit(SIGKILL);
 
 /*
--- linux-2.4.0-test11-pre4/arch/i386/kernel/traps.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/arch/i386/kernel/traps.c	Tue Nov 14 21:11:10 2000
@@ -63,7 +63,7 @@
 struct desc_struct idt_table[256] __attribute__((__section__(".data.idt"))) = { {0, 0}, };
 
 extern int console_loglevel;
-extern void bust_spinlocks(void);
+extern void bust_spinlocks(int yes);
 
 static inline void console_silent(void)
 {
@@ -209,9 +209,10 @@
 {
 	console_verbose();
 	spin_lock_irq(&die_lock);
+	bust_spinlocks(1);
 	printk("%s: %04lx\n", str, err & 0xffff);
 	show_registers(regs);
-
+	bust_spinlocks(0);
 	spin_unlock_irq(&die_lock);
 	do_exit(SIGSEGV);
 }
@@ -414,8 +415,9 @@
 	 *  here too!]
 	 */
 
-	static unsigned int last_irq_sums [NR_CPUS],
+	static unsigned int	last_irq_sums [NR_CPUS],
 				alert_counter [NR_CPUS];
+	static unsigned char	printed_message [NR_CPUS];
 
 	/*
 	 * Since current-> is always on the stack, and we always switch
@@ -432,17 +434,19 @@
 		 */
 		alert_counter[cpu]++;
 		if (alert_counter[cpu] == 5*HZ) {
-			spin_lock(&nmi_print_lock);
-			/*
-			 * We are in trouble anyway, lets at least try
-			 * to get a message out.
-			 */
-			bust_spinlocks();
-			printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
-			show_registers(regs);
-			printk("console shuts up ...\n");
-			console_silent();
-			spin_unlock(&nmi_print_lock);
+			if (printed_message[cpu] == 0) {
+				printed_message[cpu] = 1;
+				spin_lock(&nmi_print_lock);
+				/*
+				 * We are in trouble anyway, lets at least try
+				 * to get a message out.
+				 */
+				bust_spinlocks(1);
+				printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
+				show_registers(regs);
+				spin_unlock(&nmi_print_lock);
+				bust_spinlocks(0);
+			}
 			do_exit(SIGSEGV);
 		}
 	} else {
--- linux-2.4.0-test11-pre4/arch/i386/kernel/i386_ksyms.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/arch/i386/kernel/i386_ksyms.c	Tue Nov 14 21:02:18 2000
@@ -157,3 +157,5 @@
 #ifdef CONFIG_X86_PAE
 EXPORT_SYMBOL(empty_zero_page);
 #endif
+
+EXPORT_SYMBOL(do_BUG);
--- linux-2.4.0-test11-pre4/drivers/char/console.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/drivers/char/console.c	Tue Nov 14 21:02:18 2000
@@ -2135,6 +2135,7 @@
 		}
 	}
 	set_cursor(currcons);
+	poke_blanked_console();
 
 quit:
 	clear_bit(0, &printing);
@@ -2661,7 +2662,7 @@
 		return;
 	}
 	console_timer.function = blank_screen;
-	if (blankinterval) {
+	if (blankinterval && !oops_in_progress) {
 		mod_timer(&console_timer, jiffies + blankinterval);
 	}
 
@@ -2683,14 +2684,18 @@
 
 void poke_blanked_console(void)
 {
-	del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
-	if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
-		return;
-	if (console_blanked) {
-		console_timer.function = unblank_screen_t;
-		mod_timer(&console_timer, jiffies);	/* Now */
-	} else if (blankinterval) {
-		mod_timer(&console_timer, jiffies + blankinterval);
+	if (oops_in_progress) {
+		unblank_screen();	/* Just do it */
+	} else {
+		del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
+		if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
+			return;
+		if (console_blanked) {
+			console_timer.function = unblank_screen_t;
+			mod_timer(&console_timer, jiffies);	/* Now */
+		} else if (blankinterval) {
+			mod_timer(&console_timer, jiffies + blankinterval);
+		}
 	}
 }
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Pawe³ Kot]

[-- Attachment #1: Type: TEXT/PLAIN, Size: 363 bytes --]


Hi,

The whole oops produced by the 2.4.0-test11pre4 with this patch is in the
attachment. Other system info is tehere as well.

Oops is produced when runnning big-tables test from mysql sql-bench.
It's reproducable. The machine is running only mysqld.

regards
pkot
-- 
mailto:pkot@linuxnews.pl
http://urtica.linuxnews.pl/~pkot/
http://newsreader.linuxnews.pl/

[-- Attachment #2: Type: TEXT/PLAIN, Size: 9090 bytes --]

ksymoops 2.3.4 on i686 2.4.0-test11.  Options used
     -V (default)
     -k /proc/ksyms (default)
     -l /proc/modules (default)
     -o /lib/modules/2.4.0-test11/ (default)
     -m /boot/System.map-2.4.0-test11 (default)

Warning: You did not tell me where to find symbol information.  I will
assume that the log matches the kernel and modules that are running
right now and I'll use the default options above for symbol resolution.
If the current kernel and/or modules do not match the log, you can get
more accurate output by telling me the kernel version and where to find
map, modules, ksyms etc.  ksymoops -h explains the options.

Error (regular_file): read_ksyms stat /proc/ksyms failed
No modules in ksyms, skipping objects
No ksyms, skipping lsmod
CPU:    0
EIP:    0010:[<c010b5ee>]
Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00000097
eax: c0285420   ebx: f6f08000   ecx: 00000000   edx: bf1fff2c
esi: bf1fff2c   edi: f6f09fa8   ebp: f6f09fa8   esp: f6f09f90
ds: 0018   es: 0018   ss: 0018
Process mysqld (pid: 2386, stackpage=f6f09000)
Stack: c02026f7 f6f08000 00000000 bf1fff24 fffffff2 00000000 f6f09fbc c0120404 
       00000952 00000000 bf1fff28 bf1ffe34 c010c66b 00000952 00000000 bf1fff28 
       00000000 bf1fff24 bf1ffe34 0000009c 0000002b 0000002b 0000009c 081313e4 
Call Trace: [<c02026f7>] [<c0120404>] [<c010c66b>] 
Code: 78 fb f0 ff 08 78 f3 c3 90 90 90 90 90 90 90 90 90 90 55 57 

>>EIP; c010b5ee <show_registers+12e/13c>   <=====
Trace; c02026f7 <stext_lock+474b/8d34>
Trace; c0120404 <sys_nanosleep+16c/178>
Trace; c010c66b <__global_cli+b/124>
Code;  c010b5ee <show_registers+12e/13c>
00000000 <_EIP>:
Code;  c010b5ee <show_registers+12e/13c>   <=====
   0:   78 fb                     js     fffffffd <_EIP+0xfffffffd> c010b5eb <show_registers+12b/13c>   <=====
Code;  c010b5f0 <show_registers+130/13c>
   2:   f0 ff 08                  lock decl (%eax)
Code;  c010b5f3 <show_registers+133/13c>
   5:   78 f3                     js     fffffffa <_EIP+0xfffffffa> c010b5e8 <show_registers+128/13c>
Code;  c010b5f5 <show_registers+135/13c>
   7:   c3                        ret    
Code;  c010b5f6 <show_registers+136/13c>
   8:   90                        nop    
Code;  c010b5f7 <show_registers+137/13c>
   9:   90                        nop    
Code;  c010b5f8 <show_registers+138/13c>
   a:   90                        nop    
Code;  c010b5f9 <show_registers+139/13c>
   b:   90                        nop    
Code;  c010b5fa <show_registers+13a/13c>
   c:   90                        nop    
Code;  c010b5fb <show_registers+13b/13c>
   d:   90                        nop    
Code;  c010b5fc <die+0/58>
   e:   90                        nop    
Code;  c010b5fd <die+1/58>
   f:   90                        nop    
Code;  c010b5fe <die+2/58>
  10:   90                        nop    
Code;  c010b5ff <die+3/58>
  11:   90                        nop    
Code;  c010b600 <die+4/58>
  12:   55                        push   %ebp
Code;  c010b601 <die+5/58>
  13:   57                        push   %edi

NMI Watchdog detected LOCKUP on CPU1, registers:
CPU:    1
EIP:    0010:[<c0202590>]
EFLAGS: 00000082
eax: c028e7ec   ebx: 00000282   ecx: c4df0000   edx: c028e7ec
esi: c011f550   edi: 00000000   ebp: c4d59ef4   esp: c4d59ef0
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d59000)
Stack: c4df0000 c0285c40 c0128592 c4df0000 00000000 00000020 00000000 c0285c40 
       f70082a0 f7c06800 00000086 c0124de5 c02a1220 00000020 c0124cc7 00000000 
       00000001 c0286040 00000020 0000000e c0124b6c c0286040 c02a1580 c0283800 
Call Trace: [<c0128592>] [<c0124de5>] [<c0124cc7>] [<c0124b6c>] [<c010e225>] [<c010a810>] [<c010a810>] 
       [<c010c72c>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a83d>] [<c010a8a2>] [<c0116c84>] [<c0124b6c>] 
       [<c010e225>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 ba cf f1 ff 80 bb 74 06 

>>EIP; c0202590 <stext_lock+45e4/8d34>   <=====
Trace; c0128592 <vmfree_area_pages+122/174>
Trace; c0124de5 <do_generic_file_read+279/544>
Trace; c0124cc7 <do_generic_file_read+15b/544>
Trace; c0124b6c <do_generic_file_read+0/544>
Trace; c010e225 <do_vm86_irq_handling+22d/2a8>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010c72c <__global_cli+cc/124>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a83d <setup_sigcontext+7d/128>
Trace; c010a8a2 <setup_sigcontext+e2/128>
Trace; c0116c84 <getrusage+168/1ec>
Trace; c0124b6c <do_generic_file_read+0/544>
Trace; c010e225 <do_vm86_irq_handling+22d/2a8>
Code;  c0202590 <stext_lock+45e4/8d34>
00000000 <_EIP>:
Code;  c0202590 <stext_lock+45e4/8d34>   <=====
   0:   80 3d 00 54 28 c0 00      cmpb   $0x0,0xc0285400   <=====
Code;  c0202597 <stext_lock+45eb/8d34>
   7:   f3 90                     repz nop 
Code;  c0202599 <stext_lock+45ed/8d34>
   9:   7e f5                     jle    0 <_EIP>
Code;  c020259b <stext_lock+45ef/8d34>
   b:   e9 ba cf f1 ff            jmp    fff1cfca <_EIP+0xfff1cfca> c011f55a <ptrace_writedata+2a/b0>
Code;  c02025a0 <stext_lock+45f4/8d34>
  10:   80 bb 74 06 00 00 00      cmpb   $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU2, registers:
CPU:    2
EIP:    0010:[<c02025c0>]
EFLAGS: 00000082
eax: 00000000   ebx: c0285480   ecx: 00000040   edx: 00000002
esi: c4d7a000   edi: c010a810   ebp: c4d7bfac   esp: c4d7bf78
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d7b000)
Stack: c010a810 c4d7a000 c010a810 c4d7a000 c4d7a000 c010a810 00000000 00000000 
       c4d70018 c0100018 00000002 c4d7a000 c0285480 00000000 c010a8ae 00000003 
       00000000 00000000 c0116c84 c0289373 c012293f 00000003 00000602 00000000 
Call Trace: [<c010a810>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a8ae>] [<c0116c84>] [<c012293f>] 
       [<c0117204>] [<c020de7d>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 b9 d1 f1 ff 80 bb 74 06 

>>EIP; c02025c0 <stext_lock+4614/8d34>   <=====
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c010a810 <setup_sigcontext+50/128>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a8ae <setup_sigcontext+ee/128>
Trace; c0116c84 <getrusage+168/1ec>
Trace; c012293f <do_mmap_pgoff+187/3f8>
Trace; c0117204 <exec_usermodehelper+3d4/3e0>
Trace; c020de7d <call_spurious_interrupt+67dd/77c8>
Code;  c02025c0 <stext_lock+4614/8d34>
00000000 <_EIP>:
Code;  c02025c0 <stext_lock+4614/8d34>   <=====
   0:   80 3d 00 54 28 c0 00      cmpb   $0x0,0xc0285400   <=====
Code;  c02025c7 <stext_lock+461b/8d34>
   7:   f3 90                     repz nop 
Code;  c02025c9 <stext_lock+461d/8d34>
   9:   7e f5                     jle    0 <_EIP>
Code;  c02025cb <stext_lock+461f/8d34>
   b:   e9 b9 d1 f1 ff            jmp    fff1d1c9 <_EIP+0xfff1d1c9> c011f789 <mod_timer+35/ec>
Code;  c02025d0 <stext_lock+4624/8d34>
  10:   80 bb 74 06 00 00 00      cmpb   $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU3, registers:
CPU:    3
EIP:    0010:[<c0202724>]
EFLAGS: 00000082
eax: f71d4000   ebx: 00000002   ecx: f71d4000   edx: c4d66400
esi: 00000021   edi: 00000000   ebp: f6f15eb0   esp: f6f15eac
ds: 0018   es: 0018   ss: 0018
Process mysqld (pid: 2385, stackpage=f6f15000)
Stack: f71d4000 00000000 c011c3fa f71d4000 00000021 00000082 c011c4bd 00000021 
       f6f15f04 f71d4000 00040001 00000000 f6f14000 00000021 c011c941 00000021 
       f6f15f04 f71d4000 f6f14000 f71d4000 f6f14000 c0285420 00000021 00000000 
Call Trace: [<c011c3fa>] [<c011c4bd>] [<c011c941>] [<c01f2401>] [<c014d6c9>] [<c013b101>] [<c01237bb>] 
       [<c0123b28>] [<c0123b7e>] [<c010c66b>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 d5 e3 f1 ff 80 3b 00 f3 

>>EIP; c0202724 <stext_lock+4778/8d34>   <=====
Trace; c011c3fa <do_adjtimex+c2/390>
Trace; c011c4bd <do_adjtimex+185/390>
Trace; c011c941 <tasklet_hi_action+69/7c>
Trace; c01f2401 <packet_ioctl+16d/340>
Trace; c014d6c9 <mem_read+21/108>
Trace; c013b101 <pipe_read+205/210>
Trace; c01237bb <insert_vm_struct+33/174>
Trace; c0123b28 <__remove_inode_page+54/60>
Trace; c0123b7e <invalidate_inode_pages+6/bc>
Trace; c010c66b <__global_cli+b/124>
Code;  c0202724 <stext_lock+4778/8d34>
00000000 <_EIP>:
Code;  c0202724 <stext_lock+4778/8d34>   <=====
   0:   80 3d 00 54 28 c0 00      cmpb   $0x0,0xc0285400   <=====
Code;  c020272b <stext_lock+477f/8d34>
   7:   f3 90                     repz nop 
Code;  c020272d <stext_lock+4781/8d34>
   9:   7e f5                     jle    0 <_EIP>
Code;  c020272f <stext_lock+4783/8d34>
   b:   e9 d5 e3 f1 ff            jmp    fff1e3e5 <_EIP+0xfff1e3e5> c0120b09 <clear_page_tables+71/84>
Code;  c0202734 <stext_lock+4788/8d34>
  10:   80 3b 00                  cmpb   $0x0,(%ebx)
Code;  c0202737 <stext_lock+478b/8d34>
  13:   f3 00 00                  repz add %al,(%eax)


1 warning and 1 error issued.  Results may not be reliable.

[-- Attachment #3: Type: TEXT/PLAIN, Size: 1688 bytes --]

processor	: 0
vendor_id	: GenuineIntel
cpu family	: 6
model		: 10
model name	: Pentium III (Cascades)
stepping	: 1
cpu MHz		: 697.000732
cache size	: 1024 KB
fdiv_bug	: no
hlt_bug		: no
sep_bug		: no
f00f_bug	: no
coma_bug	: no
fpu		: yes
fpu_exception	: yes
cpuid level	: 3
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips	: 1392.64

processor	: 1
vendor_id	: GenuineIntel
cpu family	: 6
model		: 10
model name	: Pentium III (Cascades)
stepping	: 1
cpu MHz		: 697.000732
cache size	: 1024 KB
fdiv_bug	: no
hlt_bug		: no
sep_bug		: no
f00f_bug	: no
coma_bug	: no
fpu		: yes
fpu_exception	: yes
cpuid level	: 3
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips	: 1392.64

processor	: 2
vendor_id	: GenuineIntel
cpu family	: 6
model		: 10
model name	: Pentium III (Cascades)
stepping	: 1
cpu MHz		: 697.000732
cache size	: 1024 KB
fdiv_bug	: no
hlt_bug		: no
sep_bug		: no
f00f_bug	: no
coma_bug	: no
fpu		: yes
fpu_exception	: yes
cpuid level	: 3
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips	: 1392.64

processor	: 3
vendor_id	: GenuineIntel
cpu family	: 6
model		: 10
model name	: Pentium III (Cascades)
stepping	: 1
cpu MHz		: 697.000732
cache size	: 1024 KB
fdiv_bug	: no
hlt_bug		: no
sep_bug		: no
f00f_bug	: no
coma_bug	: no
fpu		: yes
fpu_exception	: yes
cpuid level	: 3
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 pn mmx fxsr xmm
bogomips	: 1392.64


[-- Attachment #4: Type: TEXT/PLAIN, Size: 6344 bytes --]

00:00.0 Host bridge: Relience Computer CNB20HE (rev 21)
	Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:00.1 Host bridge: Relience Computer CNB20HE (rev 01)
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ >SERR- <PERR-
	Latency: 64 set, cache line size 08

00:00.2 Host bridge: Relience Computer: Unknown device 0006
	Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ >SERR- <PERR-

00:00.3 Host bridge: Relience Computer: Unknown device 0006
	Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort+ >SERR- <PERR-

00:04.0 Ethernet controller: Intel Corporation 82557 [Ethernet Pro 100] (rev 08)
	Subsystem: Intel Corporation 82559 Fast Ethernet LAN on Motherboard
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap+ 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 8 min, 56 max, 64 set, cache line size 08
	Interrupt: pin A routed to IRQ 31
	Region 0: Memory at e6100000 (32-bit, non-prefetchable) [size=4K]
	Region 1: I/O ports at 1400 [size=64]
	Region 2: Memory at e6000000 (32-bit, non-prefetchable) [size=1M]
	Expansion ROM at <unassigned> [disabled] [size=1M]
	Capabilities: [dc] Power Management version 2
		Flags: PMEClk- AuxPwr- DSI+ D1+ D2+ PME+
		Status: D0 PME-Enable- DSel=0 DScale=2 PME-

00:0c.0 VGA compatible controller: ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC] (rev 7a) (prog-if 00 [VGA])
	Subsystem: Intel Corporation: Unknown device 4b52
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B-
	Status: Cap+ 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 8 min, 66 set, cache line size 08
	Region 0: Memory at e7000000 (32-bit, non-prefetchable) [size=16M]
	Region 1: I/O ports at 1000 [size=256]
	Region 2: Memory at e6101000 (32-bit, non-prefetchable) [size=4K]
	Expansion ROM at <unassigned> [disabled] [size=128K]
	Capabilities: [5c] Power Management version 1
		Flags: PMEClk- AuxPwr- DSI- D1- D2- PME-
		Status: D0 PME-Enable- DSel=0 DScale=0 PME-

00:0f.0 ISA bridge: Relience Computer: Unknown device 0200 (rev 50)
	Subsystem: Relience Computer: Unknown device 0200
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 0 set

00:0f.1 IDE interface: Relience Computer: Unknown device 0211 (prog-if 8e [Master SecP SecO PriP])
	Control: I/O+ Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 64 set
	Region 2: I/O ports at 1458 [size=8]
	Region 3: I/O ports at 1450 [size=4]
	Region 4: I/O ports at 1440 [size=16]

00:0f.2 USB Controller: Relience Computer: Unknown device 0220 (rev 04) (prog-if 10 [OHCI])
	Subsystem: Relience Computer: Unknown device 0220
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 80 max, 64 set, cache line size 08
	Interrupt: pin A routed to IRQ 10
	Region 0: Memory at e6102000 (32-bit, non-prefetchable) [size=4K]

01:05.0 PCI Hot-plug controller: Compaq Computer Corporation: Unknown device a0f7 (rev 11)
	Subsystem: Intel Corporation: Unknown device 002c
	Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap+ 66Mhz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Interrupt: pin A routed to IRQ 30
	Region 0: Memory at ea800000 (32-bit, non-prefetchable) [size=256]
	Capabilities: [58] Message Signalled Interrupts: 64bit+ Queue=0/0 Enable-
		Address: 0000000000000000  Data: 0000
	Capabilities: [68] #07 [0000]

01:0a.0 PCI bridge: Intel Corporation: Unknown device 0964 (rev 02) (prog-if 00 [Normal decode])
	Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
	Status: Cap- 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 64 set, cache line size 08
	Bus: primary=01, secondary=04, subordinate=04, sec-latency=64
	I/O behind bridge: 00007000-00007fff
	Memory behind bridge: ea900000-ea9fffff
	Prefetchable memory behind bridge: f6000000-f60fffff
	BridgeCtl: Parity- SERR+ NoISA+ VGA- MAbort- >Reset- FastB2B-

01:0a.1 RAID bus controller: Mylex Corporation: Unknown device 0050 (rev 02)
	Subsystem: Mylex Corporation: Unknown device 0050
	Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV+ VGASnoop- ParErr- Stepping- SERR- FastB2B-
	Status: Cap+ 66Mhz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Latency: 64 set, cache line size 08
	Interrupt: pin A routed to IRQ 25
	Region 0: Memory at f0000000 (32-bit, prefetchable) [size=8K]
	Expansion ROM at <unassigned> [disabled] [size=32K]
	Capabilities: [80] Power Management version 2
		Flags: PMEClk- AuxPwr- DSI- D1- D2- PME-
		Status: D0 PME-Enable- DSel=0 DScale=0 PME-

07:07.0 PCI Hot-plug controller: Compaq Computer Corporation: Unknown device a0f7 (rev 11)
	Subsystem: Intel Corporation: Unknown device 002d
	Control: I/O- Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
	Status: Cap+ 66Mhz+ UDF- FastB2B- ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
	Interrupt: pin A routed to IRQ 30
	Region 0: Memory at f6100000 (32-bit, non-prefetchable) [size=256]
	Capabilities: [58] Message Signalled Interrupts: 64bit+ Queue=0/0 Enable-
		Address: 0000000000000000  Data: 0000
	Capabilities: [68] #07 [0000]


[-- Attachment #5: Type: TEXT/PLAIN, Size: 1150 bytes --]

00000000-0009ebff : System RAM
0009ec00-0009ffff : reserved
000a0000-000bffff : Video RAM area
000c0000-000c7fff : Video ROM
000c8000-000c8fff : Extension ROM
000c9000-000cbfff : Extension ROM
000f0000-000fffff : System ROM
00100000-e5feffff : System RAM
  00100000-0024bdbf : Kernel code
  0024bdc0-0026167f : Kernel data
e5ff0000-e5fffbff : ACPI Tables
e5fffc00-e5ffffff : ACPI Non-volatile Storage
e6000000-e60fffff : Intel Corporation 82557 [Ethernet Pro 100]
e6100000-e6100fff : Intel Corporation 82557 [Ethernet Pro 100]
  e6100000-e6100fff : eepro100
e6101000-e6101fff : ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC]
e6102000-e6102fff : PCI device 1166:0220 (ServerWorks)
e7000000-e7ffffff : ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC]
ea800000-ea8000ff : Compaq Computer Corporation PCI Hotplug Controller
ea900000-ea9fffff : PCI Bus #04
f0000000-f0001fff : PCI device 1069:0050 (Mylex Corporation)
f6000000-f60fffff : PCI Bus #04
f6100000-f61000ff : Compaq Computer Corporation PCI Hotplug Controller (#2)
fec00000-fec0ffff : reserved
fee00000-fee00fff : reserved
ffc00000-ffffffff : reserved

[-- Attachment #6: Type: TEXT/PLAIN, Size: 603 bytes --]

0000-001f : dma1
0020-003f : pic1
0040-005f : timer
0060-006f : keyboard
0080-008f : dma page reg
00a0-00bf : pic2
00c0-00df : dma2
00f0-00ff : fpu
01f0-01f7 : ide0
02f8-02ff : serial(auto)
03c0-03df : vga+
03f6-03f6 : ide0
03f8-03ff : serial(auto)
0cf8-0cff : PCI conf1
1000-10ff : ATI Technologies Inc 3D Rage IIC 215IIC [Mach64 GT IIC]
1400-143f : Intel Corporation 82557 [Ethernet Pro 100]
  1400-143f : eepro100
1440-144f : PCI device 1166:0211 (ServerWorks)
1450-1453 : PCI device 1166:0211 (ServerWorks)
1458-145f : PCI device 1166:0211 (ServerWorks)
7000-7fff : PCI Bus #04

Re: [prepatch] removal of oops->printk deadlocks

[Ingo Molnar]

On Tue, 14 Nov 2000, Andrew Morton wrote:

> It also changes the x86 NMI oopser so that it no longer shuts the
> console up after the first NMI oops.  Instead, each CPU is allowed
> to print out NMI diagnostics a single time per reboot.

this is not how the NMI oopser works. It does *not* shut down the console
- you can still see everything in 'dmesg'. If you want to see it on the
console again, you can do 'dmesg -n 8'. Adding a 'once per reboot'
restriction is unreasonable - there are user-space applications that can
be terminated via the NMI-oopser safely. In 99% of the cases it's the
first oops that counts, and most people do not have serial console set up,
so writing the first oops down from screen is important.

	Ingo

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Andrew Morton]

Ingo Molnar wrote:
> 
> On Tue, 14 Nov 2000, Andrew Morton wrote:
> 
> > It also changes the x86 NMI oopser so that it no longer shuts the
> > console up after the first NMI oops.  Instead, each CPU is allowed
> > to print out NMI diagnostics a single time per reboot.
> 
> this is not how the NMI oopser works. It does *not* shut down the console
> - you can still see everything in 'dmesg'. If you want to see it on the
> console again, you can do 'dmesg -n 8'. Adding a 'once per reboot'
> restriction is unreasonable - there are user-space applications that can
> be terminated via the NMI-oopser safely.

The patch addresses that - it still calls do_exit() for every NMI timeout.

> In 99% of the cases it's the
> first oops that counts, and most people do not have serial console set up,
> so writing the first oops down from screen is important.

Good point.  Problem is, we're getting some reported lockups in which we
need to know what the other CPUs are doing (the other 1%).  If a critical
spinlock is stuck on, we don't get to type `dmesg'.

So...  In this updated patch the console_silent() call remains
as you designed it, but the nmi_watchdog kernel boot parameter has
been overloaded so that

	nmi_watchdog=2

will now cause _all_ NMI oops messages to be displayed on the console.

Is this OK?



--- linux-2.4.0-test11-pre4/include/linux/kernel.h	Sun Oct 15 01:27:46 2000
+++ linux-akpm/include/linux/kernel.h	Wed Nov 15 20:31:29 2000
@@ -62,6 +62,7 @@
 
 asmlinkage int printk(const char * fmt, ...)
 	__attribute__ ((format (printf, 1, 2)));
+extern int oops_in_progress;
 
 #if DEBUG
 #define pr_debug(fmt,arg...) \
--- linux-2.4.0-test11-pre4/include/asm-i386/page.h	Sat Nov  4 16:22:49 2000
+++ linux-akpm/include/asm-i386/page.h	Wed Nov 15 20:31:29 2000
@@ -88,8 +88,9 @@
  * Tell the user there is some problem. Beep too, so we can
  * see^H^H^Hhear bugs in early bootup as well!
  */
+extern void do_BUG(const char *file, int line);
 #define BUG() do { \
-	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	do_BUG(__FILE__, __LINE__); \
 	__asm__ __volatile__(".byte 0x0f,0x0b"); \
 } while (0)
 
--- linux-2.4.0-test11-pre4/kernel/printk.c	Sat Nov  4 16:22:49 2000
+++ linux-akpm/kernel/printk.c	Wed Nov 15 20:31:29 2000
@@ -51,6 +51,7 @@
 static unsigned long logged_chars;
 struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int preferred_console = -1;
+int oops_in_progress;
 
 /*
  *	Setup a list of consoles. Called from init/main.c
@@ -260,6 +261,8 @@
 	static signed char msg_level = -1;
 	long flags;
 
+	if (oops_in_progress)
+		spin_lock_init(&console_lock);
 	spin_lock_irqsave(&console_lock, flags);
 	va_start(args, fmt);
 	i = vsprintf(buf + 3, fmt, args); /* hopefully i < sizeof(buf)-4 */
@@ -308,7 +311,8 @@
 			msg_level = -1;
 	}
 	spin_unlock_irqrestore(&console_lock, flags);
-	wake_up_interruptible(&log_wait);
+	if (!oops_in_progress)
+		wake_up_interruptible(&log_wait);
 	return i;
 }
 
--- linux-2.4.0-test11-pre4/arch/i386/mm/fault.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/arch/i386/mm/fault.c	Wed Nov 15 20:31:29 2000
@@ -77,17 +77,40 @@
 	return 0;
 }
 
-extern spinlock_t console_lock, timerlist_lock;
+#ifdef CONFIG_SMP
+extern unsigned volatile int global_irq_lock;
+#endif
 
 /*
  * Unlock any spinlocks which will prevent us from getting the
- * message out (timerlist_lock is aquired through the
- * console unblank code)
+ * message out and tell the printk/console paths that an emergency
+ * message is coming through
+ */
+void bust_spinlocks(int yes)
+{
+	if (yes) {
+		oops_in_progress = 1;
+#ifdef CONFIG_SMP
+		global_irq_lock = 0;	/* Many serial drivers do __global_cli() */
+#endif
+	} else {
+		oops_in_progress = 0;
+		/*
+		 * OK, the message is on the console.  Now we call printk()
+		 * without oops_in_progress set so that printk will give syslogd
+		 * a poke.  Hold onto your hats...
+		 */
+		printk("");
+	}		
+}
+
+/*
+ * Called from the BUG() macro.
  */
-void bust_spinlocks(void)
+void do_BUG(const char *file, int line)
 {
-	spin_lock_init(&console_lock);
-	spin_lock_init(&timerlist_lock);
+	bust_spinlocks(1);
+	printk("kernel BUG at %s:%d!\n", file, line);
 }
 
 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);
@@ -264,8 +287,7 @@
  * terminate things with extreme prejudice.
  */
 
-	bust_spinlocks();
-
+	bust_spinlocks(1);
 	if (address < PAGE_SIZE)
 		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 	else
@@ -283,6 +305,7 @@
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
 	die("Oops", regs, error_code);
+	bust_spinlocks(0);
 	do_exit(SIGKILL);
 
 /*
--- linux-2.4.0-test11-pre4/arch/i386/kernel/traps.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/arch/i386/kernel/traps.c	Wed Nov 15 21:57:49 2000
@@ -63,7 +63,7 @@
 struct desc_struct idt_table[256] __attribute__((__section__(".data.idt"))) = { {0, 0}, };
 
 extern int console_loglevel;
-extern void bust_spinlocks(void);
+extern void bust_spinlocks(int yes);
 
 static inline void console_silent(void)
 {
@@ -209,9 +209,10 @@
 {
 	console_verbose();
 	spin_lock_irq(&die_lock);
+	bust_spinlocks(1);
 	printk("%s: %04lx\n", str, err & 0xffff);
 	show_registers(regs);
-
+	bust_spinlocks(0);
 	spin_unlock_irq(&die_lock);
 	do_exit(SIGSEGV);
 }
@@ -414,7 +415,7 @@
 	 *  here too!]
 	 */
 
-	static unsigned int last_irq_sums [NR_CPUS],
+	static unsigned int	last_irq_sums [NR_CPUS],
 				alert_counter [NR_CPUS];
 
 	/*
@@ -437,12 +438,15 @@
 			 * We are in trouble anyway, lets at least try
 			 * to get a message out.
 			 */
-			bust_spinlocks();
+			bust_spinlocks(1);
 			printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
 			show_registers(regs);
-			printk("console shuts up ...\n");
-			console_silent();
+			if (nmi_watchdog <= 1) {
+				printk("Console shuts up.  Use `dmesg' or boot with `nmi_watchdog=2' to see more logs.\n");
+				console_silent();
+			}
 			spin_unlock(&nmi_print_lock);
+			bust_spinlocks(0);
 			do_exit(SIGSEGV);
 		}
 	} else {
--- linux-2.4.0-test11-pre4/arch/i386/kernel/i386_ksyms.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/arch/i386/kernel/i386_ksyms.c	Wed Nov 15 20:31:29 2000
@@ -157,3 +157,5 @@
 #ifdef CONFIG_X86_PAE
 EXPORT_SYMBOL(empty_zero_page);
 #endif
+
+EXPORT_SYMBOL(do_BUG);
--- linux-2.4.0-test11-pre4/drivers/char/console.c	Mon Nov 13 18:23:49 2000
+++ linux-akpm/drivers/char/console.c	Wed Nov 15 20:31:29 2000
@@ -2135,6 +2135,7 @@
 		}
 	}
 	set_cursor(currcons);
+	poke_blanked_console();
 
 quit:
 	clear_bit(0, &printing);
@@ -2661,7 +2662,7 @@
 		return;
 	}
 	console_timer.function = blank_screen;
-	if (blankinterval) {
+	if (blankinterval && !oops_in_progress) {
 		mod_timer(&console_timer, jiffies + blankinterval);
 	}
 
@@ -2683,14 +2684,18 @@
 
 void poke_blanked_console(void)
 {
-	del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
-	if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
-		return;
-	if (console_blanked) {
-		console_timer.function = unblank_screen_t;
-		mod_timer(&console_timer, jiffies);	/* Now */
-	} else if (blankinterval) {
-		mod_timer(&console_timer, jiffies + blankinterval);
+	if (oops_in_progress) {
+		unblank_screen();	/* Just do it */
+	} else {
+		del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
+		if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
+			return;
+		if (console_blanked) {
+			console_timer.function = unblank_screen_t;
+			mod_timer(&console_timer, jiffies);	/* Now */
+		} else if (blankinterval) {
+			mod_timer(&console_timer, jiffies + blankinterval);
+		}
 	}
 }
 
--- linux-2.4.0-test11-pre4/Documentation/nmi_watchdog.txt	Thu Aug 24 21:07:14 2000
+++ linux-akpm/Documentation/nmi_watchdog.txt	Wed Nov 15 21:51:17 2000
@@ -24,8 +24,21 @@
 cannot even accept NMI interrupts, or the crash has made the kernel
 unable to print messages.
 
-NOTE: currently the NMI-oopser is enabled unconditionally on x86 SMP
-boxes.
+The NMI-oopser defaults to `1' (enabled) on x86 SMP boxes.
+
+The NMI-oopser only displays the first oops message on the console. This
+is to prevent vital information from scrolling off the scren if more
+errors occur.  You can find additional information (such as NMI oops
+information from other CPUs) by looking in your system logs or with
+the `dmesg' command.  
+
+If your system is crashing hard and preventing you from running `dmesg' 
+you can tell the NMI oopser to display all NMI oops messages (not just
+the first one) by setting
+
+	nmi_watchdog=2
+
+on the kernel boot command line or in lilo.conf, as shown above.
 
 [ feel free to send bug reports, suggestions and patches to
   Ingo Molnar <mingo@redhat.com> or the Linux SMP mailing
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Andrew Morton]

"Pawe³ Kot" wrote:
> 
> Hi,
> 
> The whole oops produced by the 2.4.0-test11pre4 with this patch is in the
> attachment. Other system info is tehere as well.
> 
> Oops is produced when runnning big-tables test from mysql sql-bench.
> It's reproducable. The machine is running only mysqld.

Thank you.  Quad 700 meg Xeons?  Nice machine, that.

The NMI oops changes are working well. We can see three CPUs are
stuck in different places spinning on the same lock with interrupts
disabled, but unfortunately something went wrong with your `ksymoops'
run.

Could you please rerun ksymoops, and make sure that you use the
correct symbol file?  Take the `System.map' from the directory
where you built the kernel and run

	ksymoops -m System.map < log_file

-
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Pawe³ Kot]

[-- Attachment #1: Type: TEXT/PLAIN, Size: 1186 bytes --]

On Wed, 15 Nov 2000, Andrew Morton wrote:

> "Paweł Kot" wrote:
> >
> > Hi,
> >
> > The whole oops produced by the 2.4.0-test11pre4 with this patch is in the
> > attachment. Other system info is tehere as well.
> >
> > Oops is produced when runnning big-tables test from mysql sql-bench.
> > It's reproducable. The machine is running only mysqld.
>
> Thank you.  Quad 700 meg Xeons?  Nice machine, that.

Yes. 54 seconds for compiling kernel is quite good result :-)

> The NMI oops changes are working well. We can see three CPUs are
> stuck in different places spinning on the same lock with interrupts
> disabled, but unfortunately something went wrong with your `ksymoops'
> run.
>
> Could you please rerun ksymoops, and make sure that you use the
> correct symbol file?  Take the `System.map' from the directory
> where you built the kernel and run
>
> 	ksymoops -m System.map < log_file

In attachment. But don't beat me. I think I found the oops reason.
/dev/shm was not mounted. After mounting it I couldn't get an oops yet.
Could this be a reason?

regards and thanks for help
pkot
-- 
mailto:pkot@linuxnews.pl
http://urtica.linuxnews.pl/~pkot/
http://newsreader.linuxnews.pl/

[-- Attachment #2: Type: TEXT/PLAIN, Size: 8560 bytes --]

ksymoops 2.3.4 on i686 2.4.0-test11.  Options used
     -V (default)
     -k /proc/ksyms (default)
     -l /proc/modules (default)
     -o /lib/modules/2.4.0-test11/ (default)
     -m /usr/src/linux/System.map (specified)

Error (regular_file): read_ksyms stat /proc/ksyms failed
No modules in ksyms, skipping objects
No ksyms, skipping lsmod
CPU:    0
EIP:    0010:[<c010b5ee>]
Using defaults from ksymoops -t elf32-i386 -a i386
EFLAGS: 00000097
eax: c0285420   ebx: f6f08000   ecx: 00000000   edx: bf1fff2c
esi: bf1fff2c   edi: f6f09fa8   ebp: f6f09fa8   esp: f6f09f90
ds: 0018   es: 0018   ss: 0018
Process mysqld (pid: 2386, stackpage=f6f09000)
Stack: c02026f7 f6f08000 00000000 bf1fff24 fffffff2 00000000 f6f09fbc c0120404 
       00000952 00000000 bf1fff28 bf1ffe34 c010c66b 00000952 00000000 bf1fff28 
       00000000 bf1fff24 bf1ffe34 0000009c 0000002b 0000002b 0000009c 081313e4 
Call Trace: [<c02026f7>] [<c0120404>] [<c010c66b>] 
Code: 78 fb f0 ff 08 78 f3 c3 90 90 90 90 90 90 90 90 90 90 55 57 

>>EIP; c010b5ee <__read_lock_failed+6/18>   <=====
Trace; c02026f7 <stext_lock+86b/8c14>
Trace; c0120404 <sys_sched_setscheduler+14/18>
Trace; c010c66b <system_call+33/38>
Code;  c010b5ee <__read_lock_failed+6/18>
00000000 <_EIP>:
Code;  c010b5ee <__read_lock_failed+6/18>   <=====
   0:   78 fb                     js     fffffffd <_EIP+0xfffffffd> c010b5eb <__read_lock_failed+3/18>   <=====
Code;  c010b5f0 <__read_lock_failed+8/18>
   2:   f0 ff 08                  lock decl (%eax)
Code;  c010b5f3 <__read_lock_failed+b/18>
   5:   78 f3                     js     fffffffa <_EIP+0xfffffffa> c010b5e8 <__read_lock_failed+0/18>
Code;  c010b5f5 <__read_lock_failed+d/18>
   7:   c3                        ret    
Code;  c010b5f6 <__read_lock_failed+e/18>
   8:   90                        nop    
Code;  c010b5f7 <__read_lock_failed+f/18>
   9:   90                        nop    
Code;  c010b5f8 <__read_lock_failed+10/18>
   a:   90                        nop    
Code;  c010b5f9 <__read_lock_failed+11/18>
   b:   90                        nop    
Code;  c010b5fa <__read_lock_failed+12/18>
   c:   90                        nop    
Code;  c010b5fb <__read_lock_failed+13/18>
   d:   90                        nop    
Code;  c010b5fc <__read_lock_failed+14/18>
   e:   90                        nop    
Code;  c010b5fd <__read_lock_failed+15/18>
   f:   90                        nop    
Code;  c010b5fe <__read_lock_failed+16/18>
  10:   90                        nop    
Code;  c010b5ff <__read_lock_failed+17/18>
  11:   90                        nop    
Code;  c010b600 <copy_siginfo_to_user+0/a8>
  12:   55                        push   %ebp
Code;  c010b601 <copy_siginfo_to_user+1/a8>
  13:   57                        push   %edi

NMI Watchdog detected LOCKUP on CPU1, registers:
CPU:    1
EIP:    0010:[<c0202590>]
EFLAGS: 00000082
eax: c028e7ec   ebx: 00000282   ecx: c4df0000   edx: c028e7ec
esi: c011f550   edi: 00000000   ebp: c4d59ef4   esp: c4d59ef0
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d59000)
Stack: c4df0000 c0285c40 c0128592 c4df0000 00000000 00000020 00000000 c0285c40 
       f70082a0 f7c06800 00000086 c0124de5 c02a1220 00000020 c0124cc7 00000000 
       00000001 c0286040 00000020 0000000e c0124b6c c0286040 c02a1580 c0283800 
Call Trace: [<c0128592>] [<c0124de5>] [<c0124cc7>] [<c0124b6c>] [<c010e225>] [<c010a810>] [<c010a810>] 
       [<c010c72c>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a83d>] [<c010a8a2>] [<c0116c84>] [<c0124b6c>] 
       [<c010e225>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 ba cf f1 ff 80 bb 74 06 

>>EIP; c0202590 <stext_lock+704/8c14>   <=====
Trace; c0128592 <timer_bh+256/2b0>
Trace; c0124de5 <bh_action+4d/b0>
Trace; c0124cc7 <tasklet_hi_action+4f/7c>
Trace; c0124b6c <do_softirq+5c/8c>
Trace; c010e225 <do_IRQ+e5/f4>
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c010c72c <ret_from_intr+0/20>
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a83d <default_idle+2d/34>
Trace; c010a8a2 <cpu_idle+3e/54>
Trace; c0116c84 <start_secondary+20/24>
Trace; c0124b6c <do_softirq+5c/8c>
Trace; c010e225 <do_IRQ+e5/f4>
Code;  c0202590 <stext_lock+704/8c14>
00000000 <_EIP>:
Code;  c0202590 <stext_lock+704/8c14>   <=====
   0:   80 3d 00 54 28 c0 00      cmpb   $0x0,0xc0285400   <=====
Code;  c0202597 <stext_lock+70b/8c14>
   7:   f3 90                     repz nop 
Code;  c0202599 <stext_lock+70d/8c14>
   9:   7e f5                     jle    0 <_EIP>
Code;  c020259b <stext_lock+70f/8c14>
   b:   e9 ba cf f1 ff            jmp    fff1cfca <_EIP+0xfff1cfca> c011f55a <process_timeout+a/60>
Code;  c02025a0 <stext_lock+714/8c14>
  10:   80 bb 74 06 00 00 00      cmpb   $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU2, registers:
CPU:    2
EIP:    0010:[<c02025c0>]
EFLAGS: 00000082
eax: 00000000   ebx: c0285480   ecx: 00000040   edx: 00000002
esi: c4d7a000   edi: c010a810   ebp: c4d7bfac   esp: c4d7bf78
ds: 0018   es: 0018   ss: 0018
Process swapper (pid: 0, stackpage=c4d7b000)
Stack: c010a810 c4d7a000 c010a810 c4d7a000 c4d7a000 c010a810 00000000 00000000 
       c4d70018 c0100018 00000002 c4d7a000 c0285480 00000000 c010a8ae 00000003 
       00000000 00000000 c0116c84 c0289373 c012293f 00000003 00000602 00000000 
Call Trace: [<c010a810>] [<c010a810>] [<c010a810>] [<c0100018>] [<c010a8ae>] [<c0116c84>] [<c012293f>] 
       [<c0117204>] [<c020de7d>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 b9 d1 f1 ff 80 bb 74 06 

>>EIP; c02025c0 <stext_lock+734/8c14>   <=====
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c010a810 <default_idle+0/34>
Trace; c0100018 <startup_32+18/cc>
Trace; c010a8ae <cpu_idle+4a/54>
Trace; c0116c84 <start_secondary+20/24>
Trace; c012293f <printk+18f/19c>
Trace; c0117204 <do_boot_cpu+544/6b0>
Trace; c020de7d <call_spurious_interrupt+2a1d/77c8>
Code;  c02025c0 <stext_lock+734/8c14>
00000000 <_EIP>:
Code;  c02025c0 <stext_lock+734/8c14>   <=====
   0:   80 3d 00 54 28 c0 00      cmpb   $0x0,0xc0285400   <=====
Code;  c02025c7 <stext_lock+73b/8c14>
   7:   f3 90                     repz nop 
Code;  c02025c9 <stext_lock+73d/8c14>
   9:   7e f5                     jle    0 <_EIP>
Code;  c02025cb <stext_lock+73f/8c14>
   b:   e9 b9 d1 f1 ff            jmp    fff1d1c9 <_EIP+0xfff1d1c9> c011f789 <schedule+b1/648>
Code;  c02025d0 <stext_lock+744/8c14>
  10:   80 bb 74 06 00 00 00      cmpb   $0x0,0x674(%ebx)

NMI Watchdog detected LOCKUP on CPU3, registers:
CPU:    3
EIP:    0010:[<c0202724>]
EFLAGS: 00000082
eax: f71d4000   ebx: 00000002   ecx: f71d4000   edx: c4d66400
esi: 00000021   edi: 00000000   ebp: f6f15eb0   esp: f6f15eac
ds: 0018   es: 0018   ss: 0018
Process mysqld (pid: 2385, stackpage=f6f15000)
Stack: f71d4000 00000000 c011c3fa f71d4000 00000021 00000082 c011c4bd 00000021 
       f6f15f04 f71d4000 00040001 00000000 f6f14000 00000021 c011c941 00000021 
       f6f15f04 f71d4000 f6f14000 f71d4000 f6f14000 c0285420 00000021 00000000 
Call Trace: [<c011c3fa>] [<c011c4bd>] [<c011c941>] [<c01f2401>] [<c014d6c9>] [<c013b101>] [<c01237bb>] 
       [<c0123b28>] [<c0123b7e>] [<c010c66b>] 
Code: 80 3d 00 54 28 c0 00 f3 90 7e f5 e9 d5 e3 f1 ff 80 3b 00 f3 

>>EIP; c0202724 <stext_lock+898/8c14>   <=====
Trace; c011c3fa <deliver_signal+4a/88>
Trace; c011c4bd <send_sig_info+85/b0>
Trace; c011c941 <do_notify_parent+c9/e0>
Trace; c01f2401 <unix_release_sock+279/294>
Trace; c014d6c9 <dput+19/154>
Trace; c013b101 <fput+79/dc>
Trace; c01237bb <exit_notify+16b/2ac>
Trace; c0123b28 <do_exit+22c/274>
Trace; c0123b7e <sys_exit+e/10>
Trace; c010c66b <system_call+33/38>
Code;  c0202724 <stext_lock+898/8c14>
00000000 <_EIP>:
Code;  c0202724 <stext_lock+898/8c14>   <=====
   0:   80 3d 00 54 28 c0 00      cmpb   $0x0,0xc0285400   <=====
Code;  c020272b <stext_lock+89f/8c14>
   7:   f3 90                     repz nop 
Code;  c020272d <stext_lock+8a1/8c14>
   9:   7e f5                     jle    0 <_EIP>
Code;  c020272f <stext_lock+8a3/8c14>
   b:   e9 d5 e3 f1 ff            jmp    fff1e3e5 <_EIP+0xfff1e3e5> c0120b09 <wake_up_process+9/60>
Code;  c0202734 <stext_lock+8a8/8c14>
  10:   80 3b 00                  cmpb   $0x0,(%ebx)
Code;  c0202737 <stext_lock+8ab/8c14>
  13:   f3 00 00                  repz add %al,(%eax)


1 error issued.  Results may not be reliable.

Re: [prepatch] removal of oops->printk deadlocks

[Andrew Morton]

"Pawe³ Kot" wrote:
> 
> In attachment. But don't beat me. I think I found the oops reason.
> /dev/shm was not mounted. After mounting it I couldn't get an oops yet.
> Could this be a reason?

No.

>>EIP; c010b5ee <__read_lock_failed+6/18>   <=====
Trace; c02026f7 <stext_lock+86b/8c14>
Trace; c0120404 <sys_sched_setscheduler+14/18>
                 ^^^^^^^^^^^^^^^^^^^^^^

This was fixed in test11-pre5.   Please retest with that kernel.

Thanks again.

-
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Ingo Molnar]

On Wed, 15 Nov 2000, Andrew Morton wrote:

> [...] Problem is, we're getting some reported lockups in which we need
> to know what the other CPUs are doing (the other 1%). If a critical
> spinlock is stuck on, we don't get to type `dmesg'.

i know - but we cannot please everyone, so i went for the 99% :-)

> So...  In this updated patch the console_silent() call remains
> as you designed it, but the nmi_watchdog kernel boot parameter has
> been overloaded so that
> 
> 	nmi_watchdog=2
> 
> will now cause _all_ NMI oops messages to be displayed on the console.
> 
> Is this OK?

this collides with the UP-IOAPIC path's use of nmi_watchdog == 2 ...

i'd rather suggest a cleaner, "nmi_watchdog=2,verbose" (default: silent)
type of boot parameter, it looks like the NMI watchdog needs more
parameters.

	Ingo

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Andrew Morton]

Ingo Molnar wrote:
> 
> > So...  In this updated patch the console_silent() call remains
> > as you designed it, but the nmi_watchdog kernel boot parameter has
> > been overloaded so that
> >
> >       nmi_watchdog=2
> >
> > will now cause _all_ NMI oops messages to be displayed on the console.
> >
> > Is this OK?
> 
> this collides with the UP-IOAPIC path's use of nmi_watchdog == 2 ...
>
> i'd rather suggest a cleaner, "nmi_watchdog=2,verbose" (default: silent)
> type of boot parameter, it looks like the NMI watchdog needs more
> parameters.


OK, so the options are now:

	nmi_watchdog=0
	nmi_watchdog=1
	nmi_watchdog=1,verbose


Third time lucky?



--- linux-2.4.0-test11-pre5/include/linux/kernel.h	Sun Oct 15 01:27:46 2000
+++ linux-akpm/include/linux/kernel.h	Wed Nov 15 23:25:48 2000
@@ -62,6 +62,7 @@
 
 asmlinkage int printk(const char * fmt, ...)
 	__attribute__ ((format (printf, 1, 2)));
+extern int oops_in_progress;
 
 #if DEBUG
 #define pr_debug(fmt,arg...) \
--- linux-2.4.0-test11-pre5/include/asm-i386/page.h	Sat Nov  4 16:22:49 2000
+++ linux-akpm/include/asm-i386/page.h	Wed Nov 15 23:27:32 2000
@@ -88,8 +88,9 @@
  * Tell the user there is some problem. Beep too, so we can
  * see^H^H^Hhear bugs in early bootup as well!
  */
+extern void do_BUG(const char *file, int line);
 #define BUG() do { \
-	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	do_BUG(__FILE__, __LINE__); \
 	__asm__ __volatile__(".byte 0x0f,0x0b"); \
 } while (0)
 
--- linux-2.4.0-test11-pre5/kernel/printk.c	Sat Nov  4 16:22:49 2000
+++ linux-akpm/kernel/printk.c	Wed Nov 15 23:25:48 2000
@@ -51,6 +51,7 @@
 static unsigned long logged_chars;
 struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
 static int preferred_console = -1;
+int oops_in_progress;
 
 /*
  *	Setup a list of consoles. Called from init/main.c
@@ -251,6 +252,9 @@
 	return do_syslog(type, buf, len);
 }
 
+/*
+ * printk("") may be used to wake up klogd
+ */
 asmlinkage int printk(const char *fmt, ...)
 {
 	va_list args;
@@ -260,6 +264,8 @@
 	static signed char msg_level = -1;
 	long flags;
 
+	if (oops_in_progress)
+		spin_lock_init(&console_lock);
 	spin_lock_irqsave(&console_lock, flags);
 	va_start(args, fmt);
 	i = vsprintf(buf + 3, fmt, args); /* hopefully i < sizeof(buf)-4 */
@@ -308,7 +314,8 @@
 			msg_level = -1;
 	}
 	spin_unlock_irqrestore(&console_lock, flags);
-	wake_up_interruptible(&log_wait);
+	if (!oops_in_progress)
+		wake_up_interruptible(&log_wait);
 	return i;
 }
 
--- linux-2.4.0-test11-pre5/arch/i386/mm/fault.c	Wed Nov 15 23:00:16 2000
+++ linux-akpm/arch/i386/mm/fault.c	Thu Nov 16 00:14:37 2000
@@ -77,17 +77,37 @@
 	return 0;
 }
 
-extern spinlock_t console_lock, timerlist_lock;
+#ifdef CONFIG_SMP
+extern unsigned volatile int global_irq_lock;
+#endif
 
 /*
  * Unlock any spinlocks which will prevent us from getting the
- * message out (timerlist_lock is aquired through the
- * console unblank code)
+ * message out and tell the printk/console paths that an emergency
+ * message is coming through
  */
-void bust_spinlocks(void)
+void bust_spinlocks(int yes)
+{
+	if (yes) {
+		oops_in_progress = 1;
+#ifdef CONFIG_SMP
+		global_irq_lock = 0;	/* Many serial drivers do __global_cli() */
+#endif
+	} else {
+		oops_in_progress = 0;
+		/*
+		 * OK, the message is on the console.  Now we call printk()
+		 * without oops_in_progress set so that printk will give klogd
+		 * a poke.  Hold onto your hats...
+		 */
+		printk("");
+	}		
+}
+
+void do_BUG(const char *file, int line)
 {
-	spin_lock_init(&console_lock);
-	spin_lock_init(&timerlist_lock);
+	bust_spinlocks(1);
+	printk("kernel BUG at %s:%d!\n", file, line);
 }
 
 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);
@@ -264,8 +284,7 @@
  * terminate things with extreme prejudice.
  */
 
-	bust_spinlocks();
-
+	bust_spinlocks(1);
 	if (address < PAGE_SIZE)
 		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 	else
@@ -283,6 +302,7 @@
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
 	die("Oops", regs, error_code);
+	bust_spinlocks(0);
 	do_exit(SIGKILL);
 
 /*
--- linux-2.4.0-test11-pre5/arch/i386/kernel/traps.c	Wed Nov 15 23:00:16 2000
+++ linux-akpm/arch/i386/kernel/traps.c	Wed Nov 15 23:49:00 2000
@@ -63,7 +63,7 @@
 struct desc_struct idt_table[256] __attribute__((__section__(".data.idt"))) = { {0, 0}, };
 
 extern int console_loglevel;
-extern void bust_spinlocks(void);
+extern void bust_spinlocks(int yes);
 
 static inline void console_silent(void)
 {
@@ -209,9 +209,10 @@
 {
 	console_verbose();
 	spin_lock_irq(&die_lock);
+	bust_spinlocks(1);
 	printk("%s: %04lx\n", str, err & 0xffff);
 	show_registers(regs);
-
+	bust_spinlocks(0);
 	spin_unlock_irq(&die_lock);
 	do_exit(SIGSEGV);
 }
@@ -386,10 +387,14 @@
 #if CONFIG_X86_IO_APIC
 
 int nmi_watchdog = 1;
+static int nmi_watchdog_verbose;
 
 static int __init setup_nmi_watchdog(char *str)
 {
-        get_option(&str, &nmi_watchdog);
+ 	if (get_option(&str, &nmi_watchdog) == 2) {
+		if (strcmp(str, "verbose") == 0)
+			nmi_watchdog_verbose = 1;
+	}
         return 1;
 }
 
@@ -414,7 +419,7 @@
 	 *  here too!]
 	 */
 
-	static unsigned int last_irq_sums [NR_CPUS],
+	static unsigned int	last_irq_sums [NR_CPUS],
 				alert_counter [NR_CPUS];
 
 	/*
@@ -437,12 +442,15 @@
 			 * We are in trouble anyway, lets at least try
 			 * to get a message out.
 			 */
-			bust_spinlocks();
+			bust_spinlocks(1);
 			printk("NMI Watchdog detected LOCKUP on CPU%d, registers:\n", cpu);
 			show_registers(regs);
-			printk("console shuts up ...\n");
-			console_silent();
+			if (nmi_watchdog_verbose == 0) {
+				printk("Console shuts up. Run dmesg or boot with `nmi_watchdog=1,verbose' for more logs\n");
+				console_silent();
+			}
 			spin_unlock(&nmi_print_lock);
+			bust_spinlocks(0);
 			do_exit(SIGSEGV);
 		}
 	} else {
--- linux-2.4.0-test11-pre5/arch/i386/kernel/i386_ksyms.c	Wed Nov 15 23:00:16 2000
+++ linux-akpm/arch/i386/kernel/i386_ksyms.c	Wed Nov 15 23:25:48 2000
@@ -157,3 +157,5 @@
 #ifdef CONFIG_X86_PAE
 EXPORT_SYMBOL(empty_zero_page);
 #endif
+
+EXPORT_SYMBOL(do_BUG);
--- linux-2.4.0-test11-pre5/drivers/char/console.c	Wed Nov 15 23:00:16 2000
+++ linux-akpm/drivers/char/console.c	Wed Nov 15 23:25:48 2000
@@ -2135,6 +2135,7 @@
 		}
 	}
 	set_cursor(currcons);
+	poke_blanked_console();
 
 quit:
 	clear_bit(0, &printing);
@@ -2661,7 +2662,7 @@
 		return;
 	}
 	console_timer.function = blank_screen;
-	if (blankinterval) {
+	if (blankinterval && !oops_in_progress) {
 		mod_timer(&console_timer, jiffies + blankinterval);
 	}
 
@@ -2683,14 +2684,18 @@
 
 void poke_blanked_console(void)
 {
-	del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
-	if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
-		return;
-	if (console_blanked) {
-		console_timer.function = unblank_screen_t;
-		mod_timer(&console_timer, jiffies);	/* Now */
-	} else if (blankinterval) {
-		mod_timer(&console_timer, jiffies + blankinterval);
+	if (oops_in_progress) {
+		unblank_screen();	/* Just do it */
+	} else {
+		del_timer(&console_timer);	/* Can't use _sync here: called from tasklet */
+		if (vt_cons[fg_console]->vc_mode == KD_GRAPHICS)
+			return;
+		if (console_blanked) {
+			console_timer.function = unblank_screen_t;
+			mod_timer(&console_timer, jiffies);	/* Now */
+		} else if (blankinterval) {
+			mod_timer(&console_timer, jiffies + blankinterval);
+		}
 	}
 }
 
--- linux-2.4.0-test11-pre5/Documentation/nmi_watchdog.txt	Thu Aug 24 21:07:14 2000
+++ linux-akpm/Documentation/nmi_watchdog.txt	Wed Nov 15 23:25:48 2000
@@ -24,8 +24,21 @@
 cannot even accept NMI interrupts, or the crash has made the kernel
 unable to print messages.
 
-NOTE: currently the NMI-oopser is enabled unconditionally on x86 SMP
-boxes.
+The NMI-oopser defaults to `1' (enabled) on x86 SMP boxes.
+
+The NMI-oopser only displays the first oops message on the console. This
+is to prevent vital information from scrolling off the scren if more
+errors occur.  You can find additional information (such as NMI oops
+information from other CPUs) by looking in your system logs or with
+the `dmesg' command.  
+
+If your system is crashing hard and preventing you from running `dmesg' 
+you can tell the NMI oopser to display all NMI oops messages (not just
+the first one) by setting
+
+	nmi_watchdog=1,verbose
+
+on the kernel boot command line or in lilo.conf, as shown above.
 
 [ feel free to send bug reports, suggestions and patches to
   Ingo Molnar <mingo@redhat.com> or the Linux SMP mailing
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/

Re: [prepatch] removal of oops->printk deadlocks

[Ingo Molnar]

On Thu, 16 Nov 2000, Andrew Morton wrote:

> OK, so the options are now:
> 
> 	nmi_watchdog=0
> 	nmi_watchdog=1
> 	nmi_watchdog=1,verbose

looks good for me!

	Ingo

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
Please read the FAQ at http://www.tux.org/lkml/