Re: [Xenomai-core] Frozen timer IRQ - now traced with kgdb :)

[Jan Kiszka <jan.kiszka@domain.hid>]

[-- Attachment #1.1: Type: text/plain, Size: 2000 bytes --]

Philippe Gerum wrote:
> Philippe Gerum wrote:
>> Jan Kiszka wrote:
>>
>>>
>>> BTW, that trace hacking reminds me that we should really think about
>>> making a kernel debugger run. I recently noticed that latest kgdb
>>> applied with a single failing hunk on top of ipipe (2.6.15, x86). Maybe
>>> it is just about making kgdb's irq-locks ipipe-aware and bypassing the
>>> ipipe for int3 and the serial IRQ (so that ipipe can be debugged as
>>> well) and catching the relevant exceptions. Hmm, the debugger seems to
>>> get initialised in the "early" stage. Is this before or after ipipe
>>> setup?
>>>
>>
>> It depends. If "kgdbwait" is set in the bootargs to halt the kernel
>> waiting for the remote GDB to connect to the target, kgdb starts
>> before the ipipe. Otherwise, it's a late init, and kgdb starts after
>> the ipipe is fully initialized.
>>
> 
> Basically, kgdb could start before the i-pipe as soon as a breakpoint is
> hit before the latter is enabled in init/main.c.
> 

Yep, I dug deeper meanwhile and also came across this.

I already have a trivial hack running here. The most tricky part for me
was to learn quilt, but now I start to love it :). Here is a snapshot
series for 2.6.15.5:

<kgdb series from CVS>
prepare-ipipe-x86.patch
adeos-ipipe-2.6.15-i386-1.2-01.patch
kgdb-ipipe-x86.patch

I'm currently wondering if it makes sense to register a kgdb domain and
"officially" capture all involved IRQs and events. So far the serial
line IRQ is hard-coded (should be retrieved from some internal kgdb
structure later). Anyway, it seems to work quite well, I'm currently
stepping through a network IRQ at ipipe-level.


While playing with this tool a bit, displaying the the ipipe structures,
and thinking about the original problem again, I wondered what could
cause a temporary (as I think to found out now) stalled xeno domain
without locking up the system? Some irq-lock leaks at driver level (i.e.
inside our own code)?

Jan

[-- Attachment #1.2: kgdb-ipipe-x86.patch --]
[-- Type: text/plain, Size: 3997 bytes --]

Index: linux-2.6.15.5/arch/i386/kernel/entry.S
===================================================================
--- linux-2.6.15.5.orig/arch/i386/kernel/entry.S	2006-04-07 16:53:39.000000000 +0200
+++ linux-2.6.15.5/arch/i386/kernel/entry.S	2006-04-07 16:53:40.000000000 +0200
@@ -194,7 +194,7 @@
 .previous
 
 
-ENTRY(ret_from_fork)
+KPROBE_ENTRY(ret_from_fork)
 	STI_COND_HW
 	pushl %eax
 	call schedule_tail
@@ -582,7 +582,7 @@
 	PUSH_XCODE(do_simd_coprocessor_error)
 	jmp error_code
 
-ENTRY(device_not_available)
+KPROBE_ENTRY(device_not_available)
 	pushl $-1			# mark this as an int
 	SAVE_ALL
 	DIVERT_EXCEPTION(device_not_available)
@@ -767,7 +767,7 @@
 	jmp error_code
 #endif
 
-ENTRY(spurious_interrupt_bug)
+KPROBE_ENTRY(spurious_interrupt_bug)
 	pushl $0
 	PUSH_XCODE(do_spurious_interrupt_bug)
 	jmp error_code
Index: linux-2.6.15.5/kernel/kgdb.c
===================================================================
--- linux-2.6.15.5.orig/kernel/kgdb.c	2006-04-07 16:30:51.000000000 +0200
+++ linux-2.6.15.5/kernel/kgdb.c	2006-04-07 16:57:35.000000000 +0200
@@ -740,7 +740,7 @@
 	unsigned long flags;
 	int processor;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	processor = smp_processor_id();
 	kgdb_info[processor].debuggerinfo = regs;
 	kgdb_info[processor].task = current;
@@ -770,7 +770,7 @@
 	/* Signal the master processor that we are done */
 	atomic_set(&procindebug[processor], 0);
 	spin_unlock(&slavecpulocks[processor]);
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 #endif
 
@@ -1033,7 +1033,7 @@
 	 * Interrupts will be restored by the 'trap return' code, except when
 	 * single stepping.
 	 */
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 
 	/* Hold debugger_active */
 	procid = smp_processor_id();
@@ -1056,7 +1056,7 @@
 	if (atomic_read(&cpu_doing_single_step) != -1 &&
 	    atomic_read(&cpu_doing_single_step) != procid) {
 		atomic_set(&debugger_active, 0);
-		local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 		goto acquirelock;
 	}
 
@@ -1556,7 +1556,7 @@
 kgdb_restore:
 	/* Free debugger_active */
 	atomic_set(&debugger_active, 0);
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 
 	return error;
 }
@@ -1925,9 +1925,9 @@
 	if (!kgdb_connected || atomic_read(&debugger_active) != 0)
 		return 0;
 	if ((code == SYS_RESTART) || (code == SYS_HALT) || (code == SYS_POWER_OFF)){
-		local_irq_save(flags);
+		local_irq_save_hw(flags);
 		put_packet("X00");
-		local_irq_restore(flags);
+		local_irq_restore_hw(flags);
 	}
 	return NOTIFY_DONE;
 }		
@@ -1942,9 +1942,9 @@
 	if (!kgdb_connected || atomic_read(&debugger_active) != 0)
 		return;
 
-	local_irq_save(flags);
+	local_irq_save_hw(flags);
 	kgdb_msg_write(s, count);
-	local_irq_restore(flags);
+	local_irq_restore_hw(flags);
 }
 
 static struct console kgdbcons = {
Index: linux-2.6.15.5/arch/i386/kernel/ipipe-root.c
===================================================================
--- linux-2.6.15.5.orig/arch/i386/kernel/ipipe-root.c	2006-04-07 16:53:39.000000000 +0200
+++ linux-2.6.15.5/arch/i386/kernel/ipipe-root.c	2006-04-07 17:48:00.000000000 +0200
@@ -111,6 +111,15 @@
 
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
+#ifdef CONFIG_KGDB
+static struct ipipe_domain kgdb_domain;
+
+static void kgdb_domain_entry(void)
+{
+	
+}
+#endif /* CONFIG_KGDB */
+
 /* __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
    interrupts are off, and secondary CPUs are still lost in space. */
 
@@ -248,6 +257,10 @@
 	ipipe_root_domain->irqs[IPIPE_SERVICE_IPI2].control &= ~IPIPE_SYSTEM_MASK;
 	ipipe_root_domain->irqs[IPIPE_SERVICE_IPI3].control &= ~IPIPE_SYSTEM_MASK;
 #endif	/* CONFIG_X86_LOCAL_APIC */
+
+#ifdef CONFIG_KGDB
+	ipipe_control_irq(4, 0, IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK|IPIPE_SYSTEM_MASK);
+#endif /* CONFIG_KGDB */
 }
 
 static inline void __fixup_if(struct pt_regs *regs)

[-- Attachment #1.3: prepare-ipipe-x86.patch --]
[-- Type: text/plain, Size: 838 bytes --]

Index: linux-2.6.15.5/arch/i386/kernel/entry.S
===================================================================
--- linux-2.6.15.5.orig/arch/i386/kernel/entry.S	2006-04-07 16:42:54.000000000 +0200
+++ linux-2.6.15.5/arch/i386/kernel/entry.S	2006-04-07 16:47:23.000000000 +0200
@@ -123,7 +123,7 @@
 .previous
 
 
-KPROBE_ENTRY(ret_from_fork)
+ENTRY(ret_from_fork)
 	pushl %eax
 	call schedule_tail
 	GET_THREAD_INFO(%ebp)
@@ -470,7 +470,7 @@
 	pushl $do_simd_coprocessor_error
 	jmp error_code
 
-KPROBE_ENTRY(device_not_available)
+ENTRY(device_not_available)
 	pushl $-1			# mark this as an int
 	SAVE_ALL
 	movl %cr0, %eax
@@ -652,7 +652,7 @@
 	jmp error_code
 #endif
 
-KPROBE_ENTRY(spurious_interrupt_bug)
+ENTRY(spurious_interrupt_bug)
 	pushl $0
 	pushl $do_spurious_interrupt_bug
 	jmp error_code

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 250 bytes --]