All of lore.kernel.org
 help / color / mirror / Atom feed
* [Xenomai-core] ipip tracer porting for blackfin
@ 2006-09-30  7:59 Li Yi (Adam)
  2006-10-01 22:16 ` [Xenomai-core] Re: [Adeos-main] " Jan Kiszka
  0 siblings, 1 reply; 9+ messages in thread
From: Li Yi (Adam) @ 2006-09-30  7:59 UTC (permalink / raw)
  To: adeos-main, xenomai


[-- Attachment #1.1: Type: text/plain, Size: 475 bytes --]

Hi,

Attached ipipe-tracer ported to blackfin. The patch is based on
http://download.gna.org/adeos/patches/v2.6/blackfin/adeos-ipipe-bf53x-R06R1RC2-1.4-03.patch.
But it requires a little hack to Blackfin GCC-R06R1 to  make the "-pg"
option work.  The latest release of bfin-gcc (06r2) has supported this
feature. And I will update the ipipe tracer patch to the latest kernel
release (r06r2).

Also attached the ipipe tracer log.
Hoping for your comments.

Best Regards,

-Yi

[-- Attachment #1.2: Type: text/html, Size: 643 bytes --]

[-- Attachment #2: adeos-bfin-tracer.patch --]
[-- Type: text/x-patch, Size: 45416 bytes --]

diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/arch/blackfin/kernel/ipipe-mcount.S linux-2.6.x-06r1-tracer/arch/blackfin/kernel/ipipe-mcount.S
--- linux-2.6.x-06r1-ipipe-orig/arch/blackfin/kernel/ipipe-mcount.S	1970-01-01 08:00:00.000000000 +0800
+++ linux-2.6.x-06r1-tracer/arch/blackfin/kernel/ipipe-mcount.S	2006-08-25 10:59:46.000000000 +0800
@@ -0,0 +1,33 @@
+/*
+ * linux/arch/blackfin/ipipe-mcount.S
+ *
+ * Copyright (C) 2006 Analog Devices Inc.
+ */
+
+#include <linux/config.h>
+#include <linux/linkage.h>
+#include <asm/blackfin.h>
+
+.text
+
+ENTRY(_mcount)
+	[--sp] = ( r7:0, p5:0 );
+	[--sp] = ASTAT;
+	p1.L = _ipipe_trace_enable;
+	p1.H = _ipipe_trace_enable;
+	r7 = [p1];
+	CC = r7==0;
+	if CC jump out;	
+	link 0x10;
+	r0 = 0x0;	
+	[sp + 0xc] = r0; /* v */
+	r0 = 0x0;	/* type: IPIPE_TRACE_FN */
+	r1 = rets;
+	p0 = [fp];	/* p0: Prior FP */
+	r2 = [p0 + 4];	/* r2: Prior RETS */
+	call ___ipipe_trace;
+	unlink;
+out:
+	ASTAT = [sp++];
+	( r7:0, p5:0 ) = [sp++];
+	rts;
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/arch/blackfin/kernel/ipipe-root.c linux-2.6.x-06r1-tracer/arch/blackfin/kernel/ipipe-root.c
--- linux-2.6.x-06r1-ipipe-orig/arch/blackfin/kernel/ipipe-root.c	2006-08-17 14:13:40.000000000 +0800
+++ linux-2.6.x-06r1-tracer/arch/blackfin/kernel/ipipe-root.c	2006-08-23 16:51:13.000000000 +0800
@@ -276,3 +276,10 @@
 EXPORT_SYMBOL(__ipipe_core_clock);
 EXPORT_SYMBOL(__ipipe_freq_scale);
 EXPORT_SYMBOL(show_stack);
+
+#ifdef CONFIG_IPIPE_TRACE
+void notrace mcount(void);
+EXPORT_SYMBOL(mcount);
+#endif /* CONFIG_IPIPE_TRACE */
+
+
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/arch/blackfin/kernel/Makefile linux-2.6.x-06r1-tracer/arch/blackfin/kernel/Makefile
--- linux-2.6.x-06r1-ipipe-orig/arch/blackfin/kernel/Makefile	2006-08-17 14:13:40.000000000 +0800
+++ linux-2.6.x-06r1-tracer/arch/blackfin/kernel/Makefile	2006-08-22 17:53:35.000000000 +0800
@@ -8,4 +8,5 @@
 obj-$(CONFIG_MODULES)		+= module.o
 obj-$(CONFIG_BFIN_DMA_5XX)	+= bfin_dma_5xx.o
 obj-$(CONFIG_IPIPE)		+= ipipe-core.o ipipe-root.o
+obj-$(CONFIG_IPIPE_TRACE)	+= ipipe-mcount.o
 obj-$(CONFIG_DUAL_CORE_TEST_MODULE)	+= dualcore_test.o
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/include/asm-blackfin/ipipe.h linux-2.6.x-06r1-tracer/include/asm-blackfin/ipipe.h
--- linux-2.6.x-06r1-ipipe-orig/include/asm-blackfin/ipipe.h	2006-08-17 14:13:40.000000000 +0800
+++ linux-2.6.x-06r1-tracer/include/asm-blackfin/ipipe.h	2006-08-23 14:50:30.000000000 +0800
@@ -108,6 +108,7 @@
 
 #define ipipe_cpu_freq()	__ipipe_core_clock
 #define ipipe_tsc2ns(_t)	(((unsigned long)(_t)) * __ipipe_freq_scale)
+#define ipipe_tsc2us(_t)	(ipipe_tsc2ns(_t) / 1000 + 1)
 
 /* Private interface -- Internal use only */
 
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/include/asm-blackfin/system.h linux-2.6.x-06r1-tracer/include/asm-blackfin/system.h
--- linux-2.6.x-06r1-ipipe-orig/include/asm-blackfin/system.h	2006-08-17 14:13:40.000000000 +0800
+++ linux-2.6.x-06r1-tracer/include/asm-blackfin/system.h	2006-08-28 15:23:16.000000000 +0800
@@ -65,6 +65,8 @@
 #endif
 #define irqs_enabled_from_flags_hw(x) ((x) != __all_masked_irq_flags)
 
+#define local_test_iflag_hw(x) irqs_enabled_from_flags_hw(x)
+
 #define local_save_flags(x)				\
 do {							\
     (x) = __ipipe_test_root()?__all_masked_irq_flags:irq_flags;		\
@@ -80,6 +82,68 @@
 #define local_irq_enable()	__ipipe_unstall_root()
 #define irqs_disabled()		__ipipe_test_root()
 
+#define local_save_flags_hw(x) asm volatile ("cli %0;"     \
+					     "sti %0;"     \
+					     :"=d"(x):);
+
+#defin	irqs_disabled_hw()		\
+({					\
+	unsigned long flags;		\
+	local_save_flags_hw(flags);	\
+	!irqs_enabled_from_flags_hw(flags);	\
+})
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+#include <linux/ipipe_trace.h>
+
+#define local_irq_disable_hw() do { \
+	int _tmp_dummy;		\
+	if (!irqs_disabled_hw()) { \
+		ipipe_trace_begin(0x80000000); \
+	} \
+	__asm__ __volatile__ ( "cli %0;":"=d" (_tmp_dummy):);	\
+} while (0)
+
+#define local_irq_enable_hw() do { \
+	if (irqs_disabled_hw()) { \
+		ipipe_trace_end(0x80000000); \
+	} \
+	__asm__ __volatile__ ("sti %0;"::"d"(irq_flags)); \
+} while (0)
+
+#define local_irq_save_hw(x) do { \
+	__save_and_cli_hw(x);	\
+	if (local_test_iflag_hw(x)) { \
+		ipipe_trace_begin(0x80000001); \
+	} \
+} while (0)
+
+#define local_irq_restore_hw(x) do { \
+	if (local_test_iflag_hw(x)) { \
+		ipipe_trace_end(0x80000001); \
+		local_irq_enable_hw_notrace();	\
+	} \
+} while (0)
+
+#define local_irq_disable_hw_notrace()  \
+do {					\
+	int _tmp_dummy;			\
+	__asm__ __volatile__ ( "cli %0;":"=d" (_tmp_dummy):);	\
+} while(0)
+
+#define local_irq_enable_hw_notrace() \
+	__asm__ __volatile__ ("sti %0;"::"d"(irq_flags));
+
+#define local_irq_save_hw_notrace(x) __save_and_cli_hw(x) 
+
+#define local_irq_restore_hw_notrace(x) do {			\
+	if (irqs_enabled_from_flags_hw(x))		\
+		local_irq_enable_hw_notrace();			\
+} while (0)
+
+#else /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
 #define local_irq_enable_hw() 		\
 do {					\
 	__asm__ __volatile__ (		\
@@ -95,9 +159,6 @@
 		:"=d" (_tmp_dummy):);	\
 } while(0)
 
-#define local_save_flags_hw(x) asm volatile ("cli %0;"     \
-					     "sti %0;"     \
-					     :"=d"(x):);
 #define local_irq_restore_hw(x) do {			\
 	if (irqs_enabled_from_flags_hw(x))		\
 		local_irq_enable_hw();			\
@@ -105,12 +166,12 @@
 
 #define local_irq_save_hw(x) __save_and_cli_hw(x)
 
-#define	irqs_disabled_hw()		\
-({					\
-	unsigned long flags;		\
-	local_save_flags_hw(flags);	\
-	!irqs_enabled_from_flags_hw(flags);	\
-})
+#define local_irq_disable_hw_notrace	local_irq_disable_hw
+#define local_irq_enable_hw_notrace	local_irq_enable_hw
+#define local_irq_save_hw_notrace	local_irq_save_hw
+#define local_irq_restore_hw_notrace	local_irq_restore_hw
+
+#endif  /* CONFIG_IPIPE_TRACE_IRQSOFF */
 
 #else /* !CONFIG_IPIPE */
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/kernel/ipipe/core.c linux-2.6.x-06r1-tracer/kernel/ipipe/core.c
--- linux-2.6.x-06r1-ipipe-orig/kernel/ipipe/core.c	2006-08-17 14:13:40.000000000 +0800
+++ linux-2.6.x-06r1-tracer/kernel/ipipe/core.c	2006-08-23 15:01:33.000000000 +0800
@@ -1012,7 +1012,7 @@
 {
 	ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
 	create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
-	__ipipe_init_trace_proc();
+	__ipipe_init_tracer();
 	__ipipe_add_domain_proc(ipipe_root_domain);
 }
 
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/kernel/ipipe/Kconfig linux-2.6.x-06r1-tracer/kernel/ipipe/Kconfig
--- linux-2.6.x-06r1-ipipe-orig/kernel/ipipe/Kconfig	2006-08-17 14:13:40.000000000 +0800
+++ linux-2.6.x-06r1-tracer/kernel/ipipe/Kconfig	2006-08-22 15:13:00.000000000 +0800
@@ -4,3 +4,43 @@
 	---help---
 	  Activate this option if you want the interrupt pipeline to be
 	  compiled in.
+
+config IPIPE_TRACE
+ 	bool "Latency Tracing"
+	depends on IPIPE && PROC_FS
+	select FRAME_POINTER
+	default n
+	---help---
+	  Activate this option if you want to use per-function tracing of
+	  the kernel. The tracer will collect data via instrumentation
+	  features like the one below or with the help of explicite calls
+	  of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+	  in-kernel tracing API. The collected data and runtime control
+	  is available via /proc/ipipe/trace/*.
+
+config IPIPE_TRACE_IRQSOFF
+	bool "Trace IRQs-off times"
+	depends on IPIPE_TRACE
+	default y
+	---help---
+	  Activate this option if I-pipe shall trace the longest path
+	  with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+	int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+	range 10 18
+	default 14
+	depends on IPIPE_TRACE
+	---help---
+	  The number of trace points to hold tracing data for each
+	  trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+	bool "Use vmalloc'ed trace buffer"
+	depends on IPIPE_TRACE
+	---help---
+	  Instead of reserving static kernel data, the required buffer
+	  is allocated via vmalloc during boot-up when this option is
+	  enabled. This can help to start systems that are low on memory,
+	  but it slightly degrades overall performance. Try this option
+	  when a traced kernel hangs unexpectedly at boot time.
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/kernel/ipipe/tracer.c linux-2.6.x-06r1-tracer/kernel/ipipe/tracer.c
--- linux-2.6.x-06r1-ipipe-orig/kernel/ipipe/tracer.c	1970-01-01 08:00:00.000000000 +0800
+++ linux-2.6.x-06r1-tracer/kernel/ipipe/tracer.c	2006-08-25 18:03:43.000000000 +0800
@@ -0,0 +1,1243 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ *               2005, 2006 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/ipipe_trace.h>
+#include <asm/uaccess.h>
+
+#define IPIPE_TRACE_PATHS           4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE        0
+#define IPIPE_DEFAULT_MAX           1
+#define IPIPE_DEFAULT_FROZEN        2
+
+#define IPIPE_TRACE_POINTS          (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point)        ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE     10
+#define IPIPE_DEFAULT_POST_TRACE    10
+#define IPIPE_DEFAULT_BACK_TRACE    30
+
+#define IPIPE_DELAY_NOTE            1000  /* in nanoseconds */
+#define IPIPE_DELAY_WARN            10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK         0x0001
+#define IPIPE_TFLG_NMI_HIT          0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ   0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF        0x0100
+#define IPIPE_TFLG_FREEZING         0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT    10   /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK     0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT   12   /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS    3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+	(point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+	((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+
+struct ipipe_trace_point{
+	short type;
+	short flags;
+	unsigned long eip;
+	unsigned long parent_eip;
+	unsigned long v;
+	unsigned long long timestamp;
+};
+
+struct ipipe_trace_path{
+	volatile int flags;
+	int dump_lock; /* separated from flags due to cross-cpu access */
+	int trace_pos; /* next point to fill */
+	int begin, end; /* finalised path begin and end */
+	int post_trace; /* non-zero when in post-trace phase */
+	unsigned long long length; /* max path length in cycles */
+	unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+	unsigned long nmi_saved_parent_eip;
+	unsigned long nmi_saved_v;
+	struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+	IPIPE_TRACE_FUNC = 0,
+	IPIPE_TRACE_BEGIN,
+	IPIPE_TRACE_END,
+	IPIPE_TRACE_FREEZE,
+	IPIPE_TRACE_SPECIAL,
+	IPIPE_TRACE_PID,
+};
+
+#define IPIPE_TYPE_MASK             0x0007
+#define IPIPE_TYPE_BITS             3
+
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+#define IPIPE_DEFAULT_TRACE_STATE   0
+
+static struct ipipe_trace_path *trace_paths[NR_CPUS];
+
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+#define IPIPE_DEFAULT_TRACE_STATE   1
+
+static struct ipipe_trace_path trace_paths[NR_CPUS][IPIPE_TRACE_PATHS] =
+	{ [0 ... NR_CPUS-1] =
+		{ [0 ... IPIPE_TRACE_PATHS-1] =
+			{ .begin = -1, .end = -1 }
+		}
+	};
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = IPIPE_DEFAULT_TRACE_STATE;
+
+static int active_path[NR_CPUS] =
+	{ [0 ... NR_CPUS-1] = IPIPE_DEFAULT_ACTIVE };
+static int max_path[NR_CPUS] =
+	{ [0 ... NR_CPUS-1] = IPIPE_DEFAULT_MAX };
+static int frozen_path[NR_CPUS] =
+	{ [0 ... NR_CPUS-1] = IPIPE_DEFAULT_FROZEN };
+static ipipe_spinlock_t global_path_lock = IPIPE_SPIN_LOCK_UNLOCKED;
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 0;
+
+static DECLARE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+static struct ipipe_trace_path *panic_path;
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point, int cpu_id)
+{
+	struct list_head *pos;
+	int i = 0;
+
+	list_for_each_prev(pos, &__ipipe_pipeline) {
+		struct ipipe_domain *ipd =
+			list_entry(pos, struct ipipe_domain, p_link);
+
+		if (test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpu_id].status))
+			point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+		if (ipd == ipipe_percpu_domain[cpu_id])
+			point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT;
+
+		if (++i > IPIPE_TFLG_DOMSTATE_BITS)
+			break;
+	}
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu_id)
+{
+	int new_active = old;
+	struct ipipe_trace_path *tp;
+
+	do {
+		if (++new_active == IPIPE_TRACE_PATHS)
+			new_active = 0;
+		tp = &trace_paths[cpu_id][new_active];
+	} while ((new_active == max_path[cpu_id]) ||
+	         (new_active == frozen_path[cpu_id]) ||
+	         tp->dump_lock);
+
+	return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+                          struct ipipe_trace_path *old_tp, int old_pos)
+{
+	int i;
+
+	new_tp->trace_pos = pre_trace+1;
+
+	for (i = new_tp->trace_pos; i > 0; i--)
+		memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+		       &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+		       sizeof(struct ipipe_trace_point));
+
+	/* mark the end (i.e. the point before point[0]) invalid */
+	new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu_id, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = active_path[cpu_id];
+	unsigned long long length;
+
+	/* do we have a new worst case? */
+	length = tp->point[tp->end].timestamp -
+	         tp->point[tp->begin].timestamp;
+	if (length > (trace_paths[cpu_id][max_path[cpu_id]]).length) {
+		/* we need protection here against other cpus trying
+		   to start a proc dump */
+		spin_lock_hw(&global_path_lock);
+
+		/* active path holds new worst case */
+		tp->length = length;
+		max_path[cpu_id] = active;
+
+		/* find next unused trace path */
+		active = __ipipe_get_free_trace_path(active, cpu_id);
+
+		spin_unlock_hw(&global_path_lock);
+
+		tp = &trace_paths[cpu_id][active];
+
+		/* migrate last entries for pre-tracing */
+		__ipipe_migrate_pre_trace(tp, old_tp, pos);
+	}
+
+	return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu_id, struct ipipe_trace_path *tp, int pos)
+{
+	struct ipipe_trace_path *old_tp = tp;
+	long active = active_path[cpu_id];
+	int i;
+
+	/* frozen paths have no core (begin=end) */
+	tp->begin = tp->end;
+
+	/* we need protection here against other cpus trying
+	 * to set their frozen path or to start a proc dump */
+	spin_lock_hw(&global_path_lock);
+
+	frozen_path[cpu_id] = active;
+
+	/* find next unused trace path */
+	active = __ipipe_get_free_trace_path(active, cpu_id);
+
+	/* check if this is the first frozen path */
+	for_each_online_cpu(i) {
+		if ((i != cpu_id) &&
+		    (trace_paths[i][frozen_path[i]].end >= 0))
+			tp->end = -1;
+	}
+
+	spin_unlock_hw(&global_path_lock);
+
+	tp = &trace_paths[cpu_id][active];
+
+	/* migrate last entries for pre-tracing */
+	__ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+	return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+              unsigned long parent_eip, unsigned long v)
+{
+	struct ipipe_trace_path *tp, *old_tp;
+	int pos, next_pos, begin;
+	struct ipipe_trace_point *point;
+	unsigned long flags;
+	int cpu_id;
+
+	local_irq_save_hw_notrace(flags);
+
+	cpu_id = ipipe_processor_id();
+restart:
+	tp = old_tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	/* here starts a race window with NMIs - catched below */
+
+	/* check for NMI recursion */
+	if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+		tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* first freeze request from NMI context? */
+		if ((type == IPIPE_TRACE_FREEZE) &&
+		    !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+			/* save arguments and mark deferred freezing */
+			tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+			tp->nmi_saved_eip = eip;
+			tp->nmi_saved_parent_eip = parent_eip;
+			tp->nmi_saved_v = v;
+		}
+		return; /* no need for restoring flags inside IRQ */
+	}
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (unlikely(tp != &trace_paths[cpu_id][active_path[cpu_id]])) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	/* get the point buffer */
+	pos = tp->trace_pos;
+	point = &tp->point[pos];
+
+	/* store all trace point data */
+	point->type = type;
+	point->flags = local_test_iflag_hw(flags) ? 0 : IPIPE_TFLG_HWIRQ_OFF;
+	point->eip = eip;
+	point->parent_eip = parent_eip;
+	point->v = v;
+	ipipe_read_tsc(point->timestamp);
+
+	__ipipe_store_domain_states(point, cpu_id);
+
+	/* forward to next point buffer */
+	next_pos = WRAP_POINT_NO(pos+1);
+	tp->trace_pos = next_pos;
+
+	/* only mark beginning if we haven't started yet */
+	begin = tp->begin;
+	if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+		tp->begin = pos;
+
+	/* end of critical path, start post-trace if not already started */
+	if (unlikely(type == IPIPE_TRACE_END) &&
+	    (begin >= 0) && !tp->post_trace)
+		tp->post_trace = post_trace + 1;
+
+	/* freeze only if the slot is free and we are not already freezing */
+	if (unlikely(type == IPIPE_TRACE_FREEZE) &&
+	    (trace_paths[cpu_id][frozen_path[cpu_id]].begin < 0) &&
+	    !(tp->flags & IPIPE_TFLG_FREEZING)) {
+		tp->post_trace = post_trace + 1;
+		tp->flags |= IPIPE_TFLG_FREEZING;
+	}
+
+	/* enforce end of trace in case of overflow */
+	if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+		tp->end = pos;
+		goto enforce_end;
+	}
+
+	/* stop tracing this path if we are in post-trace and
+	 *  a) that phase is over now or
+	 *  b) a new TRACE_BEGIN came in but we are not freezing this path */
+	if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+	             ((type == IPIPE_TRACE_BEGIN) &&
+	              !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+		/* store the path's end (i.e. excluding post-trace) */
+		tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+enforce_end:
+		if (tp->flags & IPIPE_TFLG_FREEZING)
+			tp = __ipipe_trace_freeze(cpu_id, tp, pos);
+		else
+			tp = __ipipe_trace_end(cpu_id, tp, pos);
+
+		/* reset the active path, maybe already start a new one */
+		tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+			WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+		tp->end = -1;
+		tp->post_trace = 0;
+		tp->flags = 0;
+
+		/* update active_path not earlier to avoid races with NMIs */
+		active_path[cpu_id] = tp - trace_paths[cpu_id];
+	}
+
+	/* we still have old_tp and point,
+	 * let's reset NMI lock and check for catches */
+	old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+	if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+		/* well, this late tagging may not immediately be visible for
+		 * other cpus already dumping this path - a minor issue */
+		point->flags |= IPIPE_TFLG_NMI_HIT;
+
+		/* handle deferred freezing from NMI context */
+		if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+			__ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+			              old_tp->nmi_saved_parent_eip,
+			              old_tp->nmi_saved_v);
+	}
+
+	local_irq_restore_hw_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+	unsigned long flags;
+	int cpu_id;
+	struct ipipe_trace_path *tp;
+
+	spin_lock_irqsave_hw(&global_path_lock, flags);
+
+	cpu_id = ipipe_processor_id();
+restart:
+	tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	/* here is small race window with NMIs - catched below */
+
+	/* clear NMI events and set lock (atomically per cpu) */
+	tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+	                           IPIPE_TFLG_NMI_FREEZE_REQ))
+	                       | IPIPE_TFLG_NMI_LOCK;
+
+	/* check active_path again - some nasty NMI may have switched
+	 * it meanwhile */
+	if (tp != &trace_paths[cpu_id][active_path[cpu_id]]) {
+		/* release lock on wrong path and restart */
+		tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+		/* there is no chance that the NMI got deferred
+		 * => no need to check for pending freeze requests */
+		goto restart;
+	}
+
+	return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+	int cpu_id;
+	struct ipipe_trace_path *tp;
+
+	/* release spinlock first - it's not involved in the NMI issue */
+	spin_unlock_hw(&global_path_lock);
+
+	cpu_id = ipipe_processor_id();
+	tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+	/* handle deferred freezing from NMI context */
+	if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+		__ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+		              tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+	local_irq_restore_hw(flags);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+	if (!ipipe_trace_enable)
+		return;
+	__ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+	              __BUILTIN_RETURN_ADDRESS0,
+	              __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL(ipipe_trace_pid);
+
+int ipipe_trace_max_reset(void)
+{
+	int cpu_id;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_cpu(cpu_id) {
+		path = &trace_paths[cpu_id][max_path[cpu_id]];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin     = -1;
+		path->end       = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+	int cpu_id;
+	unsigned long flags;
+	struct ipipe_trace_path *path;
+	int ret = 0;
+
+	flags = __ipipe_global_path_lock();
+
+	for_each_cpu(cpu_id) {
+		path = &trace_paths[cpu_id][frozen_path[cpu_id]];
+
+		if (path->dump_lock) {
+			ret = -EBUSY;
+			break;
+		}
+
+		path->begin = -1;
+		path->end = -1;
+		path->trace_pos = 0;
+		path->length    = 0;
+	}
+
+	__ipipe_global_path_unlock(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_frozen_reset);
+
+void ipipe_trace_panic_freeze(void)
+{
+	unsigned long flags;
+	int cpu_id;
+
+	ipipe_trace_enable = 0;
+	local_irq_save_hw_notrace(flags);
+
+	cpu_id = ipipe_processor_id();
+
+	panic_path = &trace_paths[cpu_id][active_path[cpu_id]];
+
+	local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(ipipe_trace_panic_freeze);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+                      int trylock)
+{
+	struct task_struct *task = NULL;
+	char buf[8];
+	int i;
+	int locked = 1;
+
+	if (trylock && !read_trylock(&tasklist_lock))
+		locked = 0;
+	else
+		read_lock(&tasklist_lock);
+
+	if (locked)
+		task = find_task_by_pid((pid_t)point->v);
+
+	if (task)
+		strncpy(task_info, task->comm, 11);
+	else
+		strcpy(task_info, "-<?>-");
+
+	if (locked)
+		read_unlock(&tasklist_lock);
+
+	for (i = strlen(task_info); i < 11; i++)
+		task_info[i] = ' ';
+
+	sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+	strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+void ipipe_trace_panic_dump(void)
+{
+	int cnt = back_trace;
+	int start, pos;
+	char task_info[12];
+
+	printk("I-pipe tracer log (%d points):\n", cnt);
+
+	start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+	while (cnt-- > 0) {
+		struct ipipe_trace_point *point = &panic_path->point[pos];
+		long time;
+		char buf[16];
+
+		if (!point->eip)
+			printk("-<invalid>-\n");
+		else {
+			__ipipe_trace_point_type(buf, point);
+			printk(buf);
+
+			switch (point->type & IPIPE_TYPE_MASK) {
+				case IPIPE_TRACE_FUNC:
+					printk("           ");
+					break;
+
+				case IPIPE_TRACE_PID:
+					__ipipe_get_task_info(task_info,
+					                      point, 1);
+					printk(task_info);
+					break;
+
+				default:
+					printk("0x%08lx ", point->v);
+			}
+
+			time = __ipipe_signed_tsc2us(point->timestamp -
+				panic_path->point[start].timestamp);
+			printk(" %5ld ", time);
+
+			__ipipe_print_symname(NULL, point->eip);
+			printk(" (");
+			__ipipe_print_symname(NULL, point->parent_eip);
+			printk(")\n");
+		}
+		pos = WRAP_POINT_NO(pos - 1);
+	}
+}
+EXPORT_SYMBOL(ipipe_trace_panic_dump);
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+	return ((WRAP_POINT_NO(point_no-print_path->begin) <
+	         WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+	        ((print_path->end == print_path->begin) &&
+	         (WRAP_POINT_NO(point_no-print_path->end) >
+	          print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+        unsigned long long abs_tsc;
+        long us;
+
+	/* ipipe_tsc2us works on unsigned => handle sign separately */
+        abs_tsc = (tsc >= 0) ? tsc : -tsc;
+        us = ipipe_tsc2us(abs_tsc);
+        if (tsc < 0)
+                return -us;
+        else
+                return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+	switch (point->type & IPIPE_TYPE_MASK) {
+		case IPIPE_TRACE_FUNC:
+			strcpy(buf, "func    ");
+			break;
+
+		case IPIPE_TRACE_BEGIN:
+			strcpy(buf, "begin   ");
+			break;
+
+		case IPIPE_TRACE_END:
+			strcpy(buf, "end     ");
+			break;
+
+		case IPIPE_TRACE_FREEZE:
+			strcpy(buf, "freeze  ");
+			break;
+
+		case IPIPE_TRACE_SPECIAL:
+			sprintf(buf, "(0x%02x)  ",
+				point->type >> IPIPE_TYPE_BITS);
+			break;
+
+		case IPIPE_TRACE_PID:
+			sprintf(buf, "[%5d] ", (pid_t)point->v);
+			break;
+	}
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	char mark = ' ';
+	int point_no = point - print_path->point;
+	int i;
+
+	if (print_path->end == point_no)
+		mark = '<';
+	else if (print_path->begin == point_no)
+		mark = '>';
+	else if (__ipipe_in_critical_trpath(point_no))
+		mark = ':';
+	seq_printf(m, "%c%c", mark,
+	           (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+	if (!verbose_trace)
+		return;
+
+	for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+		seq_printf(m, "%c",
+			(IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+			    (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+				'#' : '+') :
+			(IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+	unsigned long delay = 0;
+	int next;
+	char *mark = "  ";
+
+	next = WRAP_POINT_NO(point+1 - print_path->point);
+
+	if (next != print_path->trace_pos)
+		delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+		                     point->timestamp);
+
+	if (__ipipe_in_critical_trpath(point - print_path->point)) {
+		if (delay > IPIPE_DELAY_WARN)
+			mark = "! ";
+		else if (delay > IPIPE_DELAY_NOTE)
+			mark = "+ ";
+	}
+	seq_puts(m, mark);
+
+	if (verbose_trace)
+		seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+		           (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+	else
+		seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+	char namebuf[KSYM_NAME_LEN+1];
+	unsigned long size, offset;
+	const char *sym_name;
+	char *modname;
+
+	sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+	/* printing to /proc? */
+	if (m) {
+		if (sym_name) {
+			if (verbose_trace) {
+				seq_printf(m, "%s+0x%lx", sym_name, offset);
+				if (modname)
+					seq_printf(m, " [%s]", modname);
+			} else
+				seq_puts(m, sym_name);
+		} else
+			seq_printf(m, "<%08lx>", eip);
+	} else {
+		/* panic dump */
+		if (sym_name) {
+			printk("%s+0x%lx", sym_name, offset);
+			if (modname)
+				printk(" [%s]", modname);
+		}
+	}
+}
+
+#if defined(CONFIG_XENO_OPT_DEBUG) || defined(CONFIG_DEBUG_PREEMPT)
+static void __ipipe_print_dbgwarning(struct seq_file *m)
+{
+	seq_puts(m, "\n******** WARNING ********\n"
+		"The following debugging options will increase the observed "
+		"latencies:\n"
+#ifdef CONFIG_XENO_OPT_DEBUG
+		" o CONFIG_XENO_OPT_DEBUG\n"
+#endif /* CONFIG_XENO_OPT_DEBUG */
+#ifdef CONFIG_XENO_OPT_DEBUG_QUEUES
+		" o CONFIG_XENO_OPT_DEBUG_QUEUES (very costly)\n"
+#endif /* CONFIG_XENO_OPT_DEBUG */
+#ifdef CONFIG_DEBUG_PREEMPT
+		" o CONFIG_DEBUG_PREEMPT\n"
+#endif /* CONFIG_DEBUG_PREEMPT */
+		"\n");
+}
+#else /* !WARN_ON_DEBUGGING_LATENCIES */
+# define __ipipe_print_dbgwarning(m)
+#endif /* WARN_ON_DEBUGGING_LATENCIES */
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+	if (verbose_trace) {
+		const char *name[4] = { [0 ... 3] = "<unused>" };
+		struct list_head *pos;
+		int i = 0;
+
+		list_for_each_prev(pos, &__ipipe_pipeline) {
+			struct ipipe_domain *ipd =
+				list_entry(pos, struct ipipe_domain, p_link);
+
+			name[i] = ipd->name;
+			if (++i > 3)
+				break;
+		}
+
+		seq_printf(m,
+		           " +----- Hard IRQs ('|': locked)\n"
+		           " |+---- %s\n"
+		           " ||+--- %s\n"
+		           " |||+-- %s\n"
+		           " ||||+- %s%s\n"
+		           " |||||                        +---------- "
+		               "Delay flag ('+': > %d us, '!': > %d us)\n"
+		           " |||||                        |        +- "
+		               "NMI noise ('N')\n"
+		           " |||||                        |        |\n"
+		           "      Type    User Val.   Time    Delay  Function "
+		               "(Parent)\n",
+		           name[3], name[2], name[1], name[0],
+		           name[0] ? " ('*': domain stalled, '+': current, "
+		               "'#': current+stalled)" : "",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+	} else
+		seq_printf(m,
+		           " +--------------- Hard IRQs ('|': locked)\n"
+		           " |             +- Delay flag "
+		               "('+': > %d us, '!': > %d us)\n"
+		           " |             |\n"
+		           "  Type     Time   Function (Parent)\n",
+		           IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+	loff_t n = *pos;
+	
+	down(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *path;
+		unsigned long length_usecs;
+		int points, i;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the longest of all per-cpu paths */
+		print_path = NULL;
+		for_each_online_cpu(i) {
+			path = &trace_paths[i][max_path[i]];
+			if ((print_path == NULL) ||
+			    (path->length > print_path->length))
+				print_path = path;
+		}
+		print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		/* does this path actually contain data? */
+		if (print_path->end == print_path->begin)
+			return NULL;
+
+		/* number of points inside the critical path */
+		points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+		/* pre- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, pre-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = pre_trace;
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+				print_post_trace;
+
+		length_usecs = ipipe_tsc2us(print_path->length);
+		seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		__ipipe_print_dbgwarning(m);
+		seq_printf(m, "Begin: %lld cycles, Trace Points: %d (-%d/+%d), "
+			"Length: %lu us\n\n",
+			print_path->point[print_path->begin].timestamp,
+			points, print_pre_trace, print_post_trace, length_usecs);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+	loff_t n = ++*pos;
+
+	/* check if we are inside the trace range with the next entry */
+	if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+	                       print_pre_trace + print_post_trace))
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin -
+	                                        print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+	if (print_path)
+		print_path->dump_lock = 0;
+	up(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+	long time;
+	struct ipipe_trace_point *point = p;
+	char buf[16];
+
+	if (!point->eip) {
+		seq_puts(m, "-<invalid>-\n");
+		return 0;
+	}
+
+	__ipipe_print_pathmark(m, point);
+	__ipipe_trace_point_type(buf, point);
+	seq_puts(m, buf);
+	if (verbose_trace)
+		switch (point->type & IPIPE_TYPE_MASK) {
+			case IPIPE_TRACE_FUNC:
+				seq_puts(m, "           ");
+				break;
+
+			case IPIPE_TRACE_PID:
+				__ipipe_get_task_info(buf, point, 0);
+				seq_puts(m, buf);
+				break;
+
+			default:
+				seq_printf(m, "0x%08lx ", point->v);
+		}
+
+	time = __ipipe_signed_tsc2us(point->timestamp -
+		print_path->point[print_path->begin].timestamp);
+	seq_printf(m, "%5ld", time);
+
+	__ipipe_print_delay(m, point);
+	__ipipe_print_symname(m, point->eip);
+	seq_puts(m, " (");
+	__ipipe_print_symname(m, point->parent_eip);
+	seq_puts(m, ")\n");
+
+	return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+	.start = __ipipe_max_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+                  size_t count, loff_t *data)
+{
+	down(&out_mutex);
+	ipipe_trace_max_reset();
+	up(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_max_prtrace_fops = {
+	.open       = __ipipe_max_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_max_reset,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{	
+	loff_t n = *pos;
+
+	down(&out_mutex);
+
+	if (!n) {
+		struct ipipe_trace_path *path;
+		int i;
+		unsigned long flags;
+
+		/* protect against max_path/frozen_path updates while we
+		 * haven't locked our target path, also avoid recursively
+		 * taking global_path_lock from NMI context */
+		flags = __ipipe_global_path_lock();
+
+		/* find the first of all per-cpu frozen paths */
+		print_path = NULL;
+		for_each_online_cpu(i) {
+			path = &trace_paths[i][frozen_path[i]];
+			if (path->end >= 0)
+				print_path = path;
+		}
+		if (print_path)
+			print_path->dump_lock = 1;
+
+		__ipipe_global_path_unlock(flags);
+
+		if (!print_path)
+			return NULL;
+
+		/* back- and post-tracing length, post-trace length was frozen
+		   in __ipipe_trace, back-trace may have to be reduced due to
+		   buffer overrun */
+		print_pre_trace  = back_trace-1; /* substract freeze point */
+		print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+		                                 print_path->end - 1);
+		if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+			print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+				print_post_trace;
+
+		seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n"
+			"------------------------------------------------------"
+			"------\n",
+			UTS_RELEASE, IPIPE_ARCH_STRING);
+		__ipipe_print_dbgwarning(m);
+		seq_printf(m, "Freeze: %lld cycles, Trace Points: %d (+%d)\n\n",
+			print_path->point[print_path->begin].timestamp,
+			print_pre_trace+1, print_post_trace);
+		__ipipe_print_headline(m);
+	}
+
+	/* check if we are inside the trace range */
+	if (n >= print_pre_trace + 1 + print_post_trace)
+		return NULL;
+
+	/* return the next point to be shown */
+	return &print_path->point[WRAP_POINT_NO(print_path->begin-
+	                                        print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+	.start = __ipipe_frozen_prtrace_start,
+	.next  = __ipipe_prtrace_next,
+	.stop  = __ipipe_prtrace_stop,
+	.show  = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+                    size_t count, loff_t *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, pbuffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	down(&out_mutex);
+	ipipe_trace_frozen_reset();
+	if (val > 0)
+		ipipe_trace_freeze(-1);
+	up(&out_mutex);
+
+	return count;
+}
+
+struct file_operations __ipipe_frozen_prtrace_fops = {
+	.open       = __ipipe_frozen_prtrace_open,
+	.read       = seq_read,
+	.write      = __ipipe_frozen_ctrl,
+	.llseek     = seq_lseek,
+	.release    = seq_release,
+};
+
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
+                               int count, int *eof, void *data)
+{
+	int len;
+
+	len = sprintf(page, "%u\n", *(int *)data);
+	len -= off;
+	if (len <= off + count)
+		*eof = 1;
+	*start = page + off;
+	if (len > count)
+		len = count;
+	if (len < 0)
+		len = 0;
+
+	return len;
+}
+
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+                               unsigned long count, void *data)
+{
+	char *end, buf[16];
+	int val;
+	int n;
+
+	n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+	if (copy_from_user(buf, buffer, n))
+		return -EFAULT;
+
+	buf[n] = '\0';
+	val = simple_strtol(buf, &end, 0);
+
+	if (((*end != '\0') && !isspace(*end)) || (val < 0))
+		return -EINVAL;
+
+	down(&out_mutex);
+	*(int *)data = val;
+	up(&out_mutex);
+
+	return count;
+}
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+static void __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+                              const char *name, int *value_ptr)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry(name, 0644, trace_dir);
+	if (entry) {
+		entry->data = value_ptr;
+		entry->read_proc = __ipipe_rd_proc_val;
+		entry->write_proc = __ipipe_wr_proc_val;
+		entry->owner = THIS_MODULE;
+	}
+}
+
+void __init __ipipe_init_tracer(void)
+{
+	struct proc_dir_entry *trace_dir;
+	struct proc_dir_entry *entry;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+	int cpu, path;
+
+	for (cpu = 0; cpu < NR_CPUS; cpu++) {
+		trace_paths[cpu] = vmalloc(
+			sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		if (!trace_paths) {
+			printk(KERN_ERR "I-pipe: "
+			       "insufficient memory for trace buffer.\n");
+			return;
+		}
+		memset(trace_paths[cpu], 0,
+			sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+		for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+			trace_paths[cpu][path].begin = -1;
+			trace_paths[cpu][path].end   = -1;
+		}
+	}
+	ipipe_trace_enable = 1;
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+	trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
+
+	entry = create_proc_entry("max", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_max_prtrace_fops;
+
+	entry = create_proc_entry("frozen", 0644, trace_dir);
+	if (entry)
+		entry->proc_fops = &__ipipe_frozen_prtrace_fops;
+
+	__ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+	                              &pre_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+	                              &post_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+	                              &back_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "verbose",
+	                              &verbose_trace);
+	__ipipe_create_trace_proc_val(trace_dir, "enable",
+	                              &ipipe_trace_enable);
+}
diff -urN -x Entries -x Entries.Log -x Repository -x Root -x .config -x .version -x .config.old -x rootfs.img -x '*.cmd' -x '*.o' -x Tag -x linux -x '*.rej' -x '*.orig' -x '*.bak' linux-2.6.x-06r1-ipipe-orig/Makefile linux-2.6.x-06r1-tracer/Makefile
--- linux-2.6.x-06r1-ipipe-orig/Makefile	2006-07-10 17:07:55.000000000 +0800
+++ linux-2.6.x-06r1-tracer/Makefile	2006-08-22 15:13:00.000000000 +0800
@@ -511,11 +511,15 @@
 CFLAGS		+= $(call add-align,CONFIG_CC_ALIGN_LOOPS,-loops)
 CFLAGS		+= $(call add-align,CONFIG_CC_ALIGN_JUMPS,-jumps)
 
+ifdef CONFIG_IPIPE_TRACE
+CFLAGS          += -pg -fno-omit-frame-pointer $(call cc-option,-fno-optimize-sibling-calls,)
+else
 ifdef CONFIG_FRAME_POINTER
 CFLAGS		+= -fno-omit-frame-pointer $(call cc-option,-fno-optimize-sibling-calls,)
 else
 CFLAGS		+= -fomit-frame-pointer
 endif
+endif
 
 ifdef CONFIG_DEBUG_INFO
 CFLAGS		+= -g

[-- Attachment #3: bfin-ipipe-tracer.log --]
[-- Type: text/x-log, Size: 8238 bytes --]

root:~> ./latency -t0 -p 300 -f


root:~> cat /proc/ipipe/trace/max
I-pipe worst-case tracing service on 2.6.16.11.ADI-2006R1blackfin/ipipe-1.4-03
------------------------------------------------------------
Begin: 240373976502 cycles, Trace Points: 24 (-10/+1), Length: 270 us

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
 |  + begin   0x80000001   -20    0.910  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
 |  + end     0x80000001   -19    6.360  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
    + func                 -13    1.440  ___ipipe_syscall_root+0xe (system_call+0x2e)
    + func                 -11    0.890  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
 |  + begin   0x80000001   -10    1.280  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
 |  + end     0x80000001    -9    1.530  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
    + func                  -7    2.730  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
    + func                  -5    1.440  ___rt_task_wait_period+0xc (_hisyscall_event+0x134)
    + func                  -3    1.820  _rt_task_wait_period+0xc (___rt_task_wait_period+0x4c)
    + func                  -1    0.950  _xnpod_wait_thread_period+0xa (_rt_task_wait_period+0x4c)
>|  + begin   0x80000001     1+   1.900  _xnpod_wait_thread_period+0x30 (__start+0xfffff000)
:|  # func                   2+   1.640  _xnpod_suspend_thread+0xa (_xnpod_wait_thread_period+0x78)
:|  # func                   4    0.892  _xnpod_schedule+0xa (_xnpod_suspend_thread+0x168)
:|  # [   50] -<?>-   99     5+   1.658  _xnpod_schedule+0x112 (__start+0xfffff000)
:|  # [    1] init    -1     7+   2.350  _xnpod_schedule+0x270 (__start+0xfffff000)
:    +func                   9! 235.540  ___ipipe_walk_pipeline+0x1e (___ipipe_handle_irq+0x14c)
:|   +func                 244+   1.330  ___ipipe_grab_irq+0x10 (_evt_timer+0x7c)
:|   +func                 246+   1.560  ___ipipe_handle_irq+0x10 (___ipipe_grab_irq+0x32)
:|   +func                 247+   2.120  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x174)
:|   +func                 249+   1.680  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x74)
:|  # func                 251+   1.270  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xa6)
:|  # func                 252+   1.090  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func                 254+   1.380  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2e)
:|  # func                 255+   1.470  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func                 256+   1.400  _xnthread_periodic_handler+0xa (_xntimer_do_tick_aperiodic+0x1d6)
:|  # func                 258    0.790  _xnpod_resume_thread+0xa (_xnthread_periodic_handler+0x1e)
:|  # [   50] -<?>-   99   259+   2.510  _xnpod_resume_thread+0x4a (__start+0xfffff000)
:|  # func                 261+   1.890  _xnpod_schedule+0xa (_xnintr_irq_handler+0x82)
:|  # func                 263+   1.380  _xnpod_schedule_deferred+0xc (___ipipe_call_irqtail+0x1e)
:|  # func                 264    0.970  _xnpod_schedule+0xa (_xnpod_schedule_deferred+0x28)
:|  # [    1] init    -1   265+   1.950  _xnpod_schedule+0x112 (__start+0xfffff000)
:|  # [   50] -<?>-   99   267+   1.560  _xnpod_schedule+0x270 (__start+0xfffff000)
:|  # func                 269    0.690  ___ipipe_restore_pipeline_head+0xa (_xnpod_wait_thread_period+0xfa)
<|  + end     0x80000000   270    2.340  ___ipipe_restore_pipeline_head+0x90 (__start+0xfffff000)
 |  + begin   0x80000001   272    0.000  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)


root:~> cat /proc/ipipe/trace/frozen
I-pipe frozen back-tracing service on 2.6.16.11.ADI-2006R1blackfin/ipipe-1.4-03
------------------------------------------------------------
Freeze: 234204981352 cycles, Trace Points: 30 (+10)

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
:|  #*func                 -58    0.990  _xnpod_schedule+0xa (_xnpod_schedule_deferred+0x28)
:|  #*[    1] init    -1   -57+   2.210  _xnpod_schedule+0x112 (__start+0xfffff000)
:|  #*[   50] -<?>-   99   -55+   2.380  _xnpod_schedule+0x270 (__start+0xfffff000)
:|  #*func                 -52    0.710  ___ipipe_restore_pipeline_head+0xa (_xnpod_wait_thread_period+0xfa)
:|  +*end     0x80000000   -52+   3.020  ___ipipe_restore_pipeline_head+0x90 (__start+0xfffff000)
:|  +*begin   0x80000001   -49+   1.610  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
:|  +*end     0x80000001   -47+   6.900  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
:   +*func                 -40+   1.330  ___ipipe_syscall_root+0xe (system_call+0x2e)
:   +*func                 -39    0.700  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
:|  +*begin   0x80000001   -38+   1.340  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
:|  +*end     0x80000001   -37+   1.360  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
:   +*func                 -36+   2.080  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
:   +*func                 -33+   2.530  ___rt_timer_tsc2ns+0x14 (_hisyscall_event+0x134)
:   +*func                 -31+   1.490  _rt_timer_tsc2ns+0xa (___rt_timer_tsc2ns+0x5e)
:   +*func                 -29+   1.370  ___muldi3+0x14 (_rt_timer_tsc2ns+0x58)
:   +*func                 -28+   2.330  ___muldi3+0x14 (_rt_timer_tsc2ns+0x6a)
:   +*func                 -26+   4.460  ___div64_32+0xe (_rt_timer_tsc2ns+0x1d8)
:|  +*begin   0x80000001   -21+   1.320  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
:|  +*end     0x80000001   -20+   3.380  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
:   +*func                 -17+   1.330  ___ipipe_syscall_root+0xe (system_call+0x2e)
:   +*func                 -15    0.700  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
:|  +*begin   0x80000001   -15+   1.340  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
:|  +*end     0x80000001   -13+   1.360  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
:   +*func                 -12+   2.100  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
:   +*func                 -10+   2.230  _xnshadow_sys_trace+0xc (_hisyscall_event+0x134)
:   +*func                  -8+   1.470  _ipipe_trace_frozen_reset+0xa (_xnshadow_sys_trace+0xb2)
:   +*func                  -6+   1.110  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
:|  +*begin   0x80000001    -5+   3.260  ___ipipe_global_path_lock+0x1a (__start+0xfffff000)
:|  +*end     0x80000001    -2+   1.160  ___ipipe_global_path_unlock+0x44 (__start+0xfffff000)
<   +*freeze  0x0001ecf6     1    1.180  _xnshadow_sys_trace+0x9c (__start+0xfffff000)
 |  +*begin   0x80000001     2    0.930  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
 |  +*end     0x80000001     3    4.320  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
    +*func                   7    1.160  ___ipipe_syscall_root+0xe (system_call+0x2e)
    +*func                   8    0.910  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
 |  +*begin   0x80000001     9    0.950  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
 |  +*end     0x80000001    10    1.350  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
    +*func                  11    1.910  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
    +*func                  13    1.310  ___rt_task_wait_period+0xc (_hisyscall_event+0x134)
    +*func                  15    1.930  _rt_task_wait_period+0xc (___rt_task_wait_period+0x4c)
    +*func                  16    0.000  _xnpod_wait_thread_period+0xa (_rt_task_wait_period+0x4c)

^ permalink raw reply	[flat|nested] 9+ messages in thread

* [Xenomai-core] Re: [Adeos-main] ipip tracer porting for blackfin
  2006-09-30  7:59 [Xenomai-core] ipip tracer porting for blackfin Li Yi (Adam)
@ 2006-10-01 22:16 ` Jan Kiszka
  2006-11-10 13:54   ` Li Yi (Adam)
  0 siblings, 1 reply; 9+ messages in thread
From: Jan Kiszka @ 2006-10-01 22:16 UTC (permalink / raw)
  To: Li Yi (Adam); +Cc: adeos-main, xenomai

[-- Attachment #1: Type: text/plain, Size: 10134 bytes --]

Li Yi (Adam) wrote:
> Hi,
> 
> Attached ipipe-tracer ported to blackfin. The patch is based on
> http://download.gna.org/adeos/patches/v2.6/blackfin/adeos-ipipe-bf53x-R06R1RC2-1.4-03.patch.

Great!

> 
> But it requires a little hack to Blackfin GCC-R06R1 to  make the "-pg"
> option work.  The latest release of bfin-gcc (06r2) has supported this
> feature. And I will update the ipipe tracer patch to the latest kernel
> release (r06r2).
> 
> Also attached the ipipe tracer log.

It looks like a few instrumentation issues are remaining, see comments
below.

> Hoping for your comments.
> 
> Best Regards,
> 
> -Yi
> 
> 
> ------------------------------------------------------------------------
> 
> root:~> ./latency -t0 -p 300 -f
> 
> 
> root:~> cat /proc/ipipe/trace/max
> I-pipe worst-case tracing service on 2.6.16.11.ADI-2006R1blackfin/ipipe-1.4-03
> ------------------------------------------------------------
> Begin: 240373976502 cycles, Trace Points: 24 (-10/+1), Length: 270 us
> 
>  +----- Hard IRQs ('|': locked)
>  |+---- <unused>
>  ||+--- <unused>
>  |||+-- Xenomai
>  ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
>  |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
>  |||||                        |        +- NMI noise ('N')
>  |||||                        |        |
>       Type    User Val.   Time    Delay  Function (Parent)
>  |  + begin   0x80000001   -20    0.910  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
>  |  + end     0x80000001   -19    6.360  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
>     + func                 -13    1.440  ___ipipe_syscall_root+0xe (system_call+0x2e)
>     + func                 -11    0.890  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
>  |  + begin   0x80000001   -10    1.280  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
>  |  + end     0x80000001    -9    1.530  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
>     + func                  -7    2.730  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
>     + func                  -5    1.440  ___rt_task_wait_period+0xc (_hisyscall_event+0x134)
>     + func                  -3    1.820  _rt_task_wait_period+0xc (___rt_task_wait_period+0x4c)
>     + func                  -1    0.950  _xnpod_wait_thread_period+0xa (_rt_task_wait_period+0x4c)
>> |  + begin   0x80000001     1+   1.900  _xnpod_wait_thread_period+0x30 (__start+0xfffff000)
> :|  # func                   2+   1.640  _xnpod_suspend_thread+0xa (_xnpod_wait_thread_period+0x78)
> :|  # func                   4    0.892  _xnpod_schedule+0xa (_xnpod_suspend_thread+0x168)
> :|  # [   50] -<?>-   99     5+   1.658  _xnpod_schedule+0x112 (__start+0xfffff000)
> :|  # [    1] init    -1     7+   2.350  _xnpod_schedule+0x270 (__start+0xfffff000)
> :    +func                   9! 235.540  ___ipipe_walk_pipeline+0x1e (___ipipe_handle_irq+0x14c)

Looks like there are some inconsistencies in the IRQ state tracing: You
schedule from latency (PID 50, now of unknown name "-<?>-", priority 99)
to the init process under Linux (PID 1, "init", ROOT thread priority
-1). After that switch the IRQs are very likely earlier reenabled, e.g.
on interrupt return. Compare the instrumentation of x86 with your version.

Moreover, IRQ entry and exit should also be traced. This doesn't seem to
happen yet, otherwise we would see some "begin" entry with the IRQ
number as user value. Again, check the x86 code for
ipipe_trace_begin/end passages.

> :|   +func                 244+   1.330  ___ipipe_grab_irq+0x10 (_evt_timer+0x7c)
> :|   +func                 246+   1.560  ___ipipe_handle_irq+0x10 (___ipipe_grab_irq+0x32)
> :|   +func                 247+   2.120  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x174)
> :|   +func                 249+   1.680  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x74)
> :|  # func                 251+   1.270  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xa6)
> :|  # func                 252+   1.090  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
> :|  # func                 254+   1.380  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2e)
> :|  # func                 255+   1.470  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
> :|  # func                 256+   1.400  _xnthread_periodic_handler+0xa (_xntimer_do_tick_aperiodic+0x1d6)
> :|  # func                 258    0.790  _xnpod_resume_thread+0xa (_xnthread_periodic_handler+0x1e)
> :|  # [   50] -<?>-   99   259+   2.510  _xnpod_resume_thread+0x4a (__start+0xfffff000)
> :|  # func                 261+   1.890  _xnpod_schedule+0xa (_xnintr_irq_handler+0x82)
> :|  # func                 263+   1.380  _xnpod_schedule_deferred+0xc (___ipipe_call_irqtail+0x1e)
> :|  # func                 264    0.970  _xnpod_schedule+0xa (_xnpod_schedule_deferred+0x28)
> :|  # [    1] init    -1   265+   1.950  _xnpod_schedule+0x112 (__start+0xfffff000)
> :|  # [   50] -<?>-   99   267+   1.560  _xnpod_schedule+0x270 (__start+0xfffff000)
> :|  # func                 269    0.690  ___ipipe_restore_pipeline_head+0xa (_xnpod_wait_thread_period+0xfa)
> <|  + end     0x80000000   270    2.340  ___ipipe_restore_pipeline_head+0x90 (__start+0xfffff000)
>  |  + begin   0x80000001   272    0.000  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
> 
> 
> root:~> cat /proc/ipipe/trace/frozen
> I-pipe frozen back-tracing service on 2.6.16.11.ADI-2006R1blackfin/ipipe-1.4-03
> ------------------------------------------------------------
> Freeze: 234204981352 cycles, Trace Points: 30 (+10)
> 
>  +----- Hard IRQs ('|': locked)
>  |+---- <unused>
>  ||+--- <unused>
>  |||+-- Xenomai
>  ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
>  |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
>  |||||                        |        +- NMI noise ('N')
>  |||||                        |        |
>       Type    User Val.   Time    Delay  Function (Parent)
> :|  #*func                 -58    0.990  _xnpod_schedule+0xa (_xnpod_schedule_deferred+0x28)
> :|  #*[    1] init    -1   -57+   2.210  _xnpod_schedule+0x112 (__start+0xfffff000)
> :|  #*[   50] -<?>-   99   -55+   2.380  _xnpod_schedule+0x270 (__start+0xfffff000)
> :|  #*func                 -52    0.710  ___ipipe_restore_pipeline_head+0xa (_xnpod_wait_thread_period+0xfa)
> :|  +*end     0x80000000   -52+   3.020  ___ipipe_restore_pipeline_head+0x90 (__start+0xfffff000)
> :|  +*begin   0x80000001   -49+   1.610  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
> :|  +*end     0x80000001   -47+   6.900  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
> :   +*func                 -40+   1.330  ___ipipe_syscall_root+0xe (system_call+0x2e)
> :   +*func                 -39    0.700  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
> :|  +*begin   0x80000001   -38+   1.340  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
> :|  +*end     0x80000001   -37+   1.360  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
> :   +*func                 -36+   2.080  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
> :   +*func                 -33+   2.530  ___rt_timer_tsc2ns+0x14 (_hisyscall_event+0x134)
> :   +*func                 -31+   1.490  _rt_timer_tsc2ns+0xa (___rt_timer_tsc2ns+0x5e)
> :   +*func                 -29+   1.370  ___muldi3+0x14 (_rt_timer_tsc2ns+0x58)
> :   +*func                 -28+   2.330  ___muldi3+0x14 (_rt_timer_tsc2ns+0x6a)
> :   +*func                 -26+   4.460  ___div64_32+0xe (_rt_timer_tsc2ns+0x1d8)
> :|  +*begin   0x80000001   -21+   1.320  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
> :|  +*end     0x80000001   -20+   3.380  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
> :   +*func                 -17+   1.330  ___ipipe_syscall_root+0xe (system_call+0x2e)
> :   +*func                 -15    0.700  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
> :|  +*begin   0x80000001   -15+   1.340  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
> :|  +*end     0x80000001   -13+   1.360  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
> :   +*func                 -12+   2.100  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
> :   +*func                 -10+   2.230  _xnshadow_sys_trace+0xc (_hisyscall_event+0x134)
> :   +*func                  -8+   1.470  _ipipe_trace_frozen_reset+0xa (_xnshadow_sys_trace+0xb2)
> :   +*func                  -6+   1.110  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
> :|  +*begin   0x80000001    -5+   3.260  ___ipipe_global_path_lock+0x1a (__start+0xfffff000)
> :|  +*end     0x80000001    -2+   1.160  ___ipipe_global_path_unlock+0x44 (__start+0xfffff000)
> <   +*freeze  0x0001ecf6     1    1.180  _xnshadow_sys_trace+0x9c (__start+0xfffff000)

126 us (the user value of "freeze"), that looks ok given the overhead of
the tracer. If you extend your backtrace in this case to cover that
whole period, you can analyse what blocked the timer IRQ from waking up
earlier.

>  |  +*begin   0x80000001     2    0.930  ___ipipe_dispatch_event+0xb4 (__start+0xfffff000)
>  |  +*end     0x80000001     3    4.320  ___ipipe_dispatch_event+0x11a (__start+0xfffff000)
>     +*func                   7    1.160  ___ipipe_syscall_root+0xe (system_call+0x2e)
>     +*func                   8    0.910  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x24)
>  |  +*begin   0x80000001     9    0.950  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
>  |  +*end     0x80000001    10    1.350  ___ipipe_dispatch_event+0x88 (__start+0xfffff000)
>     +*func                  11    1.910  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xa0)
>     +*func                  13    1.310  ___rt_task_wait_period+0xc (_hisyscall_event+0x134)
>     +*func                  15    1.930  _rt_task_wait_period+0xc (___rt_task_wait_period+0x4c)
>     +*func                  16    0.000  _xnpod_wait_thread_period+0xa (_rt_task_wait_period+0x4c)
> 

Jan


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 249 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* [Xenomai-core] Re: [Adeos-main] ipip tracer porting for blackfin
  2006-10-01 22:16 ` [Xenomai-core] Re: [Adeos-main] " Jan Kiszka
@ 2006-11-10 13:54   ` Li Yi (Adam)
  2006-11-10 14:27     ` Philippe Gerum
  2006-11-10 16:11     ` Jan Kiszka
  0 siblings, 2 replies; 9+ messages in thread
From: Li Yi (Adam) @ 2006-11-10 13:54 UTC (permalink / raw)
  To: Jan Kiszka; +Cc: adeos-main, xenomai


[-- Attachment #1.1: Type: text/plain, Size: 14402 bytes --]

Hi Jan and Philippe,

I have tested the adeos-ipipe-bf53x-CVS-20061008-1.5-00.patch in
Xenomai-2.2.5 with latest Blackfin uclinux toolchain 06r2 release:
http://blackfin.uclinux.org/frs/download.php/1136/blackfin-toolchain-06r2-5.i386.rpm.
Since the toolchain changed a little for the "mcount" call, the ipipe-tracer
patch need to be updated a little.

Also I add trace for IRQ entry and exit as Jan suggested. Here is the
details of the patch (with my explanation)

diff -Nur linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe-mcount.S
linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-mcount.S
--- linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe-mcount.S  2006-11-10
21:04:21.000000000 +0800
+++ linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-mcount.S       2006-11-10
16:41:11.000000000 +0800
@@ -12,7 +12,7 @@

 .align 4       /* just in case */

-ENTRY(_mcount)
+ENTRY(__mcount)
        [--sp] = ( r7:0, p5:0 );
        [--sp] = ASTAT;
        p1.L = _ipipe_trace_enable;
diff -Nur linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe-root.c
linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-root.c
--- linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe- root.c    2006-11-10
21:04:21.000000000 +0800
+++ linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-root.c 2006-11-10 16:41:
59.000000000 +0800
@@ -281,6 +281,6 @@
 EXPORT_SYMBOL(show_stack);

 #ifdef CONFIG_IPIPE_TRACE_MCOUNT
-void notrace mcount(void);
-EXPORT_SYMBOL(mcount);
+void notrace _mcount(void);
+EXPORT_SYMBOL(_mcount);
 #endif /* CONFIG_IPIPE_TRACE_MCOUNT */

<Yi's note>: The  gcc  will add "Call _ _mcount" to each function entry when
there is "-pg" option

diff -Nur linux-2.6.x-ipipe-orig/arch/blackfin/mach-common/interrupt.S
linux-2.6.x-ipipe/arch/blackfin/mach-common/interrupt.S
--- linux-2.6.x-ipipe-orig/arch/blackfin/mach-common/interrupt.S
2006-11-10 21:04: 21.000000000 +0800
+++ linux-2.6.x-ipipe/arch/blackfin/mach-common/interrupt.S     2006-11-10
16:43:55.000000000 +0800
@@ -304,7 +304,10 @@
        p0.l = _irq_flags;
        p0.h = _irq_flags;
        r0 = [p0];
-       sti r0
+       sti r0;
+       /* Reenable interrupts */
+       [--sp] = reti;
+       r0 = [sp++];
        rts;

 #endif /* CONFIG_IPIPE */


<Yi's Note>: I used the ipipe-trace to trace the latency test case: "latency
-t2 -p200 -f". And found that "__ipipe_call_irqtail()" will disable global
interrupts, putting the Blackfin to a state like:
"IMASK: 0x0000FFFF, ILAT: 0x00000040, IPEND: 0x0000c010" - that is, Core
Timer irq latched, but since IPEND[4] is set, the irq handler cannot be
invoked - thus longer latency.


diff -Nur linux-2.6.x-ipipe-orig /arch/blackfin/mach-common/ints-
priority-sc.c linux-2.6.x-ipipe/arch/blackfin/mach-common/ints-priority-sc.c
--- linux-2.6.x-ipipe-orig/arch/blackfin/mach-common/ints-priority-sc.c2006-11-10
21:04:
21.000000000 +0800
+++ linux-2.6.x-ipipe/arch/blackfin/mach-common/ints-priority-sc.c
2006-11-10 16:45:44.000000000 +0800
@@ -1013,7 +1013,13 @@

 handle_irq:

+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+       ipipe_trace_begin(irq);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
        __ipipe_handle_irq(irq, regs);
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+       ipipe_trace_end(irq);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
diff -Nur linux-2.6.x-ipipe-orig /kernel/ipipe/Kconfig linux-2.6.x-ipipe
/kernel/ipipe/Kconfig
--- linux-2.6.x-ipipe-orig/kernel/ipipe/Kconfig 2006-11-10 21:04:
21.000000000 +0800
+++ linux-2.6.x-ipipe/kernel/ipipe/Kconfig      2006-11-10 16:39:
44.000000000 +0800
@@ -4,3 +4,5 @@
        ---help---
          Activate this option if you want the interrupt pipeline to be
          compiled in.
+
+source kernel/ipipe/Kconfig.debug

Also attached the new ipiper-tracer log.

Could you please have a review? The ipipe-tracer is a useful tool. I am now
using it to track the maxium interrupt latency of "latency -t2", hoping to
improve the worst cast interrupt latency.

Thanks,

-Yi



On 10/2/06, Jan Kiszka <jan.kiszka@domain.hid > wrote:
>
> Li Yi (Adam) wrote:
> > Hi,
> >
> > Attached ipipe-tracer ported to blackfin. The patch is based on
> >
> http://download.gna.org/adeos/patches/v2.6/blackfin/adeos-ipipe-bf53x-R06R1RC2-1.4-03.patch
> .
>
> Great!
>
> >
> > But it requires a little hack to Blackfin GCC-R06R1 to  make the "-pg"
> > option work.  The latest release of bfin-gcc (06r2) has supported this
> > feature. And I will update the ipipe tracer patch to the latest kernel
> > release (r06r2).
> >
> > Also attached the ipipe tracer log.
>
> It looks like a few instrumentation issues are remaining, see comments
> below.
>
> > Hoping for your comments.
> >
> > Best Regards,
> >
> > -Yi
> >
> >
> > ------------------------------------------------------------------------
> >
> > root:~> ./latency -t0 -p 300 -f
> >
> >
> > root:~> cat /proc/ipipe/trace/max
> > I-pipe worst-case tracing service on 2.6.16.11.ADI-2006R1blackfin/ipipe-
> 1.4-03
> > ------------------------------------------------------------
> > Begin: 240373976502 cycles, Trace Points: 24 (-10/+1), Length: 270 us
> >
> >  +----- Hard IRQs ('|': locked)
> >  |+---- <unused>
> >  ||+--- <unused>
> >  |||+-- Xenomai
> >  ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
> >  |||||                        +---------- Delay flag ('+': > 1 us, '!':
> > 10 us)
> >  |||||                        |        +- NMI noise ('N')
> >  |||||                        |        |
> >       Type    User Val.   Time    Delay  Function (Parent)
> >  |  + begin   0x80000001   -20     0.910  ___ipipe_dispatch_event+0xb4
> (__start+0xfffff000)
> >  |  + end     0x80000001   -19    6.360  ___ipipe_dispatch_event+0x11a
> (__start+0xfffff000)
> >     + func                 -13    1.440  ___ipipe_syscall_root+0xe
> (system_call+0x2e)
> >     + func                 -11    0.890  ___ipipe_dispatch_event+0xa
> (___ipipe_syscall_root+0x24)
> >  |  + begin   0x80000001   -10    1.280  ___ipipe_dispatch_event+0x20
> (__start+0xfffff000)
> >  |  + end     0x80000001    -9     1.530  ___ipipe_dispatch_event+0x88
> (__start+0xfffff000)
> >     + func                  -7    2.730  _hisyscall_event+0x10
> (___ipipe_dispatch_event+0xa0)
> >     + func                  -5    1.440  ___rt_task_wait_period+0xc
> (_hisyscall_event+0x134)
> >     + func                  -3    1.820  _rt_task_wait_period+0xc
> (___rt_task_wait_period+0x4c)
> >     + func                  -1    0.950  _xnpod_wait_thread_period+0xa
> (_rt_task_wait_period+0x4c)
> >> |  + begin   0x80000001     1+   1.900  _xnpod_wait_thread_period+0x30
> (__start+0xfffff000)
> > :|  # func                   2+   1.640  _xnpod_suspend_thread+0xa
> (_xnpod_wait_thread_period+0x78)
> > :|  # func                   4    0.892  _xnpod_schedule+0xa
> (_xnpod_suspend_thread+0x168)
> > :|  # [   50] -<?>-   99     5+   1.658  _xnpod_schedule+0x112
> (__start+0xfffff000)
> > :|  # [    1] init    -1     7+   2.350  _xnpod_schedule+0x270
> (__start+0xfffff000)
> > :    +func                   9! 235.540  ___ipipe_walk_pipeline+0x1e
> (___ipipe_handle_irq+0x14c)
>
> Looks like there are some inconsistencies in the IRQ state tracing: You
> schedule from latency (PID 50, now of unknown name "-<?>-", priority 99)
> to the init process under Linux (PID 1, "init", ROOT thread priority
> -1). After that switch the IRQs are very likely earlier reenabled, e.g.
> on interrupt return. Compare the instrumentation of x86 with your version.
>
>
> Moreover, IRQ entry and exit should also be traced. This doesn't seem to
> happen yet, otherwise we would see some "begin" entry with the IRQ
> number as user value. Again, check the x86 code for
> ipipe_trace_begin/end passages.
>
> > :|   +func                 244+   1.330  ___ipipe_grab_irq+0x10
> (_evt_timer+0x7c)
> > :|   +func                 246+   1.560  ___ipipe_handle_irq+0x10
> (___ipipe_grab_irq+0x32)
> > :|   +func                 247+   2.120  _rthal_timer_ack+0x8
> (___ipipe_handle_irq+0x174)
> > :|   +func                 249+   1.680  ___ipipe_dispatch_wired+0x14
> (___ipipe_handle_irq+0x74)
> > :|  # func                 251+   1.270  _xnintr_clock_handler+0x10
> (___ipipe_dispatch_wired+0xa6)
> > :|  # func                 252+   1.090  _xnintr_irq_handler+0x18
> (_xnintr_clock_handler+0x18)
> > :|  # func                 254+   1.380  _xnpod_announce_tick+0xc
> (_xnintr_irq_handler+0x2e)
> > :|  # func                 255+   1.470  _xntimer_do_tick_aperiodic+0x14
> (_xnpod_announce_tick+0x16)
> > :|  # func                 256+   1.400  _xnthread_periodic_handler+0xa
> (_xntimer_do_tick_aperiodic+0x1d6)
> > :|  # func                 258     0.790  _xnpod_resume_thread+0xa
> (_xnthread_periodic_handler+0x1e)
> > :|  # [   50] -<?>-   99   259+   2.510  _xnpod_resume_thread+0x4a
> (__start+0xfffff000)
> > :|  # func                 261+   1.890  _xnpod_schedule+0xa
> (_xnintr_irq_handler+0x82)
> > :|  # func                 263+   1.380  _xnpod_schedule_deferred+0xc
> (___ipipe_call_irqtail+0x1e)
> > :|  # func                 264    0.970  _xnpod_schedule+0xa
> (_xnpod_schedule_deferred+0x28)
> > :|  # [    1] init    -1   265+   1.950  _xnpod_schedule+0x112
> (__start+0xfffff000)
> > :|  # [   50] -<?>-   99   267+   1.560  _xnpod_schedule+0x270
> (__start+0xfffff000)
> > :|  # func                 269    0.690  ___ipipe_restore_pipeline_head+0xa
> (_xnpod_wait_thread_period+0xfa)
> > <|  + end     0x80000000   270    2.340  ___ipipe_restore_pipeline_head+0x90
> (__start+0xfffff000)
> >  |  + begin   0x80000001   272    0.000  ___ipipe_dispatch_event+0xb4
> (__start+0xfffff000)
> >
> >
> > root:~> cat /proc/ipipe/trace/frozen
> > I-pipe frozen back-tracing service on 2.6.16.11.ADI-2006R1blackfin
> /ipipe-1.4-03
> > ------------------------------------------------------------
> > Freeze: 234204981352 cycles, Trace Points: 30 (+10)
> >
> >  +----- Hard IRQs ('|': locked)
> >  |+---- <unused>
> >  ||+--- <unused>
> >  |||+-- Xenomai
> >  ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
> >  |||||                        +---------- Delay flag ('+': > 1 us, '!':
> > 10 us)
> >  |||||                        |        +- NMI noise ('N')
> >  |||||                        |        |
> >       Type    User Val.   Time    Delay  Function (Parent)
> > :|  #*func                 -58     0.990  _xnpod_schedule+0xa
> (_xnpod_schedule_deferred+0x28)
> > :|  #*[    1] init    -1   -57+   2.210  _xnpod_schedule+0x112
> (__start+0xfffff000)
> > :|  #*[   50] -<?>-   99   -55+   2.380  _xnpod_schedule+0x270
> (__start+0xfffff000)
> > :|  #*func                 -52    0.710  ___ipipe_restore_pipeline_head+0xa
> (_xnpod_wait_thread_period+0xfa)
> > :|  +*end     0x80000000   -52+   3.020  ___ipipe_restore_pipeline_head+0x90
> (__start+0xfffff000)
> > :|  +*begin   0x80000001   -49+   1.610  ___ipipe_dispatch_event+0xb4
> (__start+0xfffff000)
> > :|  +*end     0x80000001   -47+   6.900  ___ipipe_dispatch_event+0x11a
> (__start+0xfffff000)
> > :   +*func                 -40+   1.330  ___ipipe_syscall_root+0xe
> (system_call+0x2e)
> > :   +*func                 -39    0.700  ___ipipe_dispatch_event+0xa
> (___ipipe_syscall_root+0x24)
> > :|  +*begin   0x80000001   -38+   1.340  ___ipipe_dispatch_event+0x20
> (__start+0xfffff000)
> > :|  +*end     0x80000001   -37+   1.360  ___ipipe_dispatch_event+0x88
> (__start+0xfffff000)
> > :   +*func                 -36+   2.080  _hisyscall_event+0x10
> (___ipipe_dispatch_event+0xa0)
> > :   +*func                 -33+   2.530  ___rt_timer_tsc2ns+0x14
> (_hisyscall_event+0x134)
> > :   +*func                 -31+   1.490  _rt_timer_tsc2ns+0xa
> (___rt_timer_tsc2ns+0x5e)
> > :   +*func                 -29+   1.370  ___muldi3+0x14
> (_rt_timer_tsc2ns+0x58)
> > :   +*func                 -28+   2.330  ___muldi3+0x14
> (_rt_timer_tsc2ns+0x6a)
> > :   +*func                 -26+   4.460  ___div64_32+0xe
> (_rt_timer_tsc2ns+0x1d8)
> > :|  +*begin   0x80000001   -21+   1.320  ___ipipe_dispatch_event+0xb4
> (__start+0xfffff000)
> > :|  +*end     0x80000001   -20+   3.380  ___ipipe_dispatch_event+0x11a
> (__start+0xfffff000)
> > :   +*func                 -17+   1.330  ___ipipe_syscall_root+0xe
> (system_call+0x2e)
> > :   +*func                 -15    0.700  ___ipipe_dispatch_event+0xa
> (___ipipe_syscall_root+0x24)
> > :|  +*begin   0x80000001   -15+   1.340  ___ipipe_dispatch_event+0x20
> (__start+0xfffff000)
> > :|  +*end     0x80000001   -13+   1.360  ___ipipe_dispatch_event+0x88
> (__start+0xfffff000)
> > :   +*func                 -12+   2.100  _hisyscall_event+0x10
> (___ipipe_dispatch_event+0xa0)
> > :   +*func                 -10+   2.230  _xnshadow_sys_trace+0xc
> (_hisyscall_event+0x134)
> > :   +*func                  -8+   1.470  _ipipe_trace_frozen_reset+0xa
> (_xnshadow_sys_trace+0xb2)
> > :   +*func                  -6+   1.110  ___ipipe_global_path_lock+0xa
> (_ipipe_trace_frozen_reset+0xe)
> > :|  +*begin   0x80000001    -5+   3.260  ___ipipe_global_path_lock+0x1a
> (__start+0xfffff000)
> > :|  +*end     0x80000001    -2+   1.160  ___ipipe_global_path_unlock+0x44
> (__start+0xfffff000)
> > <   +*freeze  0x0001ecf6     1     1.180  _xnshadow_sys_trace+0x9c
> (__start+0xfffff000)
>
> 126 us (the user value of "freeze"), that looks ok given the overhead of
> the tracer. If you extend your backtrace in this case to cover that
> whole period, you can analyse what blocked the timer IRQ from waking up
> earlier.
>
> >  |  +*begin   0x80000001     2    0.930  ___ipipe_dispatch_event+0xb4
> (__start+0xfffff000)
> >  |  +*end     0x80000001     3    4.320  ___ipipe_dispatch_event+0x11a
> (__start+0xfffff000)
> >     +*func                   7     1.160  ___ipipe_syscall_root+0xe
> (system_call+0x2e)
> >     +*func                   8    0.910  ___ipipe_dispatch_event+0xa
> (___ipipe_syscall_root+0x24)
> >  |  +*begin   0x80000001     9    0.950  ___ipipe_dispatch_event+0x20
> (__start+0xfffff000)
> >  |  +*end     0x80000001    10    1.350  ___ipipe_dispatch_event+0x88
> (__start+0xfffff000)
> >     +*func                  11    1.910  _hisyscall_event+0x10
> (___ipipe_dispatch_event+0xa0)
> >     +*func                  13     1.310  ___rt_task_wait_period+0xc
> (_hisyscall_event+0x134)
> >     +*func                  15    1.930  _rt_task_wait_period+0xc
> (___rt_task_wait_period+0x4c)
> >     +*func                  16    0.000  _xnpod_wait_thread_period+0xa
> (_rt_task_wait_period+0x4c)
> >
>
> Jan
>
>
>
>

[-- Attachment #1.2: Type: text/html, Size: 25580 bytes --]

[-- Attachment #2: ipipe-tracer-new.log --]
[-- Type: text/x-log, Size: 23166 bytes --]

root:~> ./latency -t2 -p200 -f

root:~> cat /proc/ipipe/trace/frozen
I-pipe frozen back-tracing service on 2.6.16.27-ADI-2006R2/ipipe-1.5-00
------------------------------------------------------------
Freeze: 144879102832 cycles, Trace Points: 200 (+50)

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
:    #func                -385    0.728  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -385+   1.322  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -383+   1.680  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:    +func                -382+   1.720  ___sched_text_start+0xe (_worker_thread+0x116)
:    +func                -380+   1.378  _profile_hit+0xe (___sched_text_start+0x4c)
:    +func                -379+   1.522  _sched_clock+0xc (___sched_text_start+0x7c)
:    +func                -377+   2.620  ___muldi3+0xc (_sched_clock+0x26)
:    +func                -374    0.870  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -374+   1.108  ___ipipe_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -372+   1.822  ___ipipe_stall_root+0x30 (__start+0xfffff000)
:    #func                -371+   1.800  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func                -369+   2.560  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -366+   1.378  ___sched_text_start+0x1e6 (__start+0xfffff000)
:|   #end     0x80000001  -365+   3.712  ___sched_text_start+0x1f4 (__start+0xfffff000)
:    #func                -361    0.940  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -360+   1.460  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
:|   #end     0x80000001  -359+   1.508  ___ipipe_dispatch_event+0xcc (__start+0xfffff000)
:    #func                -357+   1.442  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -356+   2.360  _schedule_event+0x7e (__start+0xfffff000)
:|  *#func                -353+   1.100  _xnpod_schedule_runnable+0xc (_schedule_event+0xa8)
:|  *#[    0] swapper  0  -352+   1.788  _xnpod_schedule_runnable+0x28 (__start+0xfffff000)
:|  *#func                -351+   1.262  ___ipipe_restore_pipeline_head+0xa (_schedule_event+0x100)
:|   #end     0x80000000  -349+   1.050  ___ipipe_restore_pipeline_head+0x94 (__start+0xfffff000)
:|   #begin   0x80000001  -348+   1.400  ___ipipe_dispatch_event+0xfa (__start+0xfffff000)
:|   #end     0x80000001  -347    0.818  ___ipipe_dispatch_event+0x17a (__start+0xfffff000)
:    #begin   0x80000000  -346+   4.262  ___sched_text_start+0x2f0 (__start+0xfffff000)
:|   #end     0x80000000  -342+   1.340  ___sched_text_start+0x5ca (__start+0xfffff000)
:    #func                -340    0.980  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -339+   1.128  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -338+   1.352  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:    +func                -337    0.870  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x3d8)
:|   +begin   0x80000001  -336+   1.270  ___ipipe_test_and_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -335+   1.198  ___ipipe_test_and_stall_root+0x36 (__start+0xfffff000)
:    #func                -334+   1.252  ___ipipe_restore_root+0xa (___sched_text_start+0x4ec)
:    #func                -332    0.980  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -331+   1.290  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -330!  20.168  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:    +func                -310+   1.742  ___ipipe_syscall_root+0x14 (_system_call+0x2e)
:    +func                -308    0.930  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x40)
:|   +begin   0x80000001  -307+   1.670  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
:|  + end     0x80000001  -306+   1.648  ___ipipe_dispatch_event+0xcc (__start+0xfffff000)
:   + func                -304+   2.092  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xe2)
:   + func                -302+   1.770  _xnpod_schedule+0xa (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -300+   1.270  _xnpod_schedule+0x64 (__start+0xfffff000)
:|  + end     0x80000001  -299+   1.178  _xnpod_schedule+0x72 (__start+0xfffff000)
:|  + begin   0x80000001  -298+   1.322  _xnpod_schedule+0xc4 (__start+0xfffff000)
:|  # [    0] swapper  0  -296+   2.330  _xnpod_schedule+0x104 (__start+0xfffff000)
:|  # func                -294+   1.050  ___ipipe_restore_pipeline_head+0xa (_xnpod_schedule+0x2b0)
:|  + end     0x80000000  -293+   2.258  ___ipipe_restore_pipeline_head+0x94 (__start+0xfffff000)
:|  + func                -291    0.902  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  + begin   0x00000006  -290+   2.130  ___ipipe_grab_irq+0x34 (__start+0xfffff000)
:|  + func                -288+   1.510  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  + func                -286+   1.208  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|  + func                -285+   2.302  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # func                -283+   1.980  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xb6)
:|  # func                -281+   1.440  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func                -279+   1.278  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  # func                -278+   1.932  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func                -276+   1.590  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  # func                -274+   1.880  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  # func                -273+   1.488  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func                -271+   2.412  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func                -269+   4.130  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func                -265+   6.780  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # func                -258+   1.468  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  # func                -256+   1.612  _xnpod_get_time+0xa (_timer_proc+0x2a)
:|  # func                -255+   1.400  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func                -253+   1.530  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func                -252+   2.758  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func                -249+   5.582  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # begin   0x00027b80  -243+   1.370  _timer_proc+0x44 (__start+0xfffff000)
:|  # func                -242+   1.480  _xntimer_start+0xa (_timer_proc+0x74)
:|  # func                -241+   1.928  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
:|  # func                -239+   1.452  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
:|  # func                -237+   2.180  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)
:|  # func                -235+   4.120  ___div64_32+0x10 (_xntimer_do_start_aperiodic+0x5ae)
:|  # func                -231+   1.278  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1bc)
:|  # func                -230+   4.222  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1d0)
:|  # func                -225+   2.030  _xnpod_schedule_deferred+0xc (___ipipe_call_irqtail+0x1e)
:   + func                -223+   1.140  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x126)
:   + end     0x00000006  -222+   3.448  ___ipipe_grab_irq+0x42 (__start+0xfffff000)
:|  + begin   0x80000001  -219+   1.502  ___ipipe_dispatch_event+0xfa (__start+0xfffff000)
:|   +end     0x80000001  -217+   1.930  ___ipipe_dispatch_event+0xcc (__start+0xfffff000)
:    +func                -215+   2.650  _losyscall_event+0x14 (___ipipe_dispatch_event+0xe2)
:    +func                -213+   1.608  _sys_rtdm_ioctl+0xa (_losyscall_event+0x94)
:    +func                -211+   1.762  __rtdm_ioctl+0xa (_sys_rtdm_ioctl+0x1c)
:    +func                -209+   1.280  _rtdm_context_get+0x10 (__rtdm_ioctl+0x20)
:|   +begin   0x80000001  -208+   2.500  _rtdm_context_get+0x24 (__start+0xfffff000)
:|  *+func                -206+   1.928  ___ipipe_test_and_stall_root+0xe (_rtdm_context_get+0x58)
:|  *#func                -204+   1.262  ___ipipe_restore_root+0xa (_rtdm_context_get+0x6e)
:|  *#func                -202    0.980  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:|  *+end     0x80000000  -201+   1.490  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:   *+func                -200    0.728  ___ipipe_restore_pipeline_head+0xa (_rtdm_context_get+0x82)
:|  *+begin   0x80000001  -199+   1.322  ___ipipe_restore_pipeline_head+0x1e (__start+0xfffff000)
:|   +end     0x80000000  -198+   2.960  ___ipipe_restore_pipeline_head+0x94 (__start+0xfffff000)
:    +func                -195+   1.690  _rt_tmbench_ioctl_nrt+0x16 (__rtdm_ioctl+0x4a)
:    +func                -193    0.808  ___ipipe_test_and_stall_root+0xe (__rtdm_ioctl+0x50)
:|   +begin   0x80000001  -192+   1.302  ___ipipe_test_and_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -191+   1.590  ___ipipe_test_and_stall_root+0x36 (__start+0xfffff000)
:    #func                -189+   1.220  ___ipipe_restore_root+0xa (__rtdm_ioctl+0x64)
:    #func                -188    0.728  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -187+   1.322  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -186+   2.180  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:    +func                -184+   1.690  _xnshadow_harden+0x12 (_losyscall_event+0xb2)
:    +func                -182    0.708  ___ipipe_test_and_stall_root+0xe (_xnshadow_harden+0x3e)
:|   +begin   0x80000001  -182+   1.302  ___ipipe_test_and_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -180+   1.790  ___ipipe_test_and_stall_root+0x36 (__start+0xfffff000)
:    #func                -179+   1.330  ___ipipe_restore_root+0xa (_xnshadow_harden+0x56)
:    #func                -177    0.728  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -176+   1.322  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -175+   1.200  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:|   +begin   0x80000001  -174+   1.520  _xnshadow_harden+0x76 (__start+0xfffff000)
:|   +end     0x80000001  -172+   1.638  _xnshadow_harden+0x8c (__start+0xfffff000)
:    +func                -171+   1.612  ___wake_up_sync+0xc (_xnshadow_harden+0xa4)
:    +func                -169    0.840  ___ipipe_test_and_stall_root+0xe (___wake_up_sync+0x18)
:|   +begin   0x80000001  -168+   1.270  ___ipipe_test_and_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -167+   1.598  ___ipipe_test_and_stall_root+0x36 (__start+0xfffff000)
:    #func                -165+   1.782  ___wake_up_common+0xc (___wake_up_sync+0x38)
:    #func                -164+   2.160  _default_wake_function+0xa (___wake_up_common+0x38)
:    #func                -162+   1.460  _try_to_wake_up+0xa (_default_wake_function+0x10)
:    #func                -160    0.728  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #begin   0x80000001  -159+   1.322  ___ipipe_test_and_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -158+   1.760  ___ipipe_test_and_stall_root+0x36 (__start+0xfffff000)
:    #func                -156+   1.300  _sched_clock+0xc (_try_to_wake_up+0x7e)
:    #func                -155+   2.088  ___muldi3+0xc (_sched_clock+0x26)
:    #func                -153+   2.142  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:    #func                -151+   1.250  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:    #func                -149    0.890  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -149+   1.128  ___ipipe_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -147+   1.592  ___ipipe_stall_root+0x30 (__start+0xfffff000)
:    #func                -146+   1.190  ___ipipe_restore_root+0xa (___wake_up_sync+0x3e)
:    #func                -145    0.890  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -144+   1.128  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -143+   1.492  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:    +func                -141+   1.510  ___sched_text_start+0xe (_xnshadow_harden+0xa8)
:    +func                -140+   1.650  _profile_hit+0xe (___sched_text_start+0x4c)
:    +func                -138+   1.118  _sched_clock+0xc (___sched_text_start+0x7c)
:    +func                -137+   2.802  ___muldi3+0xc (_sched_clock+0x26)
:    +func                -134    0.870  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -133+   1.270  ___ipipe_stall_root+0x22 (__start+0xfffff000)
:|   #end     0x80000001  -132+   1.628  ___ipipe_stall_root+0x30 (__start+0xfffff000)
:    #func                -130+   1.492  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func                -129+   2.860  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -126+   1.540  ___sched_text_start+0x1e6 (__start+0xfffff000)
:|   #end     0x80000001  -124+   3.238  ___sched_text_start+0x1f4 (__start+0xfffff000)
:    #func                -121    0.952  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -120+   1.470  ___ipipe_dispatch_event+0x20 (__start+0xfffff000)
:|   #end     0x80000001  -119+   1.610  ___ipipe_dispatch_event+0xcc (__start+0xfffff000)
:    #func                -117+   1.078  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -116+   1.632  ___ipipe_dispatch_event+0xfa (__start+0xfffff000)
:|   #end     0x80000001  -114    0.980  ___ipipe_dispatch_event+0x17a (__start+0xfffff000)
:    #begin   0x80000000  -113+   2.120  ___sched_text_start+0x2f0 (__start+0xfffff000)
:|   #func                -111+   1.398  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x678)
:|   #func                -110+   1.272  ___ipipe_restore_root+0xa (___sched_text_start+0x68c)
:|   #func                -109+   4.110  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -105+   1.740  ___sched_text_start+0x5ca (__start+0xfffff000)
:    #func                -103    0.728  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -102+   1.322  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
:|   +end     0x80000000  -101+   2.590  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
:    +func                 -98+   1.550  _kthread_should_stop+0x10 (_gatekeeper_thread+0xec)
:|   +begin   0x80000001   -97+   2.898  _gatekeeper_thread+0x100 (__start+0xfffff000)
:|  *+func                 -94+   1.432  _xnpod_resume_thread+0xa (_gatekeeper_thread+0x186)
:|  *+[   51] -<?>-    0   -92+   5.450  _xnpod_resume_thread+0x4a (__start+0xfffff000)
:|  *+func                 -87+   1.100  _xnpod_schedule_runnable+0xc (_gatekeeper_thread+0x1be)
:|  *+[    0] swapper -1   -86+   2.788  _xnpod_schedule_runnable+0x28 (__start+0xfffff000)
:|  *+func                 -83+   2.852  _xnpod_schedule+0xa (_gatekeeper_thread+0x1ce)
:|  *+func                 -80+   2.970  _ipipe_trigger_irq+0x12 (_xnpod_schedule+0x2e6)
:|  *+func                 -77+   2.120  ___ipipe_handle_irq+0x1e (_ipipe_trigger_irq+0x86)
:|  *+func                 -75+   2.058  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  *+func                 -73+   1.722  ___ipipe_restore_pipeline_head+0xa (_gatekeeper_thread+0x138)
:|   +func                 -71+   2.130  ___ipipe_walk_pipeline+0x18 (___ipipe_restore_pipeline_head+0xd0)
:|  + func                 -69+   2.410  _ipipe_suspend_domain+0xa (___ipipe_walk_pipeline+0xac)
:|  + func                 -67+   5.048  ___ipipe_sync_stage+0xe (_ipipe_suspend_domain+0x4e)
:|  # func                 -62+   2.452  _xnpod_schedule_handler+0xe (___ipipe_sync_stage+0x186)
:|  # func                 -59+   2.130  _xnpod_schedule+0xa (_xnpod_schedule_handler+0x5c)
:|  # [    0] swapper -1   -57+   7.180  _xnpod_schedule+0x104 (__start+0xfffff000)
:|  # end     0x80000000   -50+   2.278  ___sched_text_start+0x5ca (__start+0xfffff000)
:|  # func                 -48    0.922  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x00000006   -47+   1.820  ___ipipe_grab_irq+0x34 (__start+0xfffff000)
:|  # func                 -45+   1.530  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func                 -43+   1.168  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|  # func                 -42+   1.102  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # end     0x00000006   -41+   2.010  ___ipipe_grab_irq+0x42 (__start+0xfffff000)
:   # func                 -39+   1.090  _ipipe_unstall_pipeline_head+0x8 (_xnshadow_harden+0xce)
:|  # begin   0x80000001   -38+   2.308  _ipipe_unstall_pipeline_head+0x18 (__start+0xfffff000)
:|  + func                 -36+   1.892  ___ipipe_sync_stage+0xe (_ipipe_unstall_pipeline_head+0x74)
:|  # func                 -34+   1.580  _xnintr_clock_handler+0x10 (___ipipe_sync_stage+0x186)
:|  # func                 -32+   1.460  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func                 -31+   1.728  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  # func                 -29+   2.442  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func                 -27+   1.590  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  # func                 -25+   1.880  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  # func                 -23+   1.598  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func                 -21+   2.432  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func                 -19+   4.130  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func                 -15+   6.770  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # func                  -8+   1.808  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  # func                  -6+   1.432  _ipipe_trace_frozen_reset+0xa (_eval_inner_loop+0xe0)
:|  # func                  -5+   4.400  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
<|  # freeze  0x00010ba8     1    2.370  _eval_inner_loop+0xe6 (__start+0xfffff000)
 |  # func                   3    1.308  _xnpod_get_time+0xa (_timer_proc+0x2a)
 |  # func                   4    1.482  ___muldi3+0xc (_xnpod_get_time+0x52)
 |  # func                   6    1.540  ___muldi3+0xc (_xnpod_get_time+0x6e)
 |  # func                   7    3.020  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
 |  # func                  10    5.708  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
 |  # begin   0x0001992a    16    1.842  _timer_proc+0x44 (__start+0xfffff000)
 |  # func                  18    1.940  _xntimer_start+0xa (_timer_proc+0x74)
 |  # func                  20    2.420  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
 |  # func                  22    1.498  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
 |  # func                  24    2.972  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)
 |  # func                  27    4.560  ___div64_32+0x10 (_xntimer_do_start_aperiodic+0x5ae)
 |  # func                  31    1.660  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1bc)
 |  # func                  33    5.818  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1d0)
 |  + end     0x80000000    39    1.922  _ipipe_unstall_pipeline_head+0x52 (__start+0xfffff000)
    + func                  41    1.950  _sys_rtdm_ioctl+0xa (_losyscall_event+0x94)
    + func                  43    1.730  __rtdm_ioctl+0xa (_sys_rtdm_ioctl+0x1c)
    + func                  44    0.918  _rtdm_context_get+0x10 (__rtdm_ioctl+0x20)
 |  + begin   0x80000001    45    1.522  _rtdm_context_get+0x24 (__start+0xfffff000)
 |  # func                  47    2.040  ___ipipe_test_and_stall_root+0xe (_rtdm_context_get+0x58)
 |  #*func                  49    1.240  ___ipipe_restore_root+0xa (_rtdm_context_get+0x6e)
 |  #*func                  50    0.828  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
 |  # end     0x80000000    51    1.402  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
    # func                  52    0.890  ___ipipe_restore_pipeline_head+0xa (_rtdm_context_get+0x82)
 |  # begin   0x80000001    53    0.960  ___ipipe_restore_pipeline_head+0x1e (__start+0xfffff000)
 |  + end     0x80000000    54    2.228  ___ipipe_restore_pipeline_head+0x94 (__start+0xfffff000)
    + func                  56    2.122  _rt_tmbench_ioctl_rt+0x14 (__rtdm_ioctl+0x4a)
    + func                  58    2.050  _rtdm_event_wait+0x8 (_rt_tmbench_ioctl_rt+0x58)
    + func                  60    1.080  _rtdm_event_timedwait+0xa (_rtdm_event_wait+0x14)
 |  + begin   0x80000001    62    2.288  _rtdm_event_timedwait+0x22 (__start+0xfffff000)
 |  # func                  64    3.692  _xnsynch_sleep_on+0x14 (_rtdm_event_timedwait+0xa8)
 |  # func                  68    3.470  _xnpod_suspend_thread+0xa (_xnsynch_sleep_on+0x282)
 |  # func                  71    1.890  _xnpod_schedule+0xa (_xnpod_suspend_thread+0x174)
 |  # [   51] -<?>-    0    73    3.278  _xnpod_schedule+0x104 (__start+0xfffff000)
 |  # [    8] gatekee -1    76    3.392  _xnpod_schedule+0x44e (__start+0xfffff000)
 |   +end     0x80000000    80    1.320  ___ipipe_restore_pipeline_head+0x94 (__start+0xfffff000)
 |   +begin   0x80000001    81    1.150  _gatekeeper_thread+0xac (__start+0xfffff000)
 |   +end     0x80000001    82    1.298  _gatekeeper_thread+0xc0 (__start+0xfffff000)
     +func                  83    0.902  ___ipipe_test_and_stall_root+0xe (_gatekeeper_thread+0xd0)
 |   +begin   0x80000001    84    0.940  ___ipipe_test_and_stall_root+0x22 (__start+0xfffff000)
 |   #end     0x80000001    85    1.510  ___ipipe_test_and_stall_root+0x36 (__start+0xfffff000)
     #func                  87    1.158  ___ipipe_restore_root+0xa (_gatekeeper_thread+0x154)
     #func                  88    0.922  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
     #begin   0x80000000    89    0.950  ___ipipe_unstall_root+0x1a (__start+0xfffff000)
 |   +end     0x80000000    90    1.600  ___ipipe_unstall_root+0x58 (__start+0xfffff000)
     +func                  91    1.558  ___sched_text_start+0xe (_gatekeeper_thread+0xe8)
     +func                  93    1.452  _profile_hit+0xe (___sched_text_start+0x4c)
     +func                  94    1.400  _sched_clock+0xc (___sched_text_start+0x7c)
     +func                  96    2.280  ___muldi3+0xc (_sched_clock+0x26)
     +func                  98    0.708  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
 |   +begin   0x80000001    99    0.000  ___ipipe_stall_root+0x22 (__start+0xfffff000)

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [Xenomai-core] Re: [Adeos-main] ipip tracer porting for blackfin
  2006-11-10 13:54   ` Li Yi (Adam)
@ 2006-11-10 14:27     ` Philippe Gerum
  2006-11-10 16:11     ` Jan Kiszka
  1 sibling, 0 replies; 9+ messages in thread
From: Philippe Gerum @ 2006-11-10 14:27 UTC (permalink / raw)
  To: Li Yi (Adam); +Cc: adeos-main, Jan Kiszka, xenomai

On Fri, 2006-11-10 at 21:54 +0800, Li Yi (Adam) wrote:
> Hi Jan and Philippe,
> 
> I have tested the adeos-ipipe-bf53x-CVS-20061008-1.5-00.patch in
> Xenomai-2.2.5 with latest Blackfin uclinux toolchain 06r2 release:
> http://blackfin.uclinux.org/frs/download.php/1136/blackfin-toolchain-06r2-5.i386.rpm. Since the toolchain changed a little for the "mcount" call, the ipipe-tracer patch need to be updated a little.
> 
> Also I add trace for IRQ entry and exit as Jan suggested. Here is the
> details of the patch (with my explanation) 
> 
> diff -Nur linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe-mcount.S
> linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-mcount.S
> --- linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe-mcount.S
> 2006-11-10 21:04:21.000000000 +0800
> +++ linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-mcount.S
> 2006-11-10 16:41:11.000000000 +0800
> @@ -12,7 +12,7 @@
> 
>  .align 4       /* just in case */
> 
> -ENTRY(_mcount)
> +ENTRY(__mcount)
>         [--sp] = ( r7:0, p5:0 ); 
>         [--sp] = ASTAT;
>         p1.L = _ipipe_trace_enable;
> diff -Nur linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe-root.c
> linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-root.c
> --- linux-2.6.x-ipipe-orig/arch/blackfin/kernel/ipipe- root.c
> 2006-11-10 21:04:21.000000000 +0800
> +++ linux-2.6.x-ipipe/arch/blackfin/kernel/ipipe-root.c 2006-11-10
> 16:41:59.000000000 +0800
> @@ -281,6 +281,6 @@
>  EXPORT_SYMBOL(show_stack);
> 
>  #ifdef CONFIG_IPIPE_TRACE_MCOUNT 
> -void notrace mcount(void);
> -EXPORT_SYMBOL(mcount);
> +void notrace _mcount(void);
> +EXPORT_SYMBOL(_mcount);
>  #endif /* CONFIG_IPIPE_TRACE_MCOUNT */
> 
> <Yi's note>: The  gcc  will add "Call _ _mcount" to each function
> entry when there is "-pg" option 

Ok.

> 
> diff -Nur linux-2.6.x-ipipe-orig/arch/blackfin/mach-common/interrupt.S
> linux-2.6.x-ipipe/arch/blackfin/mach-common/interrupt.S
> --- linux-2.6.x-ipipe-orig/arch/blackfin/mach-common/interrupt.S
> 2006-11-10 21:04: 21.000000000 +0800
> +++ linux-2.6.x-ipipe/arch/blackfin/mach-common/interrupt.S
> 2006-11-10 16:43:55.000000000 +0800
> @@ -304,7 +304,10 @@
>         p0.l = _irq_flags;
>         p0.h = _irq_flags;
>         r0 = [p0]; 
> -       sti r0
> +       sti r0;
> +       /* Reenable interrupts */
> +       [--sp] = reti;
> +       r0 = [sp++];
>         rts;
> 
>  #endif /* CONFIG_IPIPE */
> 
> 
> <Yi's Note>: I used the ipipe-trace to trace the latency test case:
> "latency -t2 -p200 -f". And found that "__ipipe_call_irqtail()" will
> disable global interrupts, putting the Blackfin to a state like: 
> "IMASK: 0x0000FFFF, ILAT: 0x00000040, IPEND: 0x0000c010" - that is,
> Core Timer irq latched, but since IPEND[4] is set, the irq handler
> cannot be invoked - thus longer latency.
> 

Good catch, I forgot the fact that triggering an IRQ to run the
scheduling tail would raise the global disable bit. Applied, thanks.

[...]

-- 
Philippe.




^ permalink raw reply	[flat|nested] 9+ messages in thread

* [Xenomai-core] Re: [Adeos-main] ipip tracer porting for blackfin
  2006-11-10 13:54   ` Li Yi (Adam)
  2006-11-10 14:27     ` Philippe Gerum
@ 2006-11-10 16:11     ` Jan Kiszka
  2006-11-13  3:51       ` Li Yi (Adam)
  1 sibling, 1 reply; 9+ messages in thread
From: Jan Kiszka @ 2006-11-10 16:11 UTC (permalink / raw)
  To: Li Yi (Adam), Philippe Gerum; +Cc: adeos-main, xenomai

[-- Attachment #1: Type: text/plain, Size: 4914 bytes --]

Li Yi (Adam) wrote:
> Hi Jan and Philippe,
> 

...

> 
> Also attached the new ipiper-tracer log.

Damn, no fitting reference trace at hand, but something seems to be
fishy. See comments/questions below.

> ------------------------------------------------------------------------
> 
> root:~> ./latency -t2 -p200 -f
> 
> root:~> cat /proc/ipipe/trace/frozen
> I-pipe frozen back-tracing service on 2.6.16.27-ADI-2006R2/ipipe-1.5-00
> ------------------------------------------------------------
> Freeze: 144879102832 cycles, Trace Points: 200 (+50)
> 
>  +----- Hard IRQs ('|': locked)
>  |+---- <unused>
>  ||+--- <unused>
>  |||+-- Xenomai
>  ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
>  |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
>  |||||                        |        +- NMI noise ('N')
>  |||||                        |        |
>       Type    User Val.   Time    Delay  Function (Parent)
...

> :    +func                 -98+   1.550  _kthread_should_stop+0x10 (_gatekeeper_thread+0xec)
> :|   +begin   0x80000001   -97+   2.898  _gatekeeper_thread+0x100 (__start+0xfffff000)
> :|  *+func                 -94+   1.432  _xnpod_resume_thread+0xa (_gatekeeper_thread+0x186)
> :|  *+[   51] -<?>-    0   -92+   5.450  _xnpod_resume_thread+0x4a (__start+0xfffff000)

Here we go: harden the xeno thread related to pid 51 (likely some now
gone part of latency).

http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L1555

> :|  *+func                 -87+   1.100  _xnpod_schedule_runnable+0xc (_gatekeeper_thread+0x1be)
> :|  *+[    0] swapper -1   -86+   2.788  _xnpod_schedule_runnable+0x28 (__start+0xfffff000)

This correlates to
http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2529

xnpod_schedule_runnable returns without actually switching (off-topic:
seems to be the only use-case for this function right now...).

> :|  *+func                 -83+   2.852  _xnpod_schedule+0xa (_gatekeeper_thread+0x1ce)
> :|  *+func                 -80+   2.970  _ipipe_trigger_irq+0x12 (_xnpod_schedule+0x2e6)
> :|  *+func                 -77+   2.120  ___ipipe_handle_irq+0x1e (_ipipe_trigger_irq+0x86)
> :|  *+func                 -75+   2.058  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
> :|  *+func                 -73+   1.722  ___ipipe_restore_pipeline_head+0xa (_gatekeeper_thread+0x138)
> :|   +func                 -71+   2.130  ___ipipe_walk_pipeline+0x18 (___ipipe_restore_pipeline_head+0xd0)
> :|  + func                 -69+   2.410  _ipipe_suspend_domain+0xa (___ipipe_walk_pipeline+0xac)
> :|  + func                 -67+   5.048  ___ipipe_sync_stage+0xe (_ipipe_suspend_domain+0x4e)
> :|  # func                 -62+   2.452  _xnpod_schedule_handler+0xe (___ipipe_sync_stage+0x186)
> :|  # func                 -59+   2.130  _xnpod_schedule+0xa (_xnpod_schedule_handler+0x5c)
> :|  # [    0] swapper -1   -57+   7.180  _xnpod_schedule+0x104 (__start+0xfffff000)

This must be
http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2337

saying "root thread is about the be scheduled away". But then I'm
missing code here that one would normally see on x86:

http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2442

Likely inlined on bfin, but this *must* make some noise:

http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2453

Instead we enter the pending timer interrupt. This currently makes no
sense to me, rather make me think that something is skewed. Maybe only
the tracer, maybe more...

> :|  # end     0x80000000   -50+   2.278  ___sched_text_start+0x5ca (__start+0xfffff000)
> :|  # func                 -48    0.922  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
> :|  # begin   0x00000006   -47+   1.820  ___ipipe_grab_irq+0x34 (__start+0xfffff000)
> :|  # func                 -45+   1.530  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
> :|  # func                 -43+   1.168  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
> :|  # func                 -42+   1.102  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
> :|  # end     0x00000006   -41+   2.010  ___ipipe_grab_irq+0x42 (__start+0xfffff000)
> :   # func                 -39+   1.090  _ipipe_unstall_pipeline_head+0x8 (_xnshadow_harden+0xce)
> :|  # begin   0x80000001   -38+   2.308  _ipipe_unstall_pipeline_head+0x18 (__start+0xfffff000)

That's another weirdness, but now very likely of the tracer: both
tracepoints should contain the same parent entry, but the second one is
just nonsense. And there are such kind of parents all over the place -
hmm, actually any explicit trace point shows this problem. Is there no
way on bfin to get the caller from everywhere in a function? I guess
there is, likely a register, right?

Jan


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 250 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* [Xenomai-core] Re: [Adeos-main] ipip tracer porting for blackfin
  2006-11-10 16:11     ` Jan Kiszka
@ 2006-11-13  3:51       ` Li Yi (Adam)
  2006-11-13  9:56         ` Li Yi (Adam)
  2006-11-14 20:02         ` [Xenomai-core] Re: [Adeos-main] " Jan Kiszka
  0 siblings, 2 replies; 9+ messages in thread
From: Li Yi (Adam) @ 2006-11-13  3:51 UTC (permalink / raw)
  To: Jan Kiszka; +Cc: adeos-main, xenomai

[-- Attachment #1: Type: text/plain, Size: 2397 bytes --]

Please see my comments bellow:

On 11/11/06, Jan Kiszka <jan.kiszka@domain.hid> wrote:
>
> Li Yi (Adam) wrote:
> > Hi Jan and Philippe,
>
>
> This must be
> http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2337
>
> saying "root thread is about the be scheduled away". But then I'm
> missing code here that one would normally see on x86:
>
> http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2442
>
> Likely inlined on bfin, but this *must* make some noise:
>
> http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2453
>
> Instead we enter the pending timer interrupt. This currently makes no
> sense to me, rather make me think that something is skewed. Maybe only
> the tracer, maybe more...


Do you have a tracer-log for i386 for reference? So I can do some compare.

> :|  # end     0x80000000   -50+   2.278  ___sched_text_start+0x5ca
> (__start+0xfffff000)
> > :|  # func                 -48    0.922  ___ipipe_grab_irq+0x22
> (__common_int_entry+0x72)
> > :|  # begin   0x00000006   -47+   1.820  ___ipipe_grab_irq+0x34
> (__start+0xfffff000)
> > :|  # func                 -45+   1.530  ___ipipe_handle_irq+0x1e
> (___ipipe_grab_irq+0x3c)
> > :|  # func                 -43+   1.168  _rthal_timer_ack+0x8
> (___ipipe_handle_irq+0x178)
> > :|  # func                 -42+   1.102  ___ipipe_dispatch_wired+0x14
> (___ipipe_handle_irq+0x190)
> > :|  # end     0x00000006   -41+   2.010  ___ipipe_grab_irq+0x42
> (__start+0xfffff000)
> > :   # func                 -39+   1.090  _ipipe_unstall_pipeline_head+0x8
> (_xnshadow_harden+0xce)
> > :|  # begin   0x80000001   -38+   2.308  _ipipe_unstall_pipeline_head+0x18
> (__start+0xfffff000)
>
> That's another weirdness, but now very likely of the tracer: both
> tracepoints should contain the same parent entry, but the second one is
> just nonsense. And there are such kind of parents all over the place -
> hmm, actually any explicit trace point shows this problem. Is there no
> way on bfin to get the caller from everywhere in a function? I guess
> there is, likely a register, right?


Yes, by using the frame-pointer register (FP) it is possible to get parent's
return address. The reason is that __builtin_return_address(1) is not
implemented for Blackfin gcc. I have submitted feature request for the
toolchain developer, hoping they could help.
Thanks for pointing out.

-Yi

[-- Attachment #2: Type: text/html, Size: 3973 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [Adeos-main] ipip tracer porting for blackfin
  2006-11-13  3:51       ` Li Yi (Adam)
@ 2006-11-13  9:56         ` Li Yi (Adam)
  2006-11-13 10:01           ` [Xenomai-core] Fwd: " Li Yi (Adam)
  2006-11-14 20:02         ` [Xenomai-core] Re: [Adeos-main] " Jan Kiszka
  1 sibling, 1 reply; 9+ messages in thread
From: Li Yi (Adam) @ 2006-11-13  9:56 UTC (permalink / raw)
  To: Jan Kiszka; +Cc: adeos-main, Philippe Gerum, xenomai


[-- Attachment #1.1: Type: text/plain, Size: 12426 bytes --]

I tried to trace the maximum interrupt latency of the "latency" test case
furthur, and attached here two cases of the extra delay of hanlding Core
Timer interrupt.  Do you have any suggestions on how to make improvements?

Before doing that, I did a little hack to workaround the
__builtin_return_address(1) issue (just for test purpose) before the
bfin-gcc is ready to handle it correctly.
I also recored IMASK, ILAT, IPEND register in each fuction trace point.

--- linux-2.6.x-ipipe-orig/include/linux/ipipe.h        2006-11-10 21:04:
21.000000000 +0800
+++ linux-2.6.x-ipipe-1110/include/linux/ipipe.h        2006-11-13 13:20:
12.000000000 +0800
@@ -35,9 +35,29 @@
                                 (IPIPE_MINOR_NUMBER <<  8) | \
                                 (IPIPE_PATCH_NUMBER))

+#ifdef CONFIG_BFIN
+static inline unsigned long my_return_address1(void)
+{
+       unsigned long p_rets;
+       __asm__ __volatile__ ( " \
+               [--sp] = p0; \
+               p0 = [fp]; \
+               %0 = [p0 + 4]; \
+               p0 = [sp++]; \
+               ":"=d" (p_rets):);
+       return p_rets;
+}
+#endif
+
+
+
 #ifndef BROKEN_BUILTIN_RETURN_ADDRESS
 #define __BUILTIN_RETURN_ADDRESS0 ((unsigned
long)__builtin_return_address(0))
-#define __BUILTIN_RETURN_ADDRESS1 ((unsigned
long)__builtin_return_address(1))
+#ifdef CONFIG_BFIN
+       #define __BUILTIN_RETURN_ADDRESS1 (my_return_address1())
+#else
+       #define __BUILTIN_RETURN_ADDRESS1 ((unsigned
long)__builtin_return_address(1))
+#endif


ipipe-tarce-2.log
<snip>
========================================================================================
// During this period, the Core Timer interrupt and IRQ12 are latched, but
since IMASK=0x1F, they cannot be handled.

:|  *+func   (IMASK) 0x0000001f,  (ILAT) 0x00001040, (IPEND) 0x00008000
 -100+   2.592  _xnpod_schedule+0xa (_gatekeeper_thread+0x1ce)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -97+   3.000  _ipipe_trigger_irq+0x12 (_xnpod_schedule+0x2e6)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -94+   1.710  ___ipipe_handle_irq+0x1e (_ipipe_trigger_irq+0x86)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -92+   2.460  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -90+   1.860  ___ipipe_restore_pipeline_head+0xa
(_gatekeeper_thread+0x138)
:|   +func    0x0000001f, 0x00001040, 0x00008000
  -88+   1.990  ___ipipe_walk_pipeline+0x18
(___ipipe_restore_pipeline_head+0xd0)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -86+   2.300  _ipipe_suspend_domain+0xa (___ipipe_walk_pipeline+0xac)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -84+   5.018  ___ipipe_sync_stage+0xe (_ipipe_suspend_domain+0x4e)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -79+   1.942  _xnpod_schedule_handler+0xe (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -77+   2.310  _xnpod_schedule+0xa (_xnpod_schedule_handler+0x5c)
:|  # [    0] swapper -1   -74+   6.560  _xnpod_schedule+0x104
(_xnpod_schedule_handler+0x5c)
:|  # end     0x80000000   -68+   3.250  ___sched_text_start+0x5ca
(_xnshadow_harden+0xa8)

==========================================================================================
// The Core Timer Interrupt (0x6) is handled - but the interrupt handler
registered in Xenomai  is not called

:|  # func    0x0000001f, 0x00001000, 0x00008040
  -65    0.940  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x00000006   -64+   2.300  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -61+   1.650  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -60+   1.508  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -58+   1.382  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # end     0x00000006   -57+   3.060  ___ipipe_grab_irq+0x42
(__common_int_entry+0x72)

// The handling of Core Timer interrupt returned in __ipipe_dispatch_wired()
returns and it begins to handle IRQ12 - Is this expected behavior?

============================================================================================
// IRQ12 interrupts the handling of higher priority Core Timer IRQ (IRQ6).

:|  # func    0x0000001f, 0x00000000, 0x00009000
  -54+   2.670  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x0000001a   -51+   1.940  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -49+   3.540  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -46+   2.310  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -43+   1.240  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|  # end     0x0000001a   -42+   2.118  ___ipipe_grab_irq+0x42
(__common_int_entry+0x72)
:   # func    0x0000ffff, 0x00000000, 0x00008000
  -40+   1.242  _ipipe_unstall_pipeline_head+0x8 (_xnshadow_harden+0xce)
:|  # begin   0x80000001   -39+   2.870  _ipipe_unstall_pipeline_head+0x18
(_xnshadow_harden+0xce)
:|  + func    0x0000001f, 0x00000000, 0x00008000
  -36+   2.120  ___ipipe_sync_stage+0xe (_ipipe_unstall_pipeline_head+0x74)

=======================================================================================================
// The interrupt handler for Core Timer IRQ in Xenomai is called here.

:|  # func    0x0000001f, 0x00000000, 0x00008000
  -34+   1.360  _xnintr_clock_handler+0x10 (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -32+   1.710  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -31+   1.870  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)

<snip>

ipipe-trace-1.log
<snip>
:    +func    0x0000ffff, 0x00000800, 0x00008000
 -114+   5.020  _rw_verify_area+0xa (_vfs_write+0x136)

==========================================================================

// In this case, Core Timer IRQ (IRQ6) happens just after Adeos enters
__ipipe_grab_irq() to handle IRQ11. Since the IMASK=0x1F, the handling of
IRQ6 has to be delayed until lower priority interrupt IRQ11 is handled.

:|   +func    0x0000001f, 0x00000040, 0x00008800
 -109+   2.690  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000018  -107+   2.850  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -104+   3.960  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -100+   1.760  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -98+   3.298  _ipipe_test_and_stall_pipeline_from+0xa
(___ipipe_ack_irq+0x20)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -95+   2.302  _bf533_internal_mask_irq+0xc (___ipipe_ack_irq+0x40)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -92+   3.290  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -89+   6.110  ___ipipe_sync_stage+0xe (___ipipe_walk_pipeline+0xde)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -83+   1.690  _kick_irqd+0x1c (___ipipe_sync_stage+0x290)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -81+   1.920  ___ipipe_test_and_stall_root+0xe (_kick_irqd+0x22)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -79+   1.710  ___ipipe_restore_root+0xa (_kick_irqd+0x32)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -78+   1.930  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -76+   1.848  ___up+0x8 (_kick_irqd+0x62)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -74+   1.732  ___wake_up+0xa (___up+0x16)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -72+   1.880  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -70+   2.280  ___wake_up_common+0xc (___wake_up+0x3a)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -68+   1.780  _default_wake_function+0xa (___wake_up_common+0x38)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -66+   1.690  _try_to_wake_up+0xa (_default_wake_function+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -65+   1.810  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -63+   1.660  _sched_clock+0xc (_try_to_wake_up+0x7e)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -61+   2.548  ___muldi3+0xc (_sched_clock+0x26)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -59+   2.622  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -56+   1.440  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -54+   1.580  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -53+   1.320  ___ipipe_restore_root+0xa (___wake_up+0x40)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -52+   1.700  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   +end     0x00000018   -50+   2.980  ___ipipe_grab_irq+0x42
(__common_int_entry+0x72)
==============================================================================================

// Begin to handle IRQ6.

:|   +func    0x0000001f, 0x00000000, 0x00008040
  -47+   1.060  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000006   -46+   1.928  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -44+   2.532  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -41+   1.640  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
<snip>

Thanks,

-Yi


On 11/13/06, Li Yi (Adam) <liyiadam@domain.hid> wrote:
>
> Please see my comments bellow:
>
> On 11/11/06, Jan Kiszka <jan.kiszka@domain.hid> wrote:
> >
> > Li Yi (Adam) wrote:
> > > Hi Jan and Philippe,
> >
> >
> > This must be
> > http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2337
> >
> >
> > saying "root thread is about the be scheduled away". But then I'm
> > missing code here that one would normally see on x86:
> >
> >
> > http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2442
> >
> > Likely inlined on bfin, but this *must* make some noise:
> >
> >
> > http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2453
> >
> > Instead we enter the pending timer interrupt. This currently makes no
> > sense to me, rather make me think that something is skewed. Maybe only
> > the tracer, maybe more...
>
>
> Do you have a tracer-log for i386 for reference? So I can do some compare.
>
> > :|  # end     0x80000000   -50+   2.278  ___sched_text_start+0x5ca
> > (__start+0xfffff000)
> > > :|  # func                 -48    0.922  ___ipipe_grab_irq+0x22
> > (__common_int_entry+0x72)
> > > :|  # begin   0x00000006   -47+   1.820  ___ipipe_grab_irq+0x34
> > (__start+0xfffff000)
> > > :|  # func                 -45+   1.530  ___ipipe_handle_irq+0x1e
> > (___ipipe_grab_irq+0x3c)
> > > :|  # func                 -43+   1.168  _rthal_timer_ack+0x8
> > (___ipipe_handle_irq+0x178)
> > > :|  # func                 -42+   1.102  ___ipipe_dispatch_wired+0x14
> > (___ipipe_handle_irq+0x190)
> > > :|  # end     0x00000006   -41+   2.010  ___ipipe_grab_irq+0x42
> > (__start+0xfffff000)
> > > :   # func                 -39+   1.090  _ipipe_unstall_pipeline_head+0x8
> > (_xnshadow_harden+0xce)
> > > :|  # begin   0x80000001   -38+   2.308  _ipipe_unstall_pipeline_head+0x18
> > (__start+0xfffff000)
> >
> > That's another weirdness, but now very likely of the tracer: both
> > tracepoints should contain the same parent entry, but the second one is
> > just nonsense. And there are such kind of parents all over the place -
> > hmm, actually any explicit trace point shows this problem. Is there no
> > way on bfin to get the caller from everywhere in a function? I guess
> > there is, likely a register, right?
>
>
> Yes, by using the frame-pointer register (FP) it is possible to get
> parent's return address. The reason is that __builtin_return_address(1) is
> not implemented for Blackfin gcc. I have submitted feature request for the
> toolchain developer, hoping they could help.
> Thanks for pointing out.
>
> -Yi
>

[-- Attachment #1.2: Type: text/html, Size: 18815 bytes --]

[-- Attachment #2: ipipe-trace-1.log --]
[-- Type: text/x-log, Size: 22548 bytes --]

root:~> ./latency -t2 -p500 -f
root:~> cat /proc/ipipe/trace/frozen
I-pipe frozen back-tracing service on 2.6.16.27-ADI-2006R2/ipipe-1.5-00
------------------------------------------------------------
Freeze: 1541259864240 cycles, Trace Points: 200 (+10)

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
:|   +begin   0x80000001  -374+   2.140  _do_softint+0x22 (_run_workqueue+0xb4)
:|   +end     0x80000000  -372+   1.270  _do_softint+0x3c (_run_workqueue+0xb4)
:|   +begin   0x80000001  -370+   1.330  _do_softint+0x5c (_run_workqueue+0xb4)
:|   +end     0x80000000  -369+   1.150  _do_softint+0x76 (_run_workqueue+0xb4)
:|   +begin   0x80000001  -368+   1.800  _do_softint+0x96 (_run_workqueue+0xb4)
:|   +end     0x80000000  -366+   2.100  _do_softint+0xb0 (_run_workqueue+0xb4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -364+   2.938  _dma_transmit_chars+0xc (_do_softint+0xc6)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -361+   1.882  _local_put_char+0x12 (_dma_transmit_chars+0x66)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -359+   1.270  ___ipipe_test_and_stall_root+0xe (_local_put_char+0x18)
:|   +begin   0x80000001  -358+   1.530  ___ipipe_test_and_stall_root+0x22 (_local_put_char+0x18)
:|   #end     0x80000000  -356+   2.360  ___ipipe_test_and_stall_root+0x36 (_local_put_char+0x18)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -354+   1.300  ___ipipe_restore_root+0xa (_local_put_char+0x46)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -353+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -352+   1.550  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -350+   2.088  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -348+   1.792  _schedule_work+0x10 (_dma_transmit_chars+0xb2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -346+   1.450  _queue_work+0xa (_schedule_work+0x1c)
:|   +begin   0x80000001  -345+   1.730  _queue_work+0x1e (_schedule_work+0x1c)
:|   +end     0x80000000  -343+   1.820  _queue_work+0x32 (_schedule_work+0x1c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -341+   1.660  ___queue_work+0xa (_queue_work+0x72)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -339+   1.020  ___ipipe_test_and_stall_root+0xe (___queue_work+0x12)
:|   +begin   0x80000001  -338+   1.530  ___ipipe_test_and_stall_root+0x22 (___queue_work+0x12)
:|   #end     0x80000000  -337+   2.448  ___ipipe_test_and_stall_root+0x36 (___queue_work+0x12)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -334+   1.612  ___wake_up+0xa (___queue_work+0x50)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -333+   1.290  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #begin   0x80000001  -332+   1.550  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -330+   1.900  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -328+   1.360  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -327+   1.370  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -325+   1.080  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -324+   1.558  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -323+   1.482  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -321+   1.740  ___ipipe_restore_root+0xa (___queue_work+0x56)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -319+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -318+   1.480  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -317+   1.640  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -315+   1.020  ___ipipe_test_and_stall_root+0xe (_run_workqueue+0xb8)
:|   +begin   0x80000001  -314+   1.650  ___ipipe_test_and_stall_root+0x22 (_run_workqueue+0xb8)
:|   #end     0x80000000  -313+   1.658  ___ipipe_test_and_stall_root+0x36 (_run_workqueue+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -311+   1.412  ___wake_up+0xa (_run_workqueue+0x4c)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -310+   1.290  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #begin   0x80000001  -308+   1.550  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -307+   1.700  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -305+   1.360  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -304+   1.370  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -302+   1.080  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -301+   1.558  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -300+   1.702  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -298+   1.670  ___ipipe_restore_root+0xa (_run_workqueue+0x7a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -296+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -295+   1.480  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -294+   1.070  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -293+   1.490  _run_workqueue+0x90 (_worker_thread+0x12a)
:|   +end     0x80000000  -291+   1.600  _run_workqueue+0xa2 (_worker_thread+0x12a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -290+   1.078  _do_softint+0xc (_run_workqueue+0xb4)
:|   +begin   0x80000001  -288+   1.522  _do_softint+0x22 (_run_workqueue+0xb4)
:|   +end     0x80000000  -287+   1.910  _do_softint+0x3c (_run_workqueue+0xb4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -285+   1.400  ___wake_up+0xa (_do_softint+0xf8)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -284    0.970  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   +begin   0x80000001  -283+   1.430  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -281+   1.590  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -280+   1.450  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -278+   1.398  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -277+   1.072  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -276+   1.900  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -274+   1.340  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -272+   1.330  _do_softint+0x5c (_run_workqueue+0xb4)
:|   +end     0x80000000  -271+   1.010  _do_softint+0x76 (_run_workqueue+0xb4)
:|   +begin   0x80000001  -270+   1.490  _do_softint+0x96 (_run_workqueue+0xb4)
:|   +end     0x80000000  -269+   1.480  _do_softint+0xb0 (_run_workqueue+0xb4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -267+   1.108  ___ipipe_test_and_stall_root+0xe (_run_workqueue+0xb8)
:|   +begin   0x80000001  -266+   1.522  ___ipipe_test_and_stall_root+0x22 (_run_workqueue+0xb8)
:|   #end     0x80000000  -265+   1.860  ___ipipe_test_and_stall_root+0x36 (_run_workqueue+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -263+   1.420  ___wake_up+0xa (_run_workqueue+0x4c)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -261+   1.010  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #begin   0x80000001  -260+   1.580  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -259+   1.600  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -257+   1.450  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -256+   1.418  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -254+   1.282  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -253+   1.590  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -251+   1.920  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -249+   1.300  ___ipipe_restore_root+0xa (_run_workqueue+0xe6)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -248+   1.040  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -247+   1.500  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -246+   1.780  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -244+   1.808  _worker_thread+0xd4 (_kthread+0xf0)
:|   +end     0x80000000  -242+   1.612  _worker_thread+0xe8 (_kthread+0xf0)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -240+   1.820  _kthread_should_stop+0x10 (_worker_thread+0xf8)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -239+   1.410  _add_wait_queue+0xe (_worker_thread+0x10a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -237    0.960  ___ipipe_test_and_stall_root+0xe (_add_wait_queue+0x18)
:|   +begin   0x80000001  -236+   1.490  ___ipipe_test_and_stall_root+0x22 (_add_wait_queue+0x18)
:|   #end     0x80000000  -235+   1.590  ___ipipe_test_and_stall_root+0x36 (_add_wait_queue+0x18)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -233+   1.390  ___ipipe_restore_root+0xa (_add_wait_queue+0x3e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -232+   1.088  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -231+   1.752  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -229+   1.980  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -227+   1.650  ___sched_text_start+0xe (_worker_thread+0x116)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -225+   1.360  _profile_hit+0xe (___sched_text_start+0x4c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -224+   1.600  _sched_clock+0xc (___sched_text_start+0x7c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -222+   2.730  ___muldi3+0xc (_sched_clock+0x26)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -220+   1.060  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -218+   1.538  ___ipipe_stall_root+0x22 (___sched_text_start+0xec)
:|   #end     0x80000000  -217+   2.002  ___ipipe_stall_root+0x30 (___sched_text_start+0xec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -215+   1.710  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -213+   2.720  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -211+   1.460  ___sched_text_start+0x1e6 (_worker_thread+0x116)
:|   #end     0x80000000  -209+   2.480  ___sched_text_start+0x1f4 (_worker_thread+0x116)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -207+   1.050  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -206+   1.790  ___ipipe_dispatch_event+0x20 (___sched_text_start+0x2de)
:|   #end     0x80000000  -204+   1.568  ___ipipe_dispatch_event+0xcc (___sched_text_start+0x2de)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -202+   1.562  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -201+   2.320  _schedule_event+0x7e (___ipipe_dispatch_event+0xe2)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -198+   1.270  _xnpod_schedule_runnable+0xc (_schedule_event+0xa8)
:|  *#[    0] swapper  0  -197+   2.260  _xnpod_schedule_runnable+0x28 (_schedule_event+0xa8)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -195+   1.740  ___ipipe_restore_pipeline_head+0xa (_schedule_event+0x100)
:|   #end     0x80000000  -193+   1.180  ___ipipe_restore_pipeline_head+0x94 (_schedule_event+0x100)
:|   #begin   0x80000001  -192+   1.640  ___ipipe_dispatch_event+0xfa (___sched_text_start+0x2de)
:|   #end     0x80000000  -190+   1.288  ___ipipe_dispatch_event+0x17a (___sched_text_start+0x2de)
:    #begin   0x80000000  -189+   4.112  ___sched_text_start+0x2f0 (_worker_thread+0x116)
:|   #end     0x80000000  -185+   1.950  ___sched_text_start+0x5ca (_system_call+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -183+   1.080  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -182+   1.570  ___ipipe_unstall_root+0x1a (___sched_text_start+0x3ce)
:|   +end     0x80000000  -180+   1.440  ___ipipe_unstall_root+0x58 (___sched_text_start+0x3ce)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -179+   1.010  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x3d8)
:|   +begin   0x80000001  -178+   1.530  ___ipipe_test_and_stall_root+0x22 (___sched_text_start+0x3d8)
:|   #end     0x80000000  -176+   1.728  ___ipipe_test_and_stall_root+0x36 (___sched_text_start+0x3d8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -174+   1.812  ___ipipe_restore_root+0xa (___sched_text_start+0x4ec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -173+   1.120  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -172+   1.550  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -170!  16.730  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -153+   1.690  ___ipipe_syscall_root+0x14 (_system_call+0x2e)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -152+   1.030  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x40)
:|   +begin   0x80000001  -151+   2.100  ___ipipe_dispatch_event+0x20 (___ipipe_syscall_root+0x40)
:|  + end     0x80000000  -148+   2.198  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -146+   2.522  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xe2)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -144+   1.580  _xnpod_schedule+0xa (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -142+   1.530  _xnpod_schedule+0x64 (_hisyscall_event+0xbe)
:|  + end     0x80000000  -141    0.990  _xnpod_schedule+0x72 (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -140+   1.510  _xnpod_schedule+0xc4 (_hisyscall_event+0xbe)
:|  # [    0] swapper  0  -138+   1.710  _xnpod_schedule+0x104 (_hisyscall_event+0xbe)
:|  # func    0x0000001f, 0x00000000, 0x00008000
 -136+   1.160  ___ipipe_restore_pipeline_head+0xa (_xnpod_schedule+0x2b0)
:|  + end     0x80000000  -135+   1.948  ___ipipe_restore_pipeline_head+0x94 (_xnpod_schedule+0x2b0)
:|  + begin   0x80000001  -133+   1.932  ___ipipe_dispatch_event+0xfa (___ipipe_syscall_root+0x40)
:|   +end     0x80000000  -131+   2.030  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -129+   1.270  _losyscall_event+0x14 (___ipipe_dispatch_event+0xe2)
:|   +begin   0x80000001  -128+   1.600  ___ipipe_dispatch_event+0xfa (___ipipe_syscall_root+0x40)
:|   +end     0x80000000  -126+   2.510  ___ipipe_dispatch_event+0x17a (___ipipe_syscall_root+0x40)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -124+   1.620  _sys_write+0xa (_system_call+0x86)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -122+   1.560  _fget_light+0x16 (_sys_write+0x16)
:|   +begin   0x80000001  -121+   1.628  _fget_light+0x8c (_sys_write+0x16)
:|   +end     0x80000000  -119+   2.212  _fget_light+0xa6 (_sys_write+0x16)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -117+   2.630  _vfs_write+0xc (_sys_write+0x34)
:    +func    0x0000ffff, 0x00000800, 0x00008000
 -114+   5.020  _rw_verify_area+0xa (_vfs_write+0x136)


==========================================================================

:|   +func    0x0000001f, 0x00000040, 0x00008800
 -109+   2.690  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000018  -107+   2.850  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -104+   3.960  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -100+   1.760  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -98+   3.298  _ipipe_test_and_stall_pipeline_from+0xa (___ipipe_ack_irq+0x20)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -95+   2.302  _bf533_internal_mask_irq+0xc (___ipipe_ack_irq+0x40)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -92+   3.290  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -89+   6.110  ___ipipe_sync_stage+0xe (___ipipe_walk_pipeline+0xde)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -83+   1.690  _kick_irqd+0x1c (___ipipe_sync_stage+0x290)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -81+   1.920  ___ipipe_test_and_stall_root+0xe (_kick_irqd+0x22)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -79+   1.710  ___ipipe_restore_root+0xa (_kick_irqd+0x32)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -78+   1.930  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -76+   1.848  ___up+0x8 (_kick_irqd+0x62)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -74+   1.732  ___wake_up+0xa (___up+0x16)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -72+   1.880  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -70+   2.280  ___wake_up_common+0xc (___wake_up+0x3a)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -68+   1.780  _default_wake_function+0xa (___wake_up_common+0x38)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -66+   1.690  _try_to_wake_up+0xa (_default_wake_function+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -65+   1.810  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -63+   1.660  _sched_clock+0xc (_try_to_wake_up+0x7e)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -61+   2.548  ___muldi3+0xc (_sched_clock+0x26)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -59+   2.622  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -56+   1.440  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -54+   1.580  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -53+   1.320  ___ipipe_restore_root+0xa (___wake_up+0x40)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -52+   1.700  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   +end     0x00000018   -50+   2.980  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)
==============================================================================================


:|   +func    0x0000001f, 0x00000000, 0x00008040
  -47+   1.060  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000006   -46+   1.928  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -44+   2.532  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -41+   1.640  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -40+   2.540  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -37+   2.020  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xb6)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -35+   1.910  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -33+   1.610  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -32+   1.800  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -30+   2.038  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -28+   2.562  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -25+   1.820  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -23+   2.110  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -21+   5.080  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -16+   7.120  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # func    0x0000001f, 0x00000000, 0x00008040
   -9+   2.850  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  # func    0x0000001f, 0x00000000, 0x00008040
   -6+   1.560  _ipipe_trace_frozen_reset+0xa (_eval_inner_loop+0xe0)
:|  # func    0x0000001f, 0x00000000, 0x00008040
   -5+   4.238  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
<|  # freeze  0x0001933e     1    3.132  _eval_inner_loop+0xe6 (_timer_proc+0x24)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    4    1.680  _xnpod_get_time+0xa (_timer_proc+0x2a)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    5    1.570  ___muldi3+0xc (_xnpod_get_time+0x52)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    7    1.520  ___muldi3+0xc (_xnpod_get_time+0x6e)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    8    3.390  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   12    5.670  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
 |  # begin   0x00059798    17    2.110  _timer_proc+0x44 (_xntimer_do_tick_aperiodic+0x1b2)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   20    2.558  _xntimer_start+0xa (_timer_proc+0x74)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   22    3.552  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   26    1.630  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   27    0.000  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)

[-- Attachment #3: ipipe-tarce-2.log --]
[-- Type: text/x-log, Size: 23712 bytes --]

root:~> ./latency -t2 -p200 -f
root:~> cat /proc/ipipe/trace/frozen
I-pipe frozen back-tracing service on 2.6.16.27-ADI-2006R2/ipipe-1.5-00
------------------------------------------------------------
Freeze: 90786953155 cycles, Trace Points: 200 (+10)

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -429+   3.310  ___muldi3+0xc (_sched_clock+0x26)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -426+   1.060  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -425+   1.330  ___ipipe_stall_root+0x22 (___sched_text_start+0xec)
:|   #end     0x80000000  -424+   1.710  ___ipipe_stall_root+0x30 (___sched_text_start+0xec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -422+   1.750  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -420+   3.040  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -417+   1.758  ___sched_text_start+0x1e6 (_worker_thread+0x116)
:|   #end     0x80000000  -416+   3.472  ___sched_text_start+0x1f4 (_worker_thread+0x116)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -412+   1.310  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -411+   1.770  ___ipipe_dispatch_event+0x20 (___sched_text_start+0x2de)
:|   #end     0x80000000  -409+   1.490  ___ipipe_dispatch_event+0xcc (___sched_text_start+0x2de)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -407+   1.570  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -406+   2.440  _schedule_event+0x7e (___ipipe_dispatch_event+0xe2)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -403+   2.020  _xnpod_schedule_runnable+0xc (_schedule_event+0xa8)
:|  *#[    0] swapper  0  -401+   2.228  _xnpod_schedule_runnable+0x28 (_schedule_event+0xa8)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -399+   1.892  ___ipipe_restore_pipeline_head+0xa (_schedule_event+0x100)
:|   #end     0x80000000  -397+   1.440  ___ipipe_restore_pipeline_head+0x94 (_schedule_event+0x100)
:|   #begin   0x80000001  -396+   1.640  ___ipipe_dispatch_event+0xfa (___sched_text_start+0x2de)
:|   #end     0x80000000  -394+   1.090  ___ipipe_dispatch_event+0x17a (___sched_text_start+0x2de)
:    #begin   0x80000000  -393+   4.220  ___sched_text_start+0x2f0 (_worker_thread+0x116)
:|   #end     0x80000000  -389+   1.470  ___sched_text_start+0x5ca (_system_call+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -387+   1.080  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -386+   1.558  ___ipipe_unstall_root+0x1a (___sched_text_start+0x3ce)
:|   +end     0x80000000  -385+   1.472  ___ipipe_unstall_root+0x58 (___sched_text_start+0x3ce)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -383+   1.400  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x3d8)
:|   +begin   0x80000001  -382+   1.530  ___ipipe_test_and_stall_root+0x22 (___sched_text_start+0x3d8)
:|   #end     0x80000000  -380+   1.470  ___ipipe_test_and_stall_root+0x36 (___sched_text_start+0x3d8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -379+   1.300  ___ipipe_restore_root+0xa (___sched_text_start+0x4ec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -378+   1.040  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -377+   1.670  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -375!  19.558  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -355+   1.722  ___ipipe_syscall_root+0x14 (_system_call+0x2e)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -354+   1.290  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x40)
:|   +begin   0x80000001  -352+   2.100  ___ipipe_dispatch_event+0x20 (___ipipe_syscall_root+0x40)
:|  + end     0x80000000  -350+   1.950  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -348+   1.860  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xe2)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -346+   1.900  _xnpod_schedule+0xa (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -345+   1.650  _xnpod_schedule+0x64 (_hisyscall_event+0xbe)
:|  + end     0x80000000  -343+   1.158  _xnpod_schedule+0x72 (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -342+   1.742  _xnpod_schedule+0xc4 (_hisyscall_event+0xbe)
:|  # [    0] swapper  0  -340+   1.840  _xnpod_schedule+0x104 (_hisyscall_event+0xbe)
:|  # func    0x0000001f, 0x00000000, 0x00008000
 -338+   1.310  ___ipipe_restore_pipeline_head+0xa (_xnpod_schedule+0x2b0)
:|  + end     0x80000000  -337+   3.420  ___ipipe_restore_pipeline_head+0x94 (_xnpod_schedule+0x2b0)
:|  + begin   0x80000001  -333+   1.560  ___ipipe_dispatch_event+0xfa (___ipipe_syscall_root+0x40)
:|   +end     0x80000000  -332+   1.950  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -330+   2.610  _losyscall_event+0x14 (___ipipe_dispatch_event+0xe2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -327+   1.888  _sys_rtdm_ioctl+0xa (_losyscall_event+0x94)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -325+   2.122  __rtdm_ioctl+0xa (_sys_rtdm_ioctl+0x1c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -323+   1.420  _rtdm_context_get+0x10 (__rtdm_ioctl+0x20)
:|   +begin   0x80000001  -322+   2.760  _rtdm_context_get+0x24 (__rtdm_ioctl+0x20)
:|  *+func    0x0000001f, 0x00000000, 0x00008000
 -319+   1.920  ___ipipe_test_and_stall_root+0xe (_rtdm_context_get+0x58)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -317+   1.410  ___ipipe_restore_root+0xa (_rtdm_context_get+0x6e)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -316+   1.130  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:|  *+end     0x80000000  -315+   1.680  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:   *+func    0x0000ffff, 0x00000000, 0x00008000
 -313+   1.088  ___ipipe_restore_pipeline_head+0xa (_rtdm_context_get+0x82)
:|  *+begin   0x80000001  -312+   1.542  ___ipipe_restore_pipeline_head+0x1e (_rtdm_context_get+0x82)
:|   +end     0x80000000  -310+   3.350  ___ipipe_restore_pipeline_head+0x94 (_rtdm_context_get+0x82)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -307+   1.960  _rt_tmbench_ioctl_nrt+0x16 (__rtdm_ioctl+0x4a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -305    0.970  ___ipipe_test_and_stall_root+0xe (__rtdm_ioctl+0x50)
:|   +begin   0x80000001  -304+   1.430  ___ipipe_test_and_stall_root+0x22 (__rtdm_ioctl+0x50)
:|   #end     0x80000000  -303+   3.000  ___ipipe_test_and_stall_root+0x36 (__rtdm_ioctl+0x50)

===================================================================================================
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -300+   1.340  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   #begin   0x00000006  -298+   2.488  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -296+   1.642  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -294+   1.830  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -292+   2.340  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -290+   1.570  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xb6)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -288+   1.540  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -287+   1.460  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -285+   2.070  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -283+   1.858  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -282+   2.062  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -279+   2.010  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -277+   2.130  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -275+   4.070  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -271+   6.020  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -265+   1.940  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -263+   1.640  _xnpod_get_time+0xa (_timer_proc+0x2a)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -262+   1.588  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -260+   1.662  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -258+   3.110  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -255+   5.370  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  #*begin   0x000227ae  -250+   1.500  _timer_proc+0x44 (_xntimer_do_tick_aperiodic+0x1b2)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -248+   1.590  _xntimer_start+0xa (_timer_proc+0x74)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -247+   2.120  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -245+   1.660  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -243+   1.908  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -241+   4.212  ___div64_32+0x10 (_xntimer_do_start_aperiodic+0x5ae)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -237+   1.830  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1bc)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -235+   6.750  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1d0)
:|  #*func    0x0000001f, 0x00000000, 0x00008000
 -228+   2.330  _xnpod_schedule_deferred+0xc (___ipipe_call_irqtail+0x1e)
:    #func    0x0000ffff, 0x00000000, 0x0000c000
 -226+   1.460  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x126)
:    #end     0x00000006  -225+   1.950  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)
===============================================================================================

:    #func    0x0000ffff, 0x00000000, 0x00008000
 -223+   1.390  ___ipipe_restore_root+0xa (__rtdm_ioctl+0x64)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -221+   1.138  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -220+   1.542  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -219+   2.710  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -216+   1.970  _xnshadow_harden+0x12 (_losyscall_event+0xb2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -214    0.970  ___ipipe_test_and_stall_root+0xe (_xnshadow_harden+0x3e)
:|   +begin   0x80000001  -213+   1.570  ___ipipe_test_and_stall_root+0x22 (_xnshadow_harden+0x3e)
:|   #end     0x80000000  -211+   1.730  ___ipipe_test_and_stall_root+0x36 (_xnshadow_harden+0x3e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -210+   1.390  ___ipipe_restore_root+0xa (_xnshadow_harden+0x56)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -208+   1.088  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -207+   1.542  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -206+   1.600  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -204+   1.730  _xnshadow_harden+0x76 (_losyscall_event+0xb2)
:|   +end     0x80000000  -202+   1.890  _xnshadow_harden+0x8c (_losyscall_event+0xb2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -200+   1.490  ___wake_up_sync+0xc (_xnshadow_harden+0xa4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -199+   1.040  ___ipipe_test_and_stall_root+0xe (___wake_up_sync+0x18)
:|   +begin   0x80000001  -198+   1.530  ___ipipe_test_and_stall_root+0x22 (___wake_up_sync+0x18)
:|   #end     0x80000000  -196+   2.038  ___ipipe_test_and_stall_root+0x36 (___wake_up_sync+0x18)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -194+   2.082  ___wake_up_common+0xc (___wake_up_sync+0x38)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -192+   1.880  _default_wake_function+0xa (___wake_up_common+0x38)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -190+   1.540  _try_to_wake_up+0xa (_default_wake_function+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -189+   1.010  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #begin   0x80000001  -188+   1.460  ___ipipe_test_and_stall_root+0x22 (_try_to_wake_up+0x14)
:|   #end     0x80000000  -186+   1.840  ___ipipe_test_and_stall_root+0x36 (_try_to_wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -184+   1.660  _sched_clock+0xc (_try_to_wake_up+0x7e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -183+   2.618  ___muldi3+0xc (_sched_clock+0x26)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -180+   3.222  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -177+   1.620  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -175+   1.080  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -174+   1.620  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -173+   1.530  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -171+   1.350  ___ipipe_restore_root+0xa (___wake_up_sync+0x3e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -170+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -169+   1.558  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -167+   1.592  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -166+   2.010  ___sched_text_start+0xe (_xnshadow_harden+0xa8)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -163+   1.910  _profile_hit+0xe (___sched_text_start+0x4c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -162+   1.390  _sched_clock+0xc (___sched_text_start+0x7c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -160+   3.030  ___muldi3+0xc (_sched_clock+0x26)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -157+   1.020  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -156+   1.530  ___ipipe_stall_root+0x22 (___sched_text_start+0xec)
:|   #end     0x80000000  -155+   1.578  ___ipipe_stall_root+0x30 (___sched_text_start+0xec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -153+   1.662  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -151+   3.360  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -148+   1.750  ___sched_text_start+0x1e6 (_xnshadow_harden+0xa8)
:|   #end     0x80000000  -146+   4.400  ___sched_text_start+0x1f4 (_xnshadow_harden+0xa8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -142+   1.020  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -141+   1.730  ___ipipe_dispatch_event+0x20 (___sched_text_start+0x2de)
:|   #end     0x80000000  -139+   1.740  ___ipipe_dispatch_event+0xcc (___sched_text_start+0x2de)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -137+   1.598  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -136+   1.632  ___ipipe_dispatch_event+0xfa (___sched_text_start+0x2de)
:|   #end     0x80000000  -134+   1.370  ___ipipe_dispatch_event+0x17a (___sched_text_start+0x2de)
:    #begin   0x80000000  -133+   2.320  ___sched_text_start+0x2f0 (_xnshadow_harden+0xa8)
:|   #func    0x0000001f, 0x00000000, 0x00008000
 -130+   1.670  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x678)
:|   #func    0x0000001f, 0x00000000, 0x00008000
 -129+   1.440  ___ipipe_restore_root+0xa (___sched_text_start+0x68c)
:|   #func    0x0000001f, 0x00000000, 0x00008000
 -127+   3.910  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -123+   1.860  ___sched_text_start+0x5ca (_gatekeeper_thread+0xe8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -122+   1.048  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -121+   1.512  ___ipipe_unstall_root+0x1a (___sched_text_start+0x3ce)
:|   +end     0x80000000  -119+   2.340  ___ipipe_unstall_root+0x58 (___sched_text_start+0x3ce)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -117+   1.620  _kthread_should_stop+0x10 (_gatekeeper_thread+0xec)
:|   +begin   0x80000001  -115+   3.370  _gatekeeper_thread+0x100 (_kthread+0xf0)
:|  *+func    0x0000001f, 0x00000000, 0x00008000
 -112+   2.070  _xnpod_resume_thread+0xa (_gatekeeper_thread+0x186)
:|  *+[   51] -<?>-    0  -110+   4.990  _xnpod_resume_thread+0x4a (_gatekeeper_thread+0x186)
:|  *+func    0x0000001f, 0x00001000, 0x00008000
 -105+   1.370  _xnpod_schedule_runnable+0xc (_gatekeeper_thread+0x1be)
:|  *+[    0] swapper -1  -103+   3.738  _xnpod_schedule_runnable+0x28 (_gatekeeper_thread+0x1be)

========================================================================================

:|  *+func    0x0000001f, 0x00001040, 0x00008000
 -100+   2.592  _xnpod_schedule+0xa (_gatekeeper_thread+0x1ce)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -97+   3.000  _ipipe_trigger_irq+0x12 (_xnpod_schedule+0x2e6)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -94+   1.710  ___ipipe_handle_irq+0x1e (_ipipe_trigger_irq+0x86)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -92+   2.460  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -90+   1.860  ___ipipe_restore_pipeline_head+0xa (_gatekeeper_thread+0x138)
:|   +func    0x0000001f, 0x00001040, 0x00008000
  -88+   1.990  ___ipipe_walk_pipeline+0x18 (___ipipe_restore_pipeline_head+0xd0)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -86+   2.300  _ipipe_suspend_domain+0xa (___ipipe_walk_pipeline+0xac)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -84+   5.018  ___ipipe_sync_stage+0xe (_ipipe_suspend_domain+0x4e)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -79+   1.942  _xnpod_schedule_handler+0xe (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -77+   2.310  _xnpod_schedule+0xa (_xnpod_schedule_handler+0x5c)
:|  # [    0] swapper -1   -74+   6.560  _xnpod_schedule+0x104 (_xnpod_schedule_handler+0x5c)
:|  # end     0x80000000   -68+   3.250  ___sched_text_start+0x5ca (_xnshadow_harden+0xa8)

==========================================================================================
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -65    0.940  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x00000006   -64+   2.300  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -61+   1.650  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -60+   1.508  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -58+   1.382  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # end     0x00000006   -57+   3.060  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)

============================================================================================

:|  # func    0x0000001f, 0x00000000, 0x00009000
  -54+   2.670  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x0000001a   -51+   1.940  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -49+   3.540  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -46+   2.310  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -43+   1.240  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|  # end     0x0000001a   -42+   2.118  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)
:   # func    0x0000ffff, 0x00000000, 0x00008000
  -40+   1.242  _ipipe_unstall_pipeline_head+0x8 (_xnshadow_harden+0xce)
:|  # begin   0x80000001   -39+   2.870  _ipipe_unstall_pipeline_head+0x18 (_xnshadow_harden+0xce)
:|  + func    0x0000001f, 0x00000000, 0x00008000
  -36+   2.120  ___ipipe_sync_stage+0xe (_ipipe_unstall_pipeline_head+0x74)

=======================================================================================================

:|  # func    0x0000001f, 0x00000000, 0x00008000
  -34+   1.360  _xnintr_clock_handler+0x10 (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -32+   1.710  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -31+   1.870  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -29+   2.480  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -26+   1.698  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -25+   2.042  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -22+   1.990  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -20+   2.230  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -18+   4.150  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -14+   6.120  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # func    0x0000001f, 0x00000000, 0x00008000
   -8+   1.710  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  # func    0x0000001f, 0x00000000, 0x00008000
   -6+   1.560  _ipipe_trace_frozen_reset+0xa (_eval_inner_loop+0xe0)
:|  # func    0x0000001f, 0x00000000, 0x00008000
   -5+   4.218  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
<|  # freeze  0x00017ce6     1    2.182  _eval_inner_loop+0xe6 (_timer_proc+0x24)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    3    1.850  _xnpod_get_time+0xa (_timer_proc+0x2a)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    5    1.610  ___muldi3+0xc (_xnpod_get_time+0x52)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    6    1.530  ___muldi3+0xc (_xnpod_get_time+0x6e)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    8    2.740  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   10    5.830  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
 |  # begin   0x000129ec    16    2.110  _timer_proc+0x44 (_xntimer_do_tick_aperiodic+0x1b2)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   18    2.248  _xntimer_start+0xa (_timer_proc+0x74)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   21    2.702  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   23    1.800  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   25    0.000  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)

^ permalink raw reply	[flat|nested] 9+ messages in thread

* [Xenomai-core] Fwd: ipip tracer porting for blackfin
  2006-11-13  9:56         ` Li Yi (Adam)
@ 2006-11-13 10:01           ` Li Yi (Adam)
  0 siblings, 0 replies; 9+ messages in thread
From: Li Yi (Adam) @ 2006-11-13 10:01 UTC (permalink / raw)
  To: xenomai


[-- Attachment #1.1: Type: text/plain, Size: 10296 bytes --]

Sorry that due to mis-spell the "xenomai@xenomai.org" mail address, I will
resend this mail to xenomai-core mailing list - sorry about any duplication.

Regards,
-Yi

---------- Forwarded message ----------
From: Li Yi (Adam) <liyiadam@domain.hid>
Date: Nov 13, 2006 5:56 PM
Subject: Re: [Adeos-main] ipip tracer porting for blackfin
To: Jan Kiszka <jan.kiszka@domain.hid>
Cc: Philippe Gerum <rpm@xenomai.org>, adeos-main@gna.org,
xenomai@xenomai.org

I tried to trace the maximum interrupt latency of the "latency" test case
furthur, and attached here two cases of the extra delay of hanlding Core
Timer interrupt.  Do you have any suggestions on how to make improvements?

Before doing that, I did a little hack to workaround the
__builtin_return_address(1) issue (just for test purpose) before the
bfin-gcc is ready to handle it correctly.
I also recored IMASK, ILAT, IPEND register in each fuction trace point.

--- linux-2.6.x-ipipe-orig/include/linux/ipipe.h        2006-11-10 21:04:
21.000000000 +0800
+++ linux-2.6.x-ipipe-1110/include/linux/ipipe.h        2006-11-13 13:20:
12.000000000 +0800
@@ -35,9 +35,29 @@
                                 (IPIPE_MINOR_NUMBER <<  8) | \
                                 (IPIPE_PATCH_NUMBER))

+#ifdef CONFIG_BFIN
+static inline unsigned long my_return_address1(void)
+{
+       unsigned long p_rets;
+       __asm__ __volatile__ ( " \
+               [--sp] = p0; \
+               p0 = [fp]; \
+               %0 = [p0 + 4]; \
+               p0 = [sp++]; \
+               ":"=d" (p_rets):);
+       return p_rets;
+}
+#endif
+
+
+
 #ifndef BROKEN_BUILTIN_RETURN_ADDRESS
 #define __BUILTIN_RETURN_ADDRESS0 ((unsigned
long)__builtin_return_address(0))
-#define __BUILTIN_RETURN_ADDRESS1 ((unsigned
long)__builtin_return_address(1))
+#ifdef CONFIG_BFIN
+       #define __BUILTIN_RETURN_ADDRESS1 (my_return_address1())
+#else
+       #define __BUILTIN_RETURN_ADDRESS1 ((unsigned
long)__builtin_return_address(1))
+#endif


ipipe-tarce-2.log
<snip>
========================================================================================
// During this period, the Core Timer interrupt and IRQ12 are latched, but
since IMASK=0x1F, they cannot be handled.

:|  *+func   (IMASK) 0x0000001f,  (ILAT) 0x00001040, (IPEND) 0x00008000
 -100+   2.592  _xnpod_schedule+0xa (_gatekeeper_thread+0x1ce)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -97+   3.000  _ipipe_trigger_irq+0x12 (_xnpod_schedule+0x2e6)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -94+   1.710  ___ipipe_handle_irq+0x1e (_ipipe_trigger_irq+0x86)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -92+   2.460  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -90+   1.860  ___ipipe_restore_pipeline_head+0xa
(_gatekeeper_thread+0x138)
:|   +func    0x0000001f, 0x00001040, 0x00008000
  -88+   1.990  ___ipipe_walk_pipeline+0x18
(___ipipe_restore_pipeline_head+0xd0)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -86+   2.300  _ipipe_suspend_domain+0xa (___ipipe_walk_pipeline+0xac)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -84+   5.018  ___ipipe_sync_stage+0xe (_ipipe_suspend_domain+0x4e)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -79+   1.942  _xnpod_schedule_handler+0xe (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -77+   2.310  _xnpod_schedule+0xa (_xnpod_schedule_handler+0x5c)
:|  # [    0] swapper -1   -74+   6.560  _xnpod_schedule+0x104
(_xnpod_schedule_handler+0x5c)
:|  # end     0x80000000   -68+   3.250  ___sched_text_start+0x5ca
(_xnshadow_harden+0xa8)

==========================================================================================

// The Core Timer Interrupt (0x6) is handled - but the interrupt handler
registered in Xenomai  is not called

:|  # func    0x0000001f, 0x00001000, 0x00008040
  -65    0.940  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x00000006   -64+   2.300  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -61+   1.650  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -60+   1.508  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -58+   1.382  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # end     0x00000006   -57+   3.060  ___ipipe_grab_irq+0x42
(__common_int_entry+0x72)

// The handling of Core Timer interrupt returned in __ipipe_dispatch_wired()
returns and it begins to handle IRQ12 - Is this expected behavior?

============================================================================================

// IRQ12 interrupts the handling of higher priority Core Timer IRQ (IRQ6).

:|  # func    0x0000001f, 0x00000000, 0x00009000
  -54+   2.670  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x0000001a   -51+   1.940  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -49+   3.540  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -46+   2.310  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -43+   1.240  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|  # end     0x0000001a   -42+   2.118  ___ipipe_grab_irq+0x42
(__common_int_entry+0x72)
:   # func    0x0000ffff, 0x00000000, 0x00008000
  -40+   1.242  _ipipe_unstall_pipeline_head+0x8 (_xnshadow_harden+0xce)
:|  # begin   0x80000001   -39+   2.870  _ipipe_unstall_pipeline_head+0x18
(_xnshadow_harden+0xce)
:|  + func    0x0000001f, 0x00000000, 0x00008000
  -36+   2.120  ___ipipe_sync_stage+0xe (_ipipe_unstall_pipeline_head+0x74)

=======================================================================================================

// The interrupt handler for Core Timer IRQ in Xenomai is called here.

:|  # func    0x0000001f, 0x00000000, 0x00008000
  -34+   1.360  _xnintr_clock_handler+0x10 (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -32+   1.710  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -31+   1.870  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)

<snip>

ipipe-trace-1.log
<snip>
:    +func    0x0000ffff, 0x00000800, 0x00008000
 -114+   5.020  _rw_verify_area+0xa (_vfs_write+0x136)

==========================================================================

// In this case, Core Timer IRQ (IRQ6) happens just after Adeos enters
__ipipe_grab_irq() to handle IRQ11. Since the IMASK=0x1F, the handling of
IRQ6 has to be delayed until lower priority interrupt IRQ11 is handled.

:|   +func    0x0000001f, 0x00000040, 0x00008800
 -109+   2.690  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000018  -107+   2.850  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -104+   3.960  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -100+   1.760  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -98+   3.298  _ipipe_test_and_stall_pipeline_from+0xa
(___ipipe_ack_irq+0x20)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -95+   2.302  _bf533_internal_mask_irq+0xc (___ipipe_ack_irq+0x40)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -92+   3.290  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -89+   6.110  ___ipipe_sync_stage+0xe (___ipipe_walk_pipeline+0xde)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -83+   1.690  _kick_irqd+0x1c (___ipipe_sync_stage+0x290)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -81+   1.920  ___ipipe_test_and_stall_root+0xe (_kick_irqd+0x22)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -79+   1.710  ___ipipe_restore_root+0xa (_kick_irqd+0x32)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -78+   1.930  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -76+   1.848  ___up+0x8 (_kick_irqd+0x62)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -74+   1.732  ___wake_up+0xa (___up+0x16)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -72+   1.880  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -70+   2.280  ___wake_up_common+0xc (___wake_up+0x3a)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -68+   1.780  _default_wake_function+0xa (___wake_up_common+0x38)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -66+   1.690  _try_to_wake_up+0xa (_default_wake_function+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -65+   1.810  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -63+   1.660  _sched_clock+0xc (_try_to_wake_up+0x7e)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -61+   2.548  ___muldi3+0xc (_sched_clock+0x26)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -59+   2.622  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -56+   1.440  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -54+   1.580  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -53+   1.320  ___ipipe_restore_root+0xa (___wake_up+0x40)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -52+   1.700  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   +end     0x00000018   -50+   2.980  ___ipipe_grab_irq+0x42
(__common_int_entry+0x72)
==============================================================================================

// Begin to handle IRQ6.

:|   +func    0x0000001f, 0x00000000, 0x00008040
  -47+   1.060  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000006   -46+   1.928  ___ipipe_grab_irq+0x34
(__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -44+   2.532  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -41+   1.640  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
<snip>

Thanks,

-Yi

[-- Attachment #1.2: Type: text/html, Size: 15061 bytes --]

[-- Attachment #2: ipipe-trace-1.log --]
[-- Type: text/x-log, Size: 22548 bytes --]

root:~> ./latency -t2 -p500 -f
root:~> cat /proc/ipipe/trace/frozen
I-pipe frozen back-tracing service on 2.6.16.27-ADI-2006R2/ipipe-1.5-00
------------------------------------------------------------
Freeze: 1541259864240 cycles, Trace Points: 200 (+10)

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
:|   +begin   0x80000001  -374+   2.140  _do_softint+0x22 (_run_workqueue+0xb4)
:|   +end     0x80000000  -372+   1.270  _do_softint+0x3c (_run_workqueue+0xb4)
:|   +begin   0x80000001  -370+   1.330  _do_softint+0x5c (_run_workqueue+0xb4)
:|   +end     0x80000000  -369+   1.150  _do_softint+0x76 (_run_workqueue+0xb4)
:|   +begin   0x80000001  -368+   1.800  _do_softint+0x96 (_run_workqueue+0xb4)
:|   +end     0x80000000  -366+   2.100  _do_softint+0xb0 (_run_workqueue+0xb4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -364+   2.938  _dma_transmit_chars+0xc (_do_softint+0xc6)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -361+   1.882  _local_put_char+0x12 (_dma_transmit_chars+0x66)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -359+   1.270  ___ipipe_test_and_stall_root+0xe (_local_put_char+0x18)
:|   +begin   0x80000001  -358+   1.530  ___ipipe_test_and_stall_root+0x22 (_local_put_char+0x18)
:|   #end     0x80000000  -356+   2.360  ___ipipe_test_and_stall_root+0x36 (_local_put_char+0x18)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -354+   1.300  ___ipipe_restore_root+0xa (_local_put_char+0x46)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -353+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -352+   1.550  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -350+   2.088  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -348+   1.792  _schedule_work+0x10 (_dma_transmit_chars+0xb2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -346+   1.450  _queue_work+0xa (_schedule_work+0x1c)
:|   +begin   0x80000001  -345+   1.730  _queue_work+0x1e (_schedule_work+0x1c)
:|   +end     0x80000000  -343+   1.820  _queue_work+0x32 (_schedule_work+0x1c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -341+   1.660  ___queue_work+0xa (_queue_work+0x72)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -339+   1.020  ___ipipe_test_and_stall_root+0xe (___queue_work+0x12)
:|   +begin   0x80000001  -338+   1.530  ___ipipe_test_and_stall_root+0x22 (___queue_work+0x12)
:|   #end     0x80000000  -337+   2.448  ___ipipe_test_and_stall_root+0x36 (___queue_work+0x12)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -334+   1.612  ___wake_up+0xa (___queue_work+0x50)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -333+   1.290  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #begin   0x80000001  -332+   1.550  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -330+   1.900  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -328+   1.360  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -327+   1.370  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -325+   1.080  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -324+   1.558  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -323+   1.482  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -321+   1.740  ___ipipe_restore_root+0xa (___queue_work+0x56)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -319+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -318+   1.480  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -317+   1.640  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -315+   1.020  ___ipipe_test_and_stall_root+0xe (_run_workqueue+0xb8)
:|   +begin   0x80000001  -314+   1.650  ___ipipe_test_and_stall_root+0x22 (_run_workqueue+0xb8)
:|   #end     0x80000000  -313+   1.658  ___ipipe_test_and_stall_root+0x36 (_run_workqueue+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -311+   1.412  ___wake_up+0xa (_run_workqueue+0x4c)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -310+   1.290  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #begin   0x80000001  -308+   1.550  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -307+   1.700  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -305+   1.360  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -304+   1.370  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -302+   1.080  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -301+   1.558  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -300+   1.702  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -298+   1.670  ___ipipe_restore_root+0xa (_run_workqueue+0x7a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -296+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -295+   1.480  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -294+   1.070  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -293+   1.490  _run_workqueue+0x90 (_worker_thread+0x12a)
:|   +end     0x80000000  -291+   1.600  _run_workqueue+0xa2 (_worker_thread+0x12a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -290+   1.078  _do_softint+0xc (_run_workqueue+0xb4)
:|   +begin   0x80000001  -288+   1.522  _do_softint+0x22 (_run_workqueue+0xb4)
:|   +end     0x80000000  -287+   1.910  _do_softint+0x3c (_run_workqueue+0xb4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -285+   1.400  ___wake_up+0xa (_do_softint+0xf8)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -284    0.970  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   +begin   0x80000001  -283+   1.430  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -281+   1.590  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -280+   1.450  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -278+   1.398  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -277+   1.072  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -276+   1.900  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -274+   1.340  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -272+   1.330  _do_softint+0x5c (_run_workqueue+0xb4)
:|   +end     0x80000000  -271+   1.010  _do_softint+0x76 (_run_workqueue+0xb4)
:|   +begin   0x80000001  -270+   1.490  _do_softint+0x96 (_run_workqueue+0xb4)
:|   +end     0x80000000  -269+   1.480  _do_softint+0xb0 (_run_workqueue+0xb4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -267+   1.108  ___ipipe_test_and_stall_root+0xe (_run_workqueue+0xb8)
:|   +begin   0x80000001  -266+   1.522  ___ipipe_test_and_stall_root+0x22 (_run_workqueue+0xb8)
:|   #end     0x80000000  -265+   1.860  ___ipipe_test_and_stall_root+0x36 (_run_workqueue+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -263+   1.420  ___wake_up+0xa (_run_workqueue+0x4c)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -261+   1.010  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #begin   0x80000001  -260+   1.580  ___ipipe_test_and_stall_root+0x22 (___wake_up+0x14)
:|   #end     0x80000000  -259+   1.600  ___ipipe_test_and_stall_root+0x36 (___wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -257+   1.450  ___wake_up_common+0xc (___wake_up+0x3a)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -256+   1.418  ___ipipe_restore_root+0xa (___wake_up+0x40)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -254+   1.282  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -253+   1.590  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -251+   1.920  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -249+   1.300  ___ipipe_restore_root+0xa (_run_workqueue+0xe6)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -248+   1.040  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -247+   1.500  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -246+   1.780  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -244+   1.808  _worker_thread+0xd4 (_kthread+0xf0)
:|   +end     0x80000000  -242+   1.612  _worker_thread+0xe8 (_kthread+0xf0)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -240+   1.820  _kthread_should_stop+0x10 (_worker_thread+0xf8)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -239+   1.410  _add_wait_queue+0xe (_worker_thread+0x10a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -237    0.960  ___ipipe_test_and_stall_root+0xe (_add_wait_queue+0x18)
:|   +begin   0x80000001  -236+   1.490  ___ipipe_test_and_stall_root+0x22 (_add_wait_queue+0x18)
:|   #end     0x80000000  -235+   1.590  ___ipipe_test_and_stall_root+0x36 (_add_wait_queue+0x18)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -233+   1.390  ___ipipe_restore_root+0xa (_add_wait_queue+0x3e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -232+   1.088  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -231+   1.752  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -229+   1.980  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -227+   1.650  ___sched_text_start+0xe (_worker_thread+0x116)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -225+   1.360  _profile_hit+0xe (___sched_text_start+0x4c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -224+   1.600  _sched_clock+0xc (___sched_text_start+0x7c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -222+   2.730  ___muldi3+0xc (_sched_clock+0x26)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -220+   1.060  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -218+   1.538  ___ipipe_stall_root+0x22 (___sched_text_start+0xec)
:|   #end     0x80000000  -217+   2.002  ___ipipe_stall_root+0x30 (___sched_text_start+0xec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -215+   1.710  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -213+   2.720  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -211+   1.460  ___sched_text_start+0x1e6 (_worker_thread+0x116)
:|   #end     0x80000000  -209+   2.480  ___sched_text_start+0x1f4 (_worker_thread+0x116)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -207+   1.050  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -206+   1.790  ___ipipe_dispatch_event+0x20 (___sched_text_start+0x2de)
:|   #end     0x80000000  -204+   1.568  ___ipipe_dispatch_event+0xcc (___sched_text_start+0x2de)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -202+   1.562  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -201+   2.320  _schedule_event+0x7e (___ipipe_dispatch_event+0xe2)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -198+   1.270  _xnpod_schedule_runnable+0xc (_schedule_event+0xa8)
:|  *#[    0] swapper  0  -197+   2.260  _xnpod_schedule_runnable+0x28 (_schedule_event+0xa8)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -195+   1.740  ___ipipe_restore_pipeline_head+0xa (_schedule_event+0x100)
:|   #end     0x80000000  -193+   1.180  ___ipipe_restore_pipeline_head+0x94 (_schedule_event+0x100)
:|   #begin   0x80000001  -192+   1.640  ___ipipe_dispatch_event+0xfa (___sched_text_start+0x2de)
:|   #end     0x80000000  -190+   1.288  ___ipipe_dispatch_event+0x17a (___sched_text_start+0x2de)
:    #begin   0x80000000  -189+   4.112  ___sched_text_start+0x2f0 (_worker_thread+0x116)
:|   #end     0x80000000  -185+   1.950  ___sched_text_start+0x5ca (_system_call+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -183+   1.080  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -182+   1.570  ___ipipe_unstall_root+0x1a (___sched_text_start+0x3ce)
:|   +end     0x80000000  -180+   1.440  ___ipipe_unstall_root+0x58 (___sched_text_start+0x3ce)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -179+   1.010  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x3d8)
:|   +begin   0x80000001  -178+   1.530  ___ipipe_test_and_stall_root+0x22 (___sched_text_start+0x3d8)
:|   #end     0x80000000  -176+   1.728  ___ipipe_test_and_stall_root+0x36 (___sched_text_start+0x3d8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -174+   1.812  ___ipipe_restore_root+0xa (___sched_text_start+0x4ec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -173+   1.120  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -172+   1.550  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -170!  16.730  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -153+   1.690  ___ipipe_syscall_root+0x14 (_system_call+0x2e)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -152+   1.030  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x40)
:|   +begin   0x80000001  -151+   2.100  ___ipipe_dispatch_event+0x20 (___ipipe_syscall_root+0x40)
:|  + end     0x80000000  -148+   2.198  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -146+   2.522  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xe2)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -144+   1.580  _xnpod_schedule+0xa (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -142+   1.530  _xnpod_schedule+0x64 (_hisyscall_event+0xbe)
:|  + end     0x80000000  -141    0.990  _xnpod_schedule+0x72 (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -140+   1.510  _xnpod_schedule+0xc4 (_hisyscall_event+0xbe)
:|  # [    0] swapper  0  -138+   1.710  _xnpod_schedule+0x104 (_hisyscall_event+0xbe)
:|  # func    0x0000001f, 0x00000000, 0x00008000
 -136+   1.160  ___ipipe_restore_pipeline_head+0xa (_xnpod_schedule+0x2b0)
:|  + end     0x80000000  -135+   1.948  ___ipipe_restore_pipeline_head+0x94 (_xnpod_schedule+0x2b0)
:|  + begin   0x80000001  -133+   1.932  ___ipipe_dispatch_event+0xfa (___ipipe_syscall_root+0x40)
:|   +end     0x80000000  -131+   2.030  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -129+   1.270  _losyscall_event+0x14 (___ipipe_dispatch_event+0xe2)
:|   +begin   0x80000001  -128+   1.600  ___ipipe_dispatch_event+0xfa (___ipipe_syscall_root+0x40)
:|   +end     0x80000000  -126+   2.510  ___ipipe_dispatch_event+0x17a (___ipipe_syscall_root+0x40)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -124+   1.620  _sys_write+0xa (_system_call+0x86)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -122+   1.560  _fget_light+0x16 (_sys_write+0x16)
:|   +begin   0x80000001  -121+   1.628  _fget_light+0x8c (_sys_write+0x16)
:|   +end     0x80000000  -119+   2.212  _fget_light+0xa6 (_sys_write+0x16)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -117+   2.630  _vfs_write+0xc (_sys_write+0x34)
:    +func    0x0000ffff, 0x00000800, 0x00008000
 -114+   5.020  _rw_verify_area+0xa (_vfs_write+0x136)


==========================================================================

:|   +func    0x0000001f, 0x00000040, 0x00008800
 -109+   2.690  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000018  -107+   2.850  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -104+   3.960  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000040, 0x00008800
 -100+   1.760  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -98+   3.298  _ipipe_test_and_stall_pipeline_from+0xa (___ipipe_ack_irq+0x20)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -95+   2.302  _bf533_internal_mask_irq+0xc (___ipipe_ack_irq+0x40)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -92+   3.290  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|   +func    0x0000001f, 0x00000040, 0x00008800
  -89+   6.110  ___ipipe_sync_stage+0xe (___ipipe_walk_pipeline+0xde)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -83+   1.690  _kick_irqd+0x1c (___ipipe_sync_stage+0x290)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -81+   1.920  ___ipipe_test_and_stall_root+0xe (_kick_irqd+0x22)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -79+   1.710  ___ipipe_restore_root+0xa (_kick_irqd+0x32)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -78+   1.930  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -76+   1.848  ___up+0x8 (_kick_irqd+0x62)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -74+   1.732  ___wake_up+0xa (___up+0x16)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -72+   1.880  ___ipipe_test_and_stall_root+0xe (___wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -70+   2.280  ___wake_up_common+0xc (___wake_up+0x3a)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -68+   1.780  _default_wake_function+0xa (___wake_up_common+0x38)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -66+   1.690  _try_to_wake_up+0xa (_default_wake_function+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -65+   1.810  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -63+   1.660  _sched_clock+0xc (_try_to_wake_up+0x7e)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -61+   2.548  ___muldi3+0xc (_sched_clock+0x26)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -59+   2.622  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -56+   1.440  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -54+   1.580  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -53+   1.320  ___ipipe_restore_root+0xa (___wake_up+0x40)
:|   #func    0x0000001f, 0x00000040, 0x00008800
  -52+   1.700  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   +end     0x00000018   -50+   2.980  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)
==============================================================================================


:|   +func    0x0000001f, 0x00000000, 0x00008040
  -47+   1.060  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   +begin   0x00000006   -46+   1.928  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -44+   2.532  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -41+   1.640  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|   +func    0x0000001f, 0x00000000, 0x00008040
  -40+   2.540  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -37+   2.020  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xb6)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -35+   1.910  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -33+   1.610  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -32+   1.800  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -30+   2.038  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -28+   2.562  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -25+   1.820  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -23+   2.110  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -21+   5.080  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func    0x0000001f, 0x00000000, 0x00008040
  -16+   7.120  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # func    0x0000001f, 0x00000000, 0x00008040
   -9+   2.850  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  # func    0x0000001f, 0x00000000, 0x00008040
   -6+   1.560  _ipipe_trace_frozen_reset+0xa (_eval_inner_loop+0xe0)
:|  # func    0x0000001f, 0x00000000, 0x00008040
   -5+   4.238  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
<|  # freeze  0x0001933e     1    3.132  _eval_inner_loop+0xe6 (_timer_proc+0x24)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    4    1.680  _xnpod_get_time+0xa (_timer_proc+0x2a)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    5    1.570  ___muldi3+0xc (_xnpod_get_time+0x52)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    7    1.520  ___muldi3+0xc (_xnpod_get_time+0x6e)
 |  # func    0x0000001f, 0x00000000, 0x00008040
    8    3.390  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   12    5.670  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
 |  # begin   0x00059798    17    2.110  _timer_proc+0x44 (_xntimer_do_tick_aperiodic+0x1b2)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   20    2.558  _xntimer_start+0xa (_timer_proc+0x74)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   22    3.552  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   26    1.630  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
 |  # func    0x0000001f, 0x00000000, 0x00008040
   27    0.000  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)

[-- Attachment #3: ipipe-tarce-2.log --]
[-- Type: text/x-log, Size: 23712 bytes --]

root:~> ./latency -t2 -p200 -f
root:~> cat /proc/ipipe/trace/frozen
I-pipe frozen back-tracing service on 2.6.16.27-ADI-2006R2/ipipe-1.5-00
------------------------------------------------------------
Freeze: 90786953155 cycles, Trace Points: 200 (+10)

 +----- Hard IRQs ('|': locked)
 |+---- <unused>
 ||+--- <unused>
 |||+-- Xenomai
 ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled)
 |||||                        +---------- Delay flag ('+': > 1 us, '!': > 10 us)
 |||||                        |        +- NMI noise ('N')
 |||||                        |        |
      Type    User Val.   Time    Delay  Function (Parent)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -429+   3.310  ___muldi3+0xc (_sched_clock+0x26)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -426+   1.060  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -425+   1.330  ___ipipe_stall_root+0x22 (___sched_text_start+0xec)
:|   #end     0x80000000  -424+   1.710  ___ipipe_stall_root+0x30 (___sched_text_start+0xec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -422+   1.750  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -420+   3.040  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -417+   1.758  ___sched_text_start+0x1e6 (_worker_thread+0x116)
:|   #end     0x80000000  -416+   3.472  ___sched_text_start+0x1f4 (_worker_thread+0x116)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -412+   1.310  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -411+   1.770  ___ipipe_dispatch_event+0x20 (___sched_text_start+0x2de)
:|   #end     0x80000000  -409+   1.490  ___ipipe_dispatch_event+0xcc (___sched_text_start+0x2de)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -407+   1.570  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -406+   2.440  _schedule_event+0x7e (___ipipe_dispatch_event+0xe2)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -403+   2.020  _xnpod_schedule_runnable+0xc (_schedule_event+0xa8)
:|  *#[    0] swapper  0  -401+   2.228  _xnpod_schedule_runnable+0x28 (_schedule_event+0xa8)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -399+   1.892  ___ipipe_restore_pipeline_head+0xa (_schedule_event+0x100)
:|   #end     0x80000000  -397+   1.440  ___ipipe_restore_pipeline_head+0x94 (_schedule_event+0x100)
:|   #begin   0x80000001  -396+   1.640  ___ipipe_dispatch_event+0xfa (___sched_text_start+0x2de)
:|   #end     0x80000000  -394+   1.090  ___ipipe_dispatch_event+0x17a (___sched_text_start+0x2de)
:    #begin   0x80000000  -393+   4.220  ___sched_text_start+0x2f0 (_worker_thread+0x116)
:|   #end     0x80000000  -389+   1.470  ___sched_text_start+0x5ca (_system_call+0xb8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -387+   1.080  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -386+   1.558  ___ipipe_unstall_root+0x1a (___sched_text_start+0x3ce)
:|   +end     0x80000000  -385+   1.472  ___ipipe_unstall_root+0x58 (___sched_text_start+0x3ce)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -383+   1.400  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x3d8)
:|   +begin   0x80000001  -382+   1.530  ___ipipe_test_and_stall_root+0x22 (___sched_text_start+0x3d8)
:|   #end     0x80000000  -380+   1.470  ___ipipe_test_and_stall_root+0x36 (___sched_text_start+0x3d8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -379+   1.300  ___ipipe_restore_root+0xa (___sched_text_start+0x4ec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -378+   1.040  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -377+   1.670  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -375!  19.558  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -355+   1.722  ___ipipe_syscall_root+0x14 (_system_call+0x2e)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -354+   1.290  ___ipipe_dispatch_event+0xa (___ipipe_syscall_root+0x40)
:|   +begin   0x80000001  -352+   2.100  ___ipipe_dispatch_event+0x20 (___ipipe_syscall_root+0x40)
:|  + end     0x80000000  -350+   1.950  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -348+   1.860  _hisyscall_event+0x10 (___ipipe_dispatch_event+0xe2)
:   + func    0x0000ffff, 0x00000000, 0x00008000
 -346+   1.900  _xnpod_schedule+0xa (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -345+   1.650  _xnpod_schedule+0x64 (_hisyscall_event+0xbe)
:|  + end     0x80000000  -343+   1.158  _xnpod_schedule+0x72 (_hisyscall_event+0xbe)
:|  + begin   0x80000001  -342+   1.742  _xnpod_schedule+0xc4 (_hisyscall_event+0xbe)
:|  # [    0] swapper  0  -340+   1.840  _xnpod_schedule+0x104 (_hisyscall_event+0xbe)
:|  # func    0x0000001f, 0x00000000, 0x00008000
 -338+   1.310  ___ipipe_restore_pipeline_head+0xa (_xnpod_schedule+0x2b0)
:|  + end     0x80000000  -337+   3.420  ___ipipe_restore_pipeline_head+0x94 (_xnpod_schedule+0x2b0)
:|  + begin   0x80000001  -333+   1.560  ___ipipe_dispatch_event+0xfa (___ipipe_syscall_root+0x40)
:|   +end     0x80000000  -332+   1.950  ___ipipe_dispatch_event+0xcc (___ipipe_syscall_root+0x40)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -330+   2.610  _losyscall_event+0x14 (___ipipe_dispatch_event+0xe2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -327+   1.888  _sys_rtdm_ioctl+0xa (_losyscall_event+0x94)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -325+   2.122  __rtdm_ioctl+0xa (_sys_rtdm_ioctl+0x1c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -323+   1.420  _rtdm_context_get+0x10 (__rtdm_ioctl+0x20)
:|   +begin   0x80000001  -322+   2.760  _rtdm_context_get+0x24 (__rtdm_ioctl+0x20)
:|  *+func    0x0000001f, 0x00000000, 0x00008000
 -319+   1.920  ___ipipe_test_and_stall_root+0xe (_rtdm_context_get+0x58)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -317+   1.410  ___ipipe_restore_root+0xa (_rtdm_context_get+0x6e)
:|  *#func    0x0000001f, 0x00000000, 0x00008000
 -316+   1.130  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:|  *+end     0x80000000  -315+   1.680  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:   *+func    0x0000ffff, 0x00000000, 0x00008000
 -313+   1.088  ___ipipe_restore_pipeline_head+0xa (_rtdm_context_get+0x82)
:|  *+begin   0x80000001  -312+   1.542  ___ipipe_restore_pipeline_head+0x1e (_rtdm_context_get+0x82)
:|   +end     0x80000000  -310+   3.350  ___ipipe_restore_pipeline_head+0x94 (_rtdm_context_get+0x82)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -307+   1.960  _rt_tmbench_ioctl_nrt+0x16 (__rtdm_ioctl+0x4a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -305    0.970  ___ipipe_test_and_stall_root+0xe (__rtdm_ioctl+0x50)
:|   +begin   0x80000001  -304+   1.430  ___ipipe_test_and_stall_root+0x22 (__rtdm_ioctl+0x50)
:|   #end     0x80000000  -303+   3.000  ___ipipe_test_and_stall_root+0x36 (__rtdm_ioctl+0x50)

===================================================================================================
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -300+   1.340  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|   #begin   0x00000006  -298+   2.488  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -296+   1.642  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -294+   1.830  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|   #func    0x0000001f, 0x00000000, 0x00008040
 -292+   2.340  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -290+   1.570  _xnintr_clock_handler+0x10 (___ipipe_dispatch_wired+0xb6)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -288+   1.540  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -287+   1.460  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -285+   2.070  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -283+   1.858  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -282+   2.062  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -279+   2.010  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -277+   2.130  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -275+   4.070  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -271+   6.020  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -265+   1.940  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -263+   1.640  _xnpod_get_time+0xa (_timer_proc+0x2a)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -262+   1.588  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -260+   1.662  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -258+   3.110  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -255+   5.370  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  #*begin   0x000227ae  -250+   1.500  _timer_proc+0x44 (_xntimer_do_tick_aperiodic+0x1b2)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -248+   1.590  _xntimer_start+0xa (_timer_proc+0x74)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -247+   2.120  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -245+   1.660  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -243+   1.908  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -241+   4.212  ___div64_32+0x10 (_xntimer_do_start_aperiodic+0x5ae)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -237+   1.830  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1bc)
:|  #*func    0x0000001f, 0x00000000, 0x00008040
 -235+   6.750  ___muldi3+0xc (_xntimer_do_start_aperiodic+0x1d0)
:|  #*func    0x0000001f, 0x00000000, 0x00008000
 -228+   2.330  _xnpod_schedule_deferred+0xc (___ipipe_call_irqtail+0x1e)
:    #func    0x0000ffff, 0x00000000, 0x0000c000
 -226+   1.460  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x126)
:    #end     0x00000006  -225+   1.950  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)
===============================================================================================

:    #func    0x0000ffff, 0x00000000, 0x00008000
 -223+   1.390  ___ipipe_restore_root+0xa (__rtdm_ioctl+0x64)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -221+   1.138  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -220+   1.542  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -219+   2.710  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -216+   1.970  _xnshadow_harden+0x12 (_losyscall_event+0xb2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -214    0.970  ___ipipe_test_and_stall_root+0xe (_xnshadow_harden+0x3e)
:|   +begin   0x80000001  -213+   1.570  ___ipipe_test_and_stall_root+0x22 (_xnshadow_harden+0x3e)
:|   #end     0x80000000  -211+   1.730  ___ipipe_test_and_stall_root+0x36 (_xnshadow_harden+0x3e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -210+   1.390  ___ipipe_restore_root+0xa (_xnshadow_harden+0x56)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -208+   1.088  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -207+   1.542  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -206+   1.600  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:|   +begin   0x80000001  -204+   1.730  _xnshadow_harden+0x76 (_losyscall_event+0xb2)
:|   +end     0x80000000  -202+   1.890  _xnshadow_harden+0x8c (_losyscall_event+0xb2)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -200+   1.490  ___wake_up_sync+0xc (_xnshadow_harden+0xa4)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -199+   1.040  ___ipipe_test_and_stall_root+0xe (___wake_up_sync+0x18)
:|   +begin   0x80000001  -198+   1.530  ___ipipe_test_and_stall_root+0x22 (___wake_up_sync+0x18)
:|   #end     0x80000000  -196+   2.038  ___ipipe_test_and_stall_root+0x36 (___wake_up_sync+0x18)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -194+   2.082  ___wake_up_common+0xc (___wake_up_sync+0x38)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -192+   1.880  _default_wake_function+0xa (___wake_up_common+0x38)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -190+   1.540  _try_to_wake_up+0xa (_default_wake_function+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -189+   1.010  ___ipipe_test_and_stall_root+0xe (_try_to_wake_up+0x14)
:|   #begin   0x80000001  -188+   1.460  ___ipipe_test_and_stall_root+0x22 (_try_to_wake_up+0x14)
:|   #end     0x80000000  -186+   1.840  ___ipipe_test_and_stall_root+0x36 (_try_to_wake_up+0x14)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -184+   1.660  _sched_clock+0xc (_try_to_wake_up+0x7e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -183+   2.618  ___muldi3+0xc (_sched_clock+0x26)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -180+   3.222  _enqueue_task+0xe (_try_to_wake_up+0xd2)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -177+   1.620  ___ipipe_restore_root+0xa (_try_to_wake_up+0x3c)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -175+   1.080  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #begin   0x80000001  -174+   1.620  ___ipipe_stall_root+0x22 (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -173+   1.530  ___ipipe_stall_root+0x30 (___ipipe_restore_root+0x10)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -171+   1.350  ___ipipe_restore_root+0xa (___wake_up_sync+0x3e)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -170+   1.080  ___ipipe_unstall_root+0x8 (___ipipe_restore_root+0x1a)
:    #begin   0x80000000  -169+   1.558  ___ipipe_unstall_root+0x1a (___ipipe_restore_root+0x1a)
:|   +end     0x80000000  -167+   1.592  ___ipipe_unstall_root+0x58 (___ipipe_restore_root+0x1a)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -166+   2.010  ___sched_text_start+0xe (_xnshadow_harden+0xa8)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -163+   1.910  _profile_hit+0xe (___sched_text_start+0x4c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -162+   1.390  _sched_clock+0xc (___sched_text_start+0x7c)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -160+   3.030  ___muldi3+0xc (_sched_clock+0x26)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -157+   1.020  ___ipipe_stall_root+0xe (___sched_text_start+0xec)
:|   +begin   0x80000001  -156+   1.530  ___ipipe_stall_root+0x22 (___sched_text_start+0xec)
:|   #end     0x80000000  -155+   1.578  ___ipipe_stall_root+0x30 (___sched_text_start+0xec)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -153+   1.662  _deactivate_task+0xe (___sched_text_start+0x464)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -151+   3.360  _dequeue_task+0xe (_deactivate_task+0x1c)
:|   #begin   0x80000001  -148+   1.750  ___sched_text_start+0x1e6 (_xnshadow_harden+0xa8)
:|   #end     0x80000000  -146+   4.400  ___sched_text_start+0x1f4 (_xnshadow_harden+0xa8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -142+   1.020  ___ipipe_dispatch_event+0xa (___sched_text_start+0x2de)
:|   #begin   0x80000001  -141+   1.730  ___ipipe_dispatch_event+0x20 (___sched_text_start+0x2de)
:|   #end     0x80000000  -139+   1.740  ___ipipe_dispatch_event+0xcc (___sched_text_start+0x2de)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -137+   1.598  _schedule_event+0x16 (___ipipe_dispatch_event+0xe2)
:|   #begin   0x80000001  -136+   1.632  ___ipipe_dispatch_event+0xfa (___sched_text_start+0x2de)
:|   #end     0x80000000  -134+   1.370  ___ipipe_dispatch_event+0x17a (___sched_text_start+0x2de)
:    #begin   0x80000000  -133+   2.320  ___sched_text_start+0x2f0 (_xnshadow_harden+0xa8)
:|   #func    0x0000001f, 0x00000000, 0x00008000
 -130+   1.670  ___ipipe_test_and_stall_root+0xe (___sched_text_start+0x678)
:|   #func    0x0000001f, 0x00000000, 0x00008000
 -129+   1.440  ___ipipe_restore_root+0xa (___sched_text_start+0x68c)
:|   #func    0x0000001f, 0x00000000, 0x00008000
 -127+   3.910  ___ipipe_stall_root+0xe (___ipipe_restore_root+0x10)
:|   #end     0x80000000  -123+   1.860  ___sched_text_start+0x5ca (_gatekeeper_thread+0xe8)
:    #func    0x0000ffff, 0x00000000, 0x00008000
 -122+   1.048  ___ipipe_unstall_root+0x8 (___sched_text_start+0x3ce)
:    #begin   0x80000000  -121+   1.512  ___ipipe_unstall_root+0x1a (___sched_text_start+0x3ce)
:|   +end     0x80000000  -119+   2.340  ___ipipe_unstall_root+0x58 (___sched_text_start+0x3ce)
:    +func    0x0000ffff, 0x00000000, 0x00008000
 -117+   1.620  _kthread_should_stop+0x10 (_gatekeeper_thread+0xec)
:|   +begin   0x80000001  -115+   3.370  _gatekeeper_thread+0x100 (_kthread+0xf0)
:|  *+func    0x0000001f, 0x00000000, 0x00008000
 -112+   2.070  _xnpod_resume_thread+0xa (_gatekeeper_thread+0x186)
:|  *+[   51] -<?>-    0  -110+   4.990  _xnpod_resume_thread+0x4a (_gatekeeper_thread+0x186)
:|  *+func    0x0000001f, 0x00001000, 0x00008000
 -105+   1.370  _xnpod_schedule_runnable+0xc (_gatekeeper_thread+0x1be)
:|  *+[    0] swapper -1  -103+   3.738  _xnpod_schedule_runnable+0x28 (_gatekeeper_thread+0x1be)

========================================================================================

:|  *+func    0x0000001f, 0x00001040, 0x00008000
 -100+   2.592  _xnpod_schedule+0xa (_gatekeeper_thread+0x1ce)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -97+   3.000  _ipipe_trigger_irq+0x12 (_xnpod_schedule+0x2e6)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -94+   1.710  ___ipipe_handle_irq+0x1e (_ipipe_trigger_irq+0x86)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -92+   2.460  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  *+func    0x0000001f, 0x00001040, 0x00008000
  -90+   1.860  ___ipipe_restore_pipeline_head+0xa (_gatekeeper_thread+0x138)
:|   +func    0x0000001f, 0x00001040, 0x00008000
  -88+   1.990  ___ipipe_walk_pipeline+0x18 (___ipipe_restore_pipeline_head+0xd0)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -86+   2.300  _ipipe_suspend_domain+0xa (___ipipe_walk_pipeline+0xac)
:|  + func    0x0000001f, 0x00001040, 0x00008000
  -84+   5.018  ___ipipe_sync_stage+0xe (_ipipe_suspend_domain+0x4e)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -79+   1.942  _xnpod_schedule_handler+0xe (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00001040, 0x00008000
  -77+   2.310  _xnpod_schedule+0xa (_xnpod_schedule_handler+0x5c)
:|  # [    0] swapper -1   -74+   6.560  _xnpod_schedule+0x104 (_xnpod_schedule_handler+0x5c)
:|  # end     0x80000000   -68+   3.250  ___sched_text_start+0x5ca (_xnshadow_harden+0xa8)

==========================================================================================
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -65    0.940  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x00000006   -64+   2.300  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -61+   1.650  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -60+   1.508  _rthal_timer_ack+0x8 (___ipipe_handle_irq+0x178)
:|  # func    0x0000001f, 0x00001000, 0x00008040
  -58+   1.382  ___ipipe_dispatch_wired+0x14 (___ipipe_handle_irq+0x190)
:|  # end     0x00000006   -57+   3.060  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)

============================================================================================

:|  # func    0x0000001f, 0x00000000, 0x00009000
  -54+   2.670  ___ipipe_grab_irq+0x22 (__common_int_entry+0x72)
:|  # begin   0x0000001a   -51+   1.940  ___ipipe_grab_irq+0x34 (__common_int_entry+0x72)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -49+   3.540  ___ipipe_handle_irq+0x1e (___ipipe_grab_irq+0x3c)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -46+   2.310  ___ipipe_ack_irq+0x10 (___ipipe_handle_irq+0xee)
:|  # func    0x0000001f, 0x00000000, 0x00009000
  -43+   1.240  ___ipipe_walk_pipeline+0x18 (___ipipe_handle_irq+0x148)
:|  # end     0x0000001a   -42+   2.118  ___ipipe_grab_irq+0x42 (__common_int_entry+0x72)
:   # func    0x0000ffff, 0x00000000, 0x00008000
  -40+   1.242  _ipipe_unstall_pipeline_head+0x8 (_xnshadow_harden+0xce)
:|  # begin   0x80000001   -39+   2.870  _ipipe_unstall_pipeline_head+0x18 (_xnshadow_harden+0xce)
:|  + func    0x0000001f, 0x00000000, 0x00008000
  -36+   2.120  ___ipipe_sync_stage+0xe (_ipipe_unstall_pipeline_head+0x74)

=======================================================================================================

:|  # func    0x0000001f, 0x00000000, 0x00008000
  -34+   1.360  _xnintr_clock_handler+0x10 (___ipipe_sync_stage+0x186)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -32+   1.710  _xnintr_irq_handler+0x18 (_xnintr_clock_handler+0x18)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -31+   1.870  _xnpod_announce_tick+0xc (_xnintr_irq_handler+0x2a)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -29+   2.480  _xntimer_do_tick_aperiodic+0x14 (_xnpod_announce_tick+0x16)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -26+   1.698  _timer_proc+0xc (_xntimer_do_tick_aperiodic+0x1b2)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -25+   2.042  _xnpod_get_time+0xa (_timer_proc+0x10)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -22+   1.990  ___muldi3+0xc (_xnpod_get_time+0x52)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -20+   2.230  ___muldi3+0xc (_xnpod_get_time+0x6e)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -18+   4.150  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
:|  # func    0x0000001f, 0x00000000, 0x00008000
  -14+   6.120  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
:|  # func    0x0000001f, 0x00000000, 0x00008000
   -8+   1.710  _eval_inner_loop+0xe (_timer_proc+0x24)
:|  # func    0x0000001f, 0x00000000, 0x00008000
   -6+   1.560  _ipipe_trace_frozen_reset+0xa (_eval_inner_loop+0xe0)
:|  # func    0x0000001f, 0x00000000, 0x00008000
   -5+   4.218  ___ipipe_global_path_lock+0xa (_ipipe_trace_frozen_reset+0xe)
<|  # freeze  0x00017ce6     1    2.182  _eval_inner_loop+0xe6 (_timer_proc+0x24)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    3    1.850  _xnpod_get_time+0xa (_timer_proc+0x2a)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    5    1.610  ___muldi3+0xc (_xnpod_get_time+0x52)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    6    1.530  ___muldi3+0xc (_xnpod_get_time+0x6e)
 |  # func    0x0000001f, 0x00000000, 0x00008000
    8    2.740  ___div64_32+0x10 (_xnpod_get_time+0x1f0)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   10    5.830  ___div64_32+0x10 (_xnpod_get_time+0x1e4)
 |  # begin   0x000129ec    16    2.110  _timer_proc+0x44 (_xntimer_do_tick_aperiodic+0x1b2)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   18    2.248  _xntimer_start+0xa (_timer_proc+0x74)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   21    2.702  _xntimer_do_start_aperiodic+0xa (_xntimer_start+0x62)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   23    1.800  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xd6)
 |  # func    0x0000001f, 0x00000000, 0x00008000
   25    0.000  ___muldi3+0xc (_xntimer_do_start_aperiodic+0xee)

^ permalink raw reply	[flat|nested] 9+ messages in thread

* [Xenomai-core] Re: [Adeos-main] ipip tracer porting for blackfin
  2006-11-13  3:51       ` Li Yi (Adam)
  2006-11-13  9:56         ` Li Yi (Adam)
@ 2006-11-14 20:02         ` Jan Kiszka
  1 sibling, 0 replies; 9+ messages in thread
From: Jan Kiszka @ 2006-11-14 20:02 UTC (permalink / raw)
  To: Li Yi (Adam); +Cc: adeos-main, xenomai

[-- Attachment #1: Type: text/plain, Size: 1311 bytes --]

Li Yi (Adam) wrote:
> Please see my comments bellow:
> 
> On 11/11/06, Jan Kiszka <jan.kiszka@domain.hid> wrote:
>>
>> Li Yi (Adam) wrote:
>> > Hi Jan and Philippe,
>>
>>
>> This must be
>> http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2337
>>
>>
>> saying "root thread is about the be scheduled away". But then I'm
>> missing code here that one would normally see on x86:
>>
>> http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2442
>>
>>
>> Likely inlined on bfin, but this *must* make some noise:
>>
>> http://www.rts.uni-hannover.de/xenomai/lxr/source/ksrc/nucleus/pod.c#L2453
>>
>>
>> Instead we enter the pending timer interrupt. This currently makes no
>> sense to me, rather make me think that something is skewed. Maybe only
>> the tracer, maybe more...
> 
> 
> Do you have a tracer-log for i386 for reference? So I can do some compare.

Oops, I found the same pattern on i386 - and that made me think about it
again: All fine, red herring, no bug.

What I failed to see is the special case of switching from the
gatekeeper back to the Xenomai thread that is about to be hardened. In
that case, we actually do not see the second pid trace after the switch.

Ok, the world was just proven to be still round. :)

Jan


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 249 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

end of thread, other threads:[~2006-11-14 20:02 UTC | newest]

Thread overview: 9+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2006-09-30  7:59 [Xenomai-core] ipip tracer porting for blackfin Li Yi (Adam)
2006-10-01 22:16 ` [Xenomai-core] Re: [Adeos-main] " Jan Kiszka
2006-11-10 13:54   ` Li Yi (Adam)
2006-11-10 14:27     ` Philippe Gerum
2006-11-10 16:11     ` Jan Kiszka
2006-11-13  3:51       ` Li Yi (Adam)
2006-11-13  9:56         ` Li Yi (Adam)
2006-11-13 10:01           ` [Xenomai-core] Fwd: " Li Yi (Adam)
2006-11-14 20:02         ` [Xenomai-core] Re: [Adeos-main] " Jan Kiszka

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.