We added some trace information, but from this gap point (6208.695948-> 6208.708726), it appears that the Cobalt kernel was frozen and no interrupt handling occurred. include trace point event : irq_vectors:call_function_entry irq_vectors:call_function_exit irq_vectors:call_function_single_entry irq_vectors:call_function_single_exit rt-task3 2737 [001] 6208.695888: cobalt_core:cobalt_trace_pid: pid=2737, prio=91 rt-task3 2737 [001] 6208.695888: cobalt_core:cobalt_synch_acquire: synch=0xffffc9000097a908 rt-task3 2737 [001] 6208.695890: cobalt_core:cobalt_synch_release: synch=0xffffc9000097a908 rt-task3 2737 [001] 6208.695890: cobalt_core:cobalt_synch_sleepon: synch=0xffffc9000097ab08 rt-task3 2737 [001] 6208.695890: cobalt_core:cobalt_thread_suspend: pid=2737 mask=0x2 timeout=6208672099128 timeout_mode=1 wchan=0xffffc9000097ab08 rt-task3 2737 [001] 6208.695890: cobalt_core:cobalt_timer_start: timer=0xffffc90000979cf0(MAIN-Thread) value=6208672099128 interval=0 mode=0x1 rt-task3 2737 [001] 6208.695891: cobalt_core:cobalt_schedule: status=0x10000000 rt-task3 2737 [001] 6208.695891: cobalt_core:cobalt_trace_pid: pid=2737, prio=91 rt-task3 2737 [001] 6208.695891: cobalt_core:cobalt_switch_context: prev_name=rt-task3 prev_pid=2737 prev_prio=91 prev_state=0x48046 ==> next_name=ROOT/1 next_pid=0 next_prio=-1 rt-task3 2737 [001] 6208.695892: cobalt_core:cobalt_timer_stop: timer=0xffff8884700b6828 MAIN-Thread 2798 [001] 6208.695892: cobalt_core:cobalt_trace_pid: pid=2798, prio=-1 swapper 0 [001] 6208.695901: cobalt_core:cobalt_timer_start: timer=0xffff8884700b6298([host-timer/1]) value=94752 interval=0 mode=0x0 swapper 0 [001] 6208.695906: irq:irq_pipeline_entry: irq=33027 swapper 0 [001] 6208.695906: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [001] 6208.695907: cobalt_core:cobalt_timer_expire: timer=0xffffc90000978ef0 swapper 0 [001] 6208.695907: cobalt_core:cobalt_thread_resume: name=rt-task474 pid=2738 mask=0x4 swapper 0 [001] 6208.695907: cobalt_core:cobalt_trace_pid: pid=2738, prio=85 swapper 0 [001] 6208.695907: cobalt_core:cobalt_timer_stop: timer=0xffffc90000978ef0 swapper 0 [001] 6208.695908: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [001] 6208.695908: irq:irq_pipeline_exit: irq=33027 swapper 0 [001] 6208.695908: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [001] 6208.695909: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [001] 6208.695909: cobalt_core:cobalt_switch_context: prev_name=ROOT/1 prev_pid=0 prev_prio=-1 prev_state=0x18008 ==> next_name=rt-task474 next_pid=2738 next_prio=85 swapper 0 [001] 6208.695909: cobalt_core:cobalt_timer_start: timer=0xffff8884700b6828([watchdog]) value=1000000000 interval=0 mode=0x0 rt-task4 2738 [001] 6208.695910: cobalt_core:cobalt_trace_pid: pid=2738, prio=85 rt-task4 2738 [001] 6208.695911: cobalt_core:cobalt_thread_suspend: pid=2738 mask=0x4 timeout=6208672120609 timeout_mode=1 wchan=(nil) rt-task4 2738 [001] 6208.695911: cobalt_core:cobalt_timer_start: timer=0xffffc90000978ef0(MAIN-Thread) value=6208672120609 interval=0 mode=0x1 rt-task4 2738 [001] 6208.695911: cobalt_core:cobalt_schedule: status=0x10000000 rt-task4 2738 [001] 6208.695912: cobalt_core:cobalt_trace_pid: pid=2738, prio=85 rt-task4 2738 [001] 6208.695912: cobalt_core:cobalt_switch_context: prev_name=rt-task4 prev_pid=2738 prev_prio=85 prev_state=0x48044 ==> next_name=ROOT/1 next_pid=0 next_prio=-1 rt-task4 2738 [001] 6208.695912: cobalt_core:cobalt_timer_stop: timer=0xffff8884700b6828 swapper 0 [001] 6208.695913: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [002] 6208.695920: irq:irq_pipeline_entry: irq=33027 swapper 0 [002] 6208.695920: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [002] 6208.695921: cobalt_core:cobalt_timer_expire: timer=0xffffc9000096eef0 swapper 0 [002] 6208.695921: cobalt_core:cobalt_thread_resume: name=rt-task2 pid=2762 mask=0x4 swapper 0 [002] 6208.695921: cobalt_core:cobalt_trace_pid: pid=2762, prio=85 swapper 0 [002] 6208.695921: cobalt_core:cobalt_timer_stop: timer=0xffffc9000096eef0 swapper 0 [002] 6208.695922: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [002] 6208.695922: irq:irq_pipeline_exit: irq=33027 swapper 0 [002] 6208.695922: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [002] 6208.695923: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [002] 6208.695923: cobalt_core:cobalt_switch_context: prev_name=ROOT/2 prev_pid=0 prev_prio=-1 prev_state=0x18008 ==> next_name=rt-task2 next_pid=2762 next_prio=85 swapper 0 [002] 6208.695923: cobalt_core:cobalt_timer_start: timer=0xffff888470136828([watchdog]) value=1000000000 interval=0 mode=0x0 rt-task2 2762 [002] 6208.695924: cobalt_core:cobalt_trace_pid: pid=2762, prio=85 rt-task2 2762 [002] 6208.695925: cobalt_core:cobalt_thread_suspend: pid=2762 mask=0x4 timeout=6208672084819 timeout_mode=1 wchan=(nil) rt-task2 2762 [002] 6208.695925: cobalt_core:cobalt_timer_start: timer=0xffffc9000096eef0(MAIN-Thread) value=6208672084819 interval=0 mode=0x1 swapper 0 [001] 6208.695925: irq:irq_pipeline_entry: irq=33027 rt-task2 2762 [002] 6208.695926: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [001] 6208.695926: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt rt-task2 2762 [002] 6208.695926: cobalt_core:cobalt_trace_pid: pid=2762, prio=85 rt-task2 2762 [002] 6208.695926: cobalt_core:cobalt_switch_context: prev_name=rt-task2 prev_pid=2762 prev_prio=85 prev_state=0x48044 ==> next_name=ROOT/2 next_pid=0 next_prio=-1 rt-task2 2762 [002] 6208.695927: cobalt_core:cobalt_timer_stop: timer=0xffff888470136828 swapper 0 [002] 6208.695927: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [001] 6208.695928: cobalt_core:cobalt_timer_expire: timer=0xffffc90000977ef0 swapper 0 [001] 6208.695928: cobalt_core:cobalt_thread_resume: name=rt-task1 pid=2739 mask=0x4 swapper 0 [001] 6208.695928: cobalt_core:cobalt_trace_pid: pid=2739, prio=86 swapper 0 [001] 6208.695928: cobalt_core:cobalt_timer_stop: timer=0xffffc90000977ef0 swapper 0 [001] 6208.695929: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [001] 6208.695929: irq:irq_pipeline_exit: irq=33027 swapper 0 [001] 6208.695929: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [001] 6208.695930: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [001] 6208.695930: cobalt_core:cobalt_switch_context: prev_name=ROOT/1 prev_pid=0 prev_prio=-1 prev_state=0x18008 ==> next_name=rt-task1 next_pid=2739 next_prio=86 swapper 0 [001] 6208.695930: cobalt_core:cobalt_timer_start: timer=0xffff8884700b6828([watchdog]) value=1000000000 interval=0 mode=0x0 rt-task1 2739 [001] 6208.695931: cobalt_core:cobalt_trace_pid: pid=2739, prio=86 rt-task1 2739 [001] 6208.695933: cobalt_core:cobalt_thread_suspend: pid=2739 mask=0x4 timeout=6208672142529 timeout_mode=1 wchan=(nil) rt-task1 2739 [001] 6208.695933: cobalt_core:cobalt_timer_start: timer=0xffffc90000977ef0(MAIN-Thread) value=6208672142529 interval=0 mode=0x1 rt-task1 2739 [001] 6208.695933: cobalt_core:cobalt_schedule: status=0x10000000 rt-task1 2739 [001] 6208.695934: cobalt_core:cobalt_trace_pid: pid=2739, prio=86 rt-task1 2739 [001] 6208.695934: cobalt_core:cobalt_switch_context: prev_name=rt-task1 prev_pid=2739 prev_prio=86 prev_state=0x48044 ==> next_name=ROOT/1 next_pid=0 next_prio=-1 rt-task1 2739 [001] 6208.695935: cobalt_core:cobalt_timer_stop: timer=0xffff8884700b6828 swapper 0 [001] 6208.695935: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [002] 6208.695937: irq:irq_pipeline_entry: irq=33027 swapper 0 [002] 6208.695937: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [002] 6208.695937: cobalt_core:cobalt_timer_expire: timer=0xffffc9000096e0f0 swapper 0 [002] 6208.695938: cobalt_core:cobalt_thread_resume: name=rt-task5 pid=2764 mask=0x4 swapper 0 [002] 6208.695938: cobalt_core:cobalt_trace_pid: pid=2764, prio=92 swapper 0 [002] 6208.695938: cobalt_core:cobalt_timer_stop: timer=0xffffc9000096e0f0 swapper 0 [002] 6208.695939: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [002] 6208.695939: irq:irq_pipeline_exit: irq=33027 swapper 0 [002] 6208.695939: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [002] 6208.695940: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [002] 6208.695940: cobalt_core:cobalt_switch_context: prev_name=ROOT/2 prev_pid=0 prev_prio=-1 prev_state=0x18008 ==> next_name=rt-task5 next_pid=2764 next_prio=92 swapper 0 [002] 6208.695940: cobalt_core:cobalt_timer_start: timer=0xffff888470136828([watchdog]) value=1000000000 interval=0 mode=0x0 swapper 0 [000] 6208.695941: irq:irq_pipeline_entry: irq=33027 rt-task5 2764 [002] 6208.695941: cobalt_core:cobalt_trace_pid: pid=2764, prio=92 swapper 0 [000] 6208.695941: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [000] 6208.695942: cobalt_core:cobalt_timer_expire: timer=0xffff888470036298 swapper 0 [000] 6208.695942: irq:irq_handler_exit: irq=33027 ret=handled rt-task5 2764 [002] 6208.695942: cobalt_core:cobalt_thread_suspend: pid=2764 mask=0x4 timeout=6208672301501 timeout_mode=1 wchan=(nil) swapper 0 [000] 6208.695942: irq:irq_pipeline_exit: irq=33027 rt-task5 2764 [002] 6208.695942: cobalt_core:cobalt_timer_start: timer=0xffffc9000096e0f0(MAIN-Thread) value=6208672301501 interval=0 mode=0x1 swapper 0 [000] 6208.695943: irq:irq_handler_entry: irq=27 name=proxy tick rt-task5 2764 [002] 6208.695943: cobalt_core:cobalt_schedule: status=0x10000000 rt-task5 2764 [002] 6208.695943: cobalt_core:cobalt_trace_pid: pid=2764, prio=92 rt-task5 2764 [002] 6208.695943: cobalt_core:cobalt_switch_context: prev_name=rt-task5 prev_pid=2764 prev_prio=92 prev_state=0x48044 ==> next_name=ROOT/2 next_pid=0 next_prio=-1 rt-task5 2764 [002] 6208.695944: cobalt_core:cobalt_timer_stop: timer=0xffff888470136828 swapper 0 [002] 6208.695944: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [000] 6208.695948: cobalt_core:cobalt_timer_start: timer=0xffff888470036298([host-timer/0]) value=107378 interval=0 mode=0x0 swapper 0 [000] 6208.695948: irq:irq_handler_exit: irq=27 ret=handled swapper 0 [000] 6208.708726: irq:irq_pipeline_entry: irq=33027 swapper 0 [001] 6208.708726: irq:irq_pipeline_entry: irq=33027 swapper 0 [002] 6208.708726: irq:irq_pipeline_entry: irq=33027 swapper 0 [000] 6208.708726: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [003] 6208.708727: irq:irq_pipeline_entry: irq=33027 swapper 0 [002] 6208.708727: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [000] 6208.708727: cobalt_core:cobalt_timer_expire: timer=0xffff888470036298 swapper 0 [001] 6208.708727: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [003] 6208.708727: irq:irq_handler_entry: irq=33027 name=Out-of-band LAPIC timer interrupt swapper 0 [000] 6208.708727: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [002] 6208.708727: cobalt_core:cobalt_timer_expire: timer=0xffffc900009744f0 swapper 0 [000] 6208.708728: irq:irq_pipeline_exit: irq=33027 swapper 0 [002] 6208.708728: cobalt_core:cobalt_thread_resume: name=rt-task8 pid=2750 mask=0x4 swapper 0 [002] 6208.708728: cobalt_core:cobalt_trace_pid: pid=2750, prio=85 swapper 0 [000] 6208.708728: irq:irq_handler_entry: irq=27 name=proxy tick swapper 0 [002] 6208.708728: cobalt_core:cobalt_timer_stop: timer=0xffffc900009744f0 swapper 0 [002] 6208.708729: cobalt_core:cobalt_timer_expire: timer=0xffffc900009736f0 swapper 0 [002] 6208.708729: cobalt_core:cobalt_thread_resume: name=rt-task7 pid=2753 mask=0x4 swapper 0 [002] 6208.708729: cobalt_core:cobalt_trace_pid: pid=2753, prio=85 swapper 0 [002] 6208.708729: cobalt_core:cobalt_timer_stop: timer=0xffffc900009736f0 swapper 0 [002] 6208.708730: cobalt_core:cobalt_timer_expire: timer=0xffffc9000096eef0 swapper 0 [002] 6208.708730: cobalt_core:cobalt_thread_resume: name=rt-task2 pid=2762 mask=0x4 swapper 0 [002] 6208.708730: cobalt_core:cobalt_trace_pid: pid=2762, prio=85 swapper 0 [002] 6208.708730: cobalt_core:cobalt_timer_stop: timer=0xffffc9000096eef0 swapper 0 [002] 6208.708731: cobalt_core:cobalt_timer_expire: timer=0xffffc9000096fcf0 swapper 0 [002] 6208.708731: cobalt_core:cobalt_thread_resume: name=rt-task6 pid=2761 mask=0x4 swapper 0 [002] 6208.708731: cobalt_core:cobalt_trace_pid: pid=2761, prio=92 swapper 0 [002] 6208.708731: cobalt_core:cobalt_timer_stop: timer=0xffffc9000096fcf0 swapper 0 [002] 6208.708732: cobalt_core:cobalt_timer_expire: timer=0xffffc9000096e0f0 swapper 0 [002] 6208.708732: cobalt_core:cobalt_thread_resume: name=rt-task5 pid=2764 mask=0x4 swapper 0 [002] 6208.708732: cobalt_core:cobalt_trace_pid: pid=2764, prio=92 swapper 0 [002] 6208.708733: cobalt_core:cobalt_timer_stop: timer=0xffffc9000096e0f0 swapper 0 [002] 6208.708733: cobalt_core:cobalt_timer_expire: timer=0xffff888470136298 swapper 0 [002] 6208.708733: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [001] 6208.708733: cobalt_core:cobalt_timer_expire: timer=0xffff8884700b6298 swapper 0 [002] 6208.708734: irq:irq_pipeline_exit: irq=33027 swapper 0 [001] 6208.708734: cobalt_core:cobalt_timer_expire: timer=0xffffc90000979cf0 swapper 0 [002] 6208.708734: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [001] 6208.708734: cobalt_core:cobalt_thread_resume: name=rt-task3 pid=2737 mask=0x4 swapper 0 [001] 6208.708735: cobalt_core:cobalt_trace_pid: pid=2737, prio=91 swapper 0 [001] 6208.708735: cobalt_core:cobalt_timer_stop: timer=0xffffc90000979cf0 swapper 0 [001] 6208.708735: cobalt_core:cobalt_synch_forget: synch=0xffffc9000097ab08 swapper 0 [001] 6208.708735: cobalt_core:cobalt_timer_expire: timer=0xffffc90000978ef0 swapper 0 [001] 6208.708736: cobalt_core:cobalt_thread_resume: name=rt-task474 pid=2738 mask=0x4 swapper 0 [001] 6208.708736: cobalt_core:cobalt_trace_pid: pid=2738, prio=85 swapper 0 [001] 6208.708736: cobalt_core:cobalt_timer_stop: timer=0xffffc90000978ef0 swapper 0 [001] 6208.708737: cobalt_core:cobalt_timer_expire: timer=0xffffc90000977ef0 swapper 0 [001] 6208.708737: cobalt_core:cobalt_thread_resume: name=rt-task1 pid=2739 mask=0x4 swapper 0 [001] 6208.708737: cobalt_core:cobalt_trace_pid: pid=2739, prio=86 swapper 0 [001] 6208.708737: cobalt_core:cobalt_timer_stop: timer=0xffffc90000977ef0 swapper 0 [001] 6208.708738: cobalt_core:cobalt_timer_expire: timer=0xffffffffa0a556b8 swapper 0 [001] 6208.708738: cobalt_core:cobalt_thread_resume: name=kernel-drv-task10 pid=2456 mask=0x4 swapper 0 [001] 6208.708738: cobalt_core:cobalt_trace_pid: pid=2456, prio=94 swapper 0 [001] 6208.708738: cobalt_core:cobalt_timer_stop: timer=0xffffffffa0a556b8 swapper 0 [001] 6208.708739: cobalt_core:cobalt_synch_forget: synch=0xffffffffa0a559e8 swapper 0 [001] 6208.708739: cobalt_core:cobalt_timer_expire: timer=0xffffc900009770f0 swapper 0 [001] 6208.708739: cobalt_core:cobalt_thread_resume: name=1ms-polling-task pid=2740 mask=0x4 swapper 0 [001] 6208.708740: cobalt_core:cobalt_trace_pid: pid=2740, prio=93 swapper 0 [001] 6208.708740: cobalt_core:cobalt_timer_stop: timer=0xffffc900009770f0 swapper 0 [001] 6208.708740: cobalt_core:cobalt_timer_expire: timer=0xffffc900009752f0 swapper 0 [001] 6208.708740: cobalt_core:cobalt_thread_resume: name=rt-task20 pid=2742 mask=0x4 swapper 0 [001] 6208.708741: cobalt_core:cobalt_trace_pid: pid=2742, prio=93 swapper 0 [001] 6208.708741: cobalt_core:cobalt_timer_stop: timer=0xffffc900009752f0 swapper 0 [001] 6208.708741: cobalt_core:cobalt_synch_forget: synch=0xffffffffa0a86d80 swapper 0 [001] 6208.708741: cobalt_core:cobalt_timer_expire: timer=0xffffffffa0a86fe0 swapper 0 [001] 6208.708742: cobalt_core:cobalt_thread_resume: name=kernel-drv-rttask11 pid=2462 mask=0x4 swapper 0 [001] 6208.708742: cobalt_core:cobalt_trace_pid: pid=2462, prio=98 swapper 0 [001] 6208.708742: cobalt_core:cobalt_timer_stop: timer=0xffffffffa0a86fe0 swapper 0 [001] 6208.708742: cobalt_core:cobalt_synch_forget: synch=0xffffffffa0a87310 swapper 0 [001] 6208.708743: cobalt_core:cobalt_timer_expire: timer=0xffffc900009760f0 swapper 0 [001] 6208.708743: cobalt_core:cobalt_thread_resume: name=rt-task19 pid=2741 mask=0x4 swapper 0 [001] 6208.708743: cobalt_core:cobalt_trace_pid: pid=2741, prio=93 swapper 0 [001] 6208.708744: cobalt_core:cobalt_timer_stop: timer=0xffffc900009760f0 swapper 0 [001] 6208.708744: cobalt_core:cobalt_synch_forget: synch=0xffffffffa0aabe00 swapper 0 [001] 6208.708744: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [003] 6208.708744: cobalt_core:cobalt_timer_expire: timer=0xffff8884701b6298 swapper 0 [001] 6208.708745: irq:irq_pipeline_exit: irq=33027 swapper 0 [003] 6208.708745: irq:irq_handler_exit: irq=33027 ret=handled swapper 0 [001] 6208.708745: cobalt_core:cobalt_schedule: status=0x10000000 swapper 0 [002] 6208.708745: cobalt_core:cobalt_trace_pid: pid=0, prio=-1 swapper 0 [003] 6208.708745: irq:irq_pipeline_exit: irq=33027 swapper 0 [002] 6208.708746: cobalt_core:cobalt_switch_context: prev_name=ROOT/2 prev_pid=0 prev_prio=-1 prev_state=0x18008 ==> next_name=rt-task6 next_pid=2761 next_prio=92 swapper 0 [002] 6208.708746: cobalt_core:cobalt_timer_start: timer=0xffff888470136828([watchdog]) value=1000000000 interval=0 mode=0x0 swapper 0 [003] 6208.708746: irq:irq_handler_entry: irq=27 name=proxy tick