From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from sentry-two.sandia.gov ([132.175.109.14]:44330 "EHLO sentry-two.sandia.gov" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753471Ab2IYT1N (ORCPT ); Tue, 25 Sep 2012 15:27:13 -0400 Message-ID: <50620147.4060106@sandia.gov> Date: Tue, 25 Sep 2012 13:08:55 -0600 From: "Jim Schutt" MIME-Version: 1.0 To: linux-btrfs@vger.kernel.org cc: "Josef Bacik" Subject: various processes blocked in btrfs_tree_read_lock for more than 120 secs Content-Type: text/plain; charset=utf-8; format=flowed Sender: linux-btrfs-owner@vger.kernel.org List-ID: Hi, I'm hitting the following on a btrfs filesystem used as a Ceph OSD data store, under a heavy write load. My kernel is current Linus master (commit 56d27adcb536) merged with Josef Bacik's btrfs-next master (commit d5b04fb3bbb6). What can I do to help resolve this? [ 1558.754105] INFO: task btrfs-endio-wri:28012 blocked for more than 120 seconds. [ 1558.761569] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1558.769575] btrfs-endio-wri D ffffffff8160b9c0 0 28012 2 0x00000000 [ 1558.777063] ffff8806230b39c0 0000000000000046 0000000000000002 ffff8806230b3fd8 [ 1558.785725] ffff8806230b2010 ffff8806230b2000 ffff8806230b2000 ffff8806230b2000 [ 1558.793776] ffff8806230b3fd8 ffff8806230b2000 ffff8806245a9740 ffff880615825d00 [ 1558.802396] Call Trace: [ 1558.805018] [] schedule+0x5d/0x60 [ 1558.810189] [] btrfs_tree_read_lock+0xf3/0x140 [btrfs] [ 1558.817223] [] ? wake_up_bit+0x40/0x40 [ 1558.822851] [] btrfs_read_lock_root_node+0x23/0x50 [btrfs] [ 1558.830150] [] btrfs_search_slot+0x1f0/0x730 [btrfs] [ 1558.837722] [] btrfs_lookup_file_extent+0x3d/0x40 [btrfs] [ 1558.845099] [] __btrfs_drop_extents+0x171/0xad0 [btrfs] [ 1558.852506] [] ? kmem_cache_alloc+0xcb/0x160 [ 1558.858602] [] btrfs_drop_extents+0x73/0xa0 [btrfs] [ 1558.865457] [] insert_reserved_file_extent.clone.0+0x80/0x2a0 [btrfs] [ 1558.873999] [] ? start_transaction+0x3cb/0x450 [btrfs] [ 1558.881239] [] btrfs_finish_ordered_io+0x339/0x4d0 [btrfs] [ 1558.888608] [] finish_ordered_fn+0x15/0x20 [btrfs] [ 1558.895352] [] worker_loop+0x1a2/0x400 [btrfs] [ 1558.901753] [] ? check_pending_worker_creates+0xe0/0xe0 [btrfs] [ 1558.910128] [] ? check_pending_worker_creates+0xe0/0xe0 [btrfs] [ 1558.918228] [] kthread+0x96/0xa0 [ 1558.923339] [] kernel_thread_helper+0x4/0x10 [ 1558.929821] [] ? retint_restore_args+0xe/0xe [ 1558.935920] [] ? __init_kthread_worker+0x40/0x40 [ 1558.942694] [] ? gs_change+0xb/0xb [ 1558.948170] INFO: task flush-btrfs-21:31685 blocked for more than 120 seconds. [ 1558.955408] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1558.963274] flush-btrfs-21 D ffffffff8160b9c0 0 31685 2 0x00000000 [ 1558.970602] ffff8806074d3470 0000000000000046 0000000000000002 ffff8806074d3fd8 [ 1558.978333] ffff8806074d2010 ffff8806074d2000 ffff8806074d2000 ffff8806074d2000 [ 1558.986052] ffff8806074d3fd8 ffff8806074d2000 ffff8806245aae80 ffff8804cf048000 [ 1558.993744] Call Trace: [ 1558.996238] [] schedule+0x5d/0x60 [ 1559.001243] [] btrfs_tree_read_lock+0xf3/0x140 [btrfs] [ 1559.008057] [] ? wake_up_bit+0x40/0x40 [ 1559.013554] [] btrfs_read_lock_root_node+0x23/0x50 [btrfs] [ 1559.020788] [] btrfs_search_slot+0x1f0/0x730 [btrfs] [ 1559.027451] [] ? free_extent_map+0x87/0x90 [btrfs] [ 1559.033957] [] btrfs_lookup_file_extent+0x3d/0x40 [btrfs] [ 1559.041062] [] __btrfs_drop_extents+0x171/0xad0 [btrfs] [ 1559.047977] [] ? kmem_cache_alloc+0xcb/0x160 [ 1559.053948] [] btrfs_drop_extents+0x73/0xa0 [btrfs] [ 1559.060698] [] cow_file_range_inline+0xe5/0x1c0 [btrfs] [ 1559.067807] [] cow_file_range+0x172/0x4a0 [btrfs] [ 1559.074296] [] run_delalloc_range+0x7b/0xa0 [btrfs] [ 1559.080967] [] __extent_writepage+0x22d/0x740 [btrfs] [ 1559.087868] [] ? btrfs_add_delayed_iput+0x77/0xe0 [btrfs] [ 1559.095034] [] ? find_get_pages_tag+0x148/0x190 [ 1559.101329] [] extent_write_cache_pages.clone.3+0x242/0x3d0 [btrfs] [ 1559.109465] [] extent_writepages+0x47/0x60 [btrfs] [ 1559.116008] [] ? btrfs_update_time+0xb0/0xb0 [btrfs] [ 1559.122716] [] btrfs_writepages+0x27/0x30 [btrfs] [ 1559.129211] [] do_writepages+0x23/0x40 [ 1559.134701] [] __writeback_single_inode+0x4b/0x180 [ 1559.141198] [] ? bit_waitqueue+0x17/0xc0 [ 1559.147201] [] writeback_sb_inodes+0x286/0x390 [ 1559.153365] [] __writeback_inodes_wb+0x86/0xd0 [ 1559.159509] [] wb_writeback+0x18b/0x320 [ 1559.165082] [] wb_do_writeback+0x1f4/0x290 [ 1559.170988] [] ? schedule_timeout+0x1d2/0x240 [ 1559.177249] [] bdi_writeback_thread+0xe3/0x2c0 [ 1559.183685] [] ? wb_do_writeback+0x290/0x290 [ 1559.189740] [] ? wb_do_writeback+0x290/0x290 [ 1559.195878] [] kthread+0x96/0xa0 [ 1559.200874] [] kernel_thread_helper+0x4/0x10 [ 1559.207540] [] ? retint_restore_args+0xe/0xe [ 1559.213718] [] ? __init_kthread_worker+0x40/0x40 [ 1559.220087] [] ? gs_change+0xb/0xb [ 1559.226637] INFO: task ceph-osd:38318 blocked for more than 120 seconds. [ 1559.233338] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1559.241157] ceph-osd D 000000000000000e 0 38318 1 0x00000000 [ 1559.248729] ffff880320e499c8 0000000000000086 0000000000000002 ffff880320e49fd8 [ 1559.256282] ffff880320e48010 ffff880320e48000 ffff880320e48000 ffff880320e48000 [ 1559.263784] ffff880320e49fd8 ffff880320e48000 ffff88032a7fc5c0 ffff880320e40000 [ 1559.271254] Call Trace: [ 1559.273783] [] schedule+0x5d/0x60 [ 1559.278805] [] btrfs_tree_read_lock+0xf3/0x140 [btrfs] [ 1559.285636] [] ? wake_up_bit+0x40/0x40 [ 1559.291082] [] btrfs_read_lock_root_node+0x23/0x50 [btrfs] [ 1559.298278] [] btrfs_search_slot+0x1f0/0x730 [btrfs] [ 1559.304910] [] btrfs_lookup_dir_item+0x7d/0xd0 [btrfs] [ 1559.311720] [] btrfs_lookup_dentry+0xe4/0x360 [btrfs] [ 1559.318439] [] btrfs_lookup+0x1d/0x70 [btrfs] [ 1559.324543] [] ? lookup_dcache+0x9b/0xd0 [ 1559.330117] [] lookup_real+0x36/0x60 [ 1559.335352] [] __lookup_hash+0x38/0x50 [ 1559.340761] [] lookup_slow+0x62/0xe0 [ 1559.345990] [] path_lookupat+0xf7/0x6c0 [ 1559.351494] [] ? kmem_cache_alloc+0x4f/0x160 [ 1559.357465] [] do_path_lookup+0x31/0xd0 [ 1559.362990] [] user_path_at_empty+0x6c/0xb0 [ 1559.368871] [] ? cp_new_stat+0x129/0x140 [ 1559.374551] [] user_path_at+0x11/0x20 [ 1559.379855] [] vfs_fstatat+0x47/0x80 [ 1559.385077] [] vfs_stat+0x1b/0x20 [ 1559.390086] [] sys_newstat+0x24/0x50 [ 1559.395415] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1559.401871] [] ? putname+0x33/0x50 [ 1559.406939] [] system_call_fastpath+0x16/0x1b [ 1559.412949] INFO: task ceph-osd:38320 blocked for more than 120 seconds. [ 1559.419648] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1559.427471] ceph-osd D ffffffff8160b9c0 0 38320 1 0x00000000 [ 1559.434574] ffff880320e4d578 0000000000000086 ffff880320e4d538 ffff880320e4dfd8 [ 1559.442059] ffff880320e4c010 ffff880320e4c000 ffff880320e4c000 ffff880320e4c000 [ 1559.449571] ffff880320e4dfd8 ffff880320e4c000 ffff8806244d5d00 ffff880320e42e80 [ 1559.457108] Call Trace: [ 1559.459591] [] schedule+0x5d/0x60 [ 1559.464589] [] schedule_preempt_disabled+0x27/0x40 [ 1559.471055] [] __mutex_lock_common+0x205/0x360 [ 1559.477175] [] __mutex_lock_slowpath+0x1b/0x20 [ 1559.483291] [] mutex_lock+0x31/0x50 [ 1559.488489] [] do_chunk_alloc+0x1c4/0x300 [btrfs] [ 1559.494872] [] find_free_extent+0x9fc/0xb30 [btrfs] [ 1559.501439] [] btrfs_reserve_extent+0x82/0x190 [btrfs] [ 1559.508279] [] btrfs_alloc_free_block+0x85/0x230 [btrfs] [ 1559.515272] [] ? clear_extent_bit+0x352/0x3d0 [btrfs] [ 1559.522000] [] __btrfs_cow_block+0x14a/0x4b0 [btrfs] [ 1559.528611] [] ? _raw_spin_unlock+0xe/0x20 [ 1559.534376] [] btrfs_cow_block+0x18d/0x230 [btrfs] [ 1559.540852] [] btrfs_search_slot+0x360/0x730 [btrfs] [ 1559.547493] [] btrfs_insert_empty_items+0x8d/0x100 [btrfs] [ 1559.554637] [] ? can_overcommit+0x8b/0xf0 [btrfs] [ 1559.560995] [] insert_with_overflow+0x43/0x120 [btrfs] [ 1559.567782] [] btrfs_insert_xattr_item+0xb6/0x1d0 [btrfs] [ 1559.574821] [] ? kmem_cache_alloc+0xcb/0x160 [ 1559.580754] [] do_setxattr+0x17a/0x240 [btrfs] [ 1559.586857] [] __btrfs_setxattr+0x8b/0x110 [btrfs] [ 1559.593309] [] btrfs_setxattr+0xa7/0xc0 [btrfs] [ 1559.599485] [] __vfs_setxattr_noperm+0x7b/0x150 [ 1559.605665] [] vfs_setxattr+0x8c/0xc0 [ 1559.610975] [] setxattr+0x19e/0x210 [ 1559.616118] [] ? kmem_cache_free+0x20/0x120 [ 1559.621941] [] ? putname+0x33/0x50 [ 1559.626985] [] ? __sb_start_write+0xe3/0x130 [ 1559.632903] [] ? __fput+0x280/0x290 [ 1559.638039] [] sys_setxattr+0x6f/0xb0 [ 1559.643341] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1559.649778] [] system_call_fastpath+0x16/0x1b [ 1559.655773] INFO: task ceph-osd:38324 blocked for more than 120 seconds. [ 1559.662459] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1559.670273] ceph-osd D ffffffff8160b9c0 0 38324 1 0x00000000 [ 1559.677362] ffff880320e6f258 0000000000000086 0000000000000000 ffff880320e6ffd8 [ 1559.684838] ffff880320e6e010 ffff880320e6e000 ffff880320e6e000 ffff880320e6e000 [ 1559.692325] ffff880320e6ffd8 ffff880320e6e000 ffff8806245f1740 ffff880320e71740 [ 1559.699807] Call Trace: [ 1559.702267] [] schedule+0x5d/0x60 [ 1559.707233] [] schedule_preempt_disabled+0x27/0x40 [ 1559.713661] [] __mutex_lock_common+0x205/0x360 [ 1559.719745] [] __mutex_lock_slowpath+0x1b/0x20 [ 1559.725831] [] mutex_lock+0x31/0x50 [ 1559.730979] [] do_chunk_alloc+0x181/0x300 [btrfs] [ 1559.737351] [] find_free_extent+0x9fc/0xb30 [btrfs] [ 1559.743896] [] btrfs_reserve_extent+0x82/0x190 [btrfs] [ 1559.750686] [] btrfs_alloc_free_block+0x85/0x230 [btrfs] [ 1559.757663] [] ? generic_bin_search+0xac/0x170 [btrfs] [ 1559.764477] [] __btrfs_cow_block+0x14a/0x4b0 [btrfs] [ 1559.771148] [] btrfs_cow_block+0x18d/0x230 [btrfs] [ 1559.777604] [] btrfs_search_slot+0x360/0x730 [btrfs] [ 1559.784229] [] btrfs_insert_empty_items+0x8d/0x100 [btrfs] [ 1559.791351] [] ? kmem_cache_alloc+0xcb/0x160 [ 1559.797288] [] btrfs_alloc_dev_extent+0xb5/0x190 [btrfs] [ 1559.804267] [] __btrfs_alloc_chunk+0x6a7/0x750 [btrfs] [ 1559.811076] [] ? free_extent_buffer+0x75/0x90 [btrfs] [ 1559.817842] [] btrfs_alloc_chunk+0x5e/0x90 [btrfs] [ 1559.824324] [] ? check_system_chunk+0x71/0x130 [btrfs] [ 1559.831192] [] do_chunk_alloc+0x28e/0x300 [btrfs] [ 1559.837588] [] find_free_extent+0x9fc/0xb30 [btrfs] [ 1559.844132] [] btrfs_reserve_extent+0x82/0x190 [btrfs] [ 1559.850950] [] cow_file_range+0x296/0x4a0 [btrfs] [ 1559.857350] [] run_delalloc_range+0x7b/0xa0 [btrfs] [ 1559.863935] [] __extent_writepage+0x22d/0x740 [btrfs] [ 1559.870642] [] ? find_get_pages_tag+0x148/0x190 [ 1559.876870] [] extent_write_cache_pages.clone.3+0x242/0x3d0 [btrfs] [ 1559.884865] [] extent_writepages+0x47/0x60 [btrfs] [ 1559.891351] [] ? btrfs_update_time+0xb0/0xb0 [btrfs] [ 1559.897991] [] btrfs_writepages+0x27/0x30 [btrfs] [ 1559.904350] [] do_writepages+0x23/0x40 [ 1559.909793] [] __filemap_fdatawrite_range+0x4e/0x50 [ 1559.916340] [] filemap_fdatawrite_range+0x13/0x20 [ 1559.922701] [] sys_sync_file_range+0x10c/0x170 [ 1559.928825] [] system_call_fastpath+0x16/0x1b [ 1559.934837] INFO: task ceph-osd:38834 blocked for more than 120 seconds. [ 1559.941541] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1559.949360] ceph-osd D 0000000000000013 0 38834 1 0x00000000 [ 1559.956459] ffff880332339bc8 0000000000000086 ffff880c1ae97878 ffff880332339fd8 [ 1559.963954] ffff880332338010 ffff880332338000 ffff880332338000 ffff880332338000 [ 1559.971478] ffff880332339fd8 ffff880332338000 ffff88051c1c0000 ffff88038f535d00 [ 1559.979005] Call Trace: [ 1559.981465] [] schedule+0x5d/0x60 [ 1559.986421] [] schedule_preempt_disabled+0x27/0x40 [ 1559.992869] [] __mutex_lock_common+0x205/0x360 [ 1559.998963] [] __mutex_lock_slowpath+0x1b/0x20 [ 1560.005071] [] mutex_lock+0x31/0x50 [ 1560.010211] [] lookup_slow+0x54/0xe0 [ 1560.015443] [] path_lookupat+0xf7/0x6c0 [ 1560.020956] [] ? kmem_cache_alloc+0x4f/0x160 [ 1560.026904] [] do_path_lookup+0x31/0xd0 [ 1560.032400] [] ? emulate_vsyscall+0x1d9/0x360 [ 1560.038403] [] user_path_at_empty+0x6c/0xb0 [ 1560.044240] [] ? bad_area_nosemaphore+0x13/0x20 [ 1560.050414] [] ? do_page_fault+0x228/0x4a0 [ 1560.056156] [] user_path_at+0x11/0x20 [ 1560.061482] [] vfs_fstatat+0x47/0x80 [ 1560.066719] [] vfs_stat+0x1b/0x20 [ 1560.071690] [] sys_newstat+0x24/0x50 [ 1560.076948] [] ? trace_hardirqs_off_thunk+0x3a/0x6c [ 1560.083470] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1560.089898] [] system_call_fastpath+0x16/0x1b [ 1560.095893] INFO: task ceph-osd:38835 blocked for more than 120 seconds. [ 1560.102581] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1560.110393] ceph-osd D 000000000000000b 0 38835 1 0x00000000 [ 1560.117482] ffff8803672d5bc8 0000000000000086 ffff880c1ae97878 ffff8803672d5fd8 [ 1560.124969] ffff8803672d4010 ffff8803672d4000 ffff8803672d4000 ffff8803672d4000 [ 1560.132457] ffff8803672d5fd8 ffff8803672d4000 ffff88060f092e80 ffff8803672d8000 [ 1560.139940] Call Trace: [ 1560.142401] [] schedule+0x5d/0x60 [ 1560.147368] [] schedule_preempt_disabled+0x27/0x40 [ 1560.153810] [] __mutex_lock_common+0x205/0x360 [ 1560.159929] [] __mutex_lock_slowpath+0x1b/0x20 [ 1560.166015] [] mutex_lock+0x31/0x50 [ 1560.171153] [] lookup_slow+0x54/0xe0 [ 1560.176374] [] path_lookupat+0xf7/0x6c0 [ 1560.181851] [] ? kmem_cache_alloc+0x4f/0x160 [ 1560.187767] [] do_path_lookup+0x31/0xd0 [ 1560.193266] [] user_path_at_empty+0x6c/0xb0 [ 1560.199099] [] user_path_at+0x11/0x20 [ 1560.204415] [] vfs_fstatat+0x47/0x80 [ 1560.209640] [] vfs_stat+0x1b/0x20 [ 1560.214602] [] sys_newstat+0x24/0x50 [ 1560.219819] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1560.226255] [] system_call_fastpath+0x16/0x1b [ 1560.232257] INFO: task ceph-osd:38841 blocked for more than 120 seconds. [ 1560.238945] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1560.246783] ceph-osd D ffffffff8160b9c0 0 38841 1 0x00000000 [ 1560.253897] ffff880610037bc8 0000000000000086 ffff880c1ae97878 ffff880610037fd8 [ 1560.261372] ffff880610036010 ffff880610036000 ffff880610036000 ffff880610036000 [ 1560.268866] ffff880610037fd8 ffff880610036000 ffff8806245cae80 ffff8803672dc5c0 [ 1560.276335] Call Trace: [ 1560.278793] [] schedule+0x5d/0x60 [ 1560.283763] [] schedule_preempt_disabled+0x27/0x40 [ 1560.290204] [] __mutex_lock_common+0x205/0x360 [ 1560.296293] [] __mutex_lock_slowpath+0x1b/0x20 [ 1560.302384] [] mutex_lock+0x31/0x50 [ 1560.307518] [] lookup_slow+0x54/0xe0 [ 1560.312732] [] path_lookupat+0xf7/0x6c0 [ 1560.318226] [] ? getname_flags+0x37/0x100 [ 1560.323886] [] ? kmem_cache_alloc+0x9c/0x160 [ 1560.329815] [] do_path_lookup+0x31/0xd0 [ 1560.335304] [] user_path_at_empty+0x6c/0xb0 [ 1560.341136] [] user_path_at+0x11/0x20 [ 1560.346456] [] vfs_fstatat+0x47/0x80 [ 1560.351694] [] vfs_stat+0x1b/0x20 [ 1560.356686] [] sys_newstat+0x24/0x50 [ 1560.361918] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1560.368371] [] system_call_fastpath+0x16/0x1b [ 1560.374385] INFO: task ceph-osd:38844 blocked for more than 120 seconds. [ 1560.381082] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1560.388911] ceph-osd D ffffffff8160b9c0 0 38844 1 0x00000000 [ 1560.396023] ffff88061003dbc8 0000000000000086 ffff880c1ae97878 ffff88061003dfd8 [ 1560.403509] ffff88061003c010 ffff88061003c000 ffff88061003c000 ffff88061003c000 [ 1560.411003] ffff88061003dfd8 ffff88061003c000 ffff8806245c8000 ffff8803672ddd00 [ 1560.418522] Call Trace: [ 1560.420995] [] schedule+0x5d/0x60 [ 1560.425972] [] schedule_preempt_disabled+0x27/0x40 [ 1560.432421] [] __mutex_lock_common+0x205/0x360 [ 1560.438545] [] __mutex_lock_slowpath+0x1b/0x20 [ 1560.444647] [] mutex_lock+0x31/0x50 [ 1560.449801] [] lookup_slow+0x54/0xe0 [ 1560.455039] [] path_lookupat+0xf7/0x6c0 [ 1560.460542] [] ? kmem_cache_alloc+0x4f/0x160 [ 1560.466463] [] do_path_lookup+0x31/0xd0 [ 1560.471953] [] user_path_at_empty+0x6c/0xb0 [ 1560.477788] [] user_path_at+0x11/0x20 [ 1560.483098] [] vfs_fstatat+0x47/0x80 [ 1560.488330] [] vfs_stat+0x1b/0x20 [ 1560.493296] [] sys_newstat+0x24/0x50 [ 1560.498528] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1560.504962] [] system_call_fastpath+0x16/0x1b [ 1560.510961] INFO: task ceph-osd:38847 blocked for more than 120 seconds. [ 1560.517647] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1560.525458] ceph-osd D 000000000000000b 0 38847 1 0x00000000 [ 1560.532555] ffff88061003fbc8 0000000000000086 ffff880c1ae97878 ffff88061003ffd8 [ 1560.540041] ffff88061003e010 ffff88061003e000 ffff88061003e000 ffff88061003e000 [ 1560.547516] ffff88061003ffd8 ffff88061003e000 ffff880347818000 ffff8803672e0000 [ 1560.554994] Call Trace: [ 1560.557455] [] schedule+0x5d/0x60 [ 1560.562423] [] schedule_preempt_disabled+0x27/0x40 [ 1560.568861] [] __mutex_lock_common+0x205/0x360 [ 1560.574962] [] __mutex_lock_slowpath+0x1b/0x20 [ 1560.581066] [] mutex_lock+0x31/0x50 [ 1560.586229] [] lookup_slow+0x54/0xe0 [ 1560.591482] [] path_lookupat+0xf7/0x6c0 [ 1560.596971] [] ? kmem_cache_alloc+0x4f/0x160 [ 1560.602889] [] do_path_lookup+0x31/0xd0 [ 1560.608379] [] user_path_at_empty+0x6c/0xb0 [ 1560.614217] [] user_path_at+0x11/0x20 [ 1560.619536] [] vfs_fstatat+0x47/0x80 [ 1560.624772] [] vfs_stat+0x1b/0x20 [ 1560.629749] [] sys_newstat+0x24/0x50 [ 1560.634973] [] ? trace_hardirqs_on_thunk+0x3a/0x3c [ 1560.641412] [] system_call_fastpath+0x16/0x1b Thanks -- Jim