From mboxrd@z Thu Jan 1 00:00:00 1970 From: Stefan Priebe Subject: Re: FS / Kernel question choosing the correct kernel version Date: Tue, 26 Jun 2012 19:49:52 +0200 Message-ID: <4FE9F640.7060600@profihost.ag> References: <4FE60A65.2030800@profihost.ag> <4FE97BE8.6010603@profihost.ag> <4FE9E006.6030404@profihost.ag> <4FE9E360.8050807@inktank.com> <4FE9E6CB.8080803@profihost.ag> <4FE9EA85.9030809@inktank.com> <4FE9F60E.3020105@profihost.ag> Mime-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-1; format=flowed Content-Transfer-Encoding: 7bit Return-path: Received: from mail.profihost.ag ([85.158.179.208]:48877 "EHLO mail.profihost.ag" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754029Ab2FZRt4 (ORCPT ); Tue, 26 Jun 2012 13:49:56 -0400 In-Reply-To: <4FE9F60E.3020105@profihost.ag> Sender: ceph-devel-owner@vger.kernel.org List-ID: To: Mark Nelson Cc: Sage Weil , "ceph-devel@vger.kernel.org" , "hch@infradead.org >> Christoph Hellwig" and these: 359.919732] INFO: task xfsaild/sdd1:2247 blocked for more than 120 seconds. [ 359.934620] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 359.950059] xfsaild/sdd1 D 0000000000000007 0 2247 2 0x00000000 [ 359.966026] ffff8810349d9cf0 0000000000000046 ffff881034809810 0000000000012200 [ 359.982249] ffff8810349d9fd8 ffff8810349d8010 0000000000012200 0000000000012200 [ 359.998634] ffff8810349d9fd8 0000000000012200 ffff881038ef8000 ffff881034809810 [ 360.015139] Call Trace: [ 360.031552] [] schedule+0x24/0x70 [ 360.048356] [] xlog_cil_force_lsn+0xea/0x110 [ 360.065363] [] ? try_to_wake_up+0x2b0/0x2b0 [ 360.082348] [] ? schedule_timeout+0x189/0x2e0 [ 360.099128] [] _xfs_log_force+0x64/0x240 [ 360.115616] [] xfs_log_force+0x13/0x40 [ 360.132031] [] xfsaild+0x106/0x6d0 [ 360.148296] [] ? xfs_trans_ail_update_bulk+0x260/0x260 [ 360.164963] [] kthread+0x96/0xa0 [ 360.181658] [] kernel_thread_helper+0x4/0x10 [ 360.198516] [] ? kthread_worker_fn+0x130/0x130 [ 360.215534] [] ? gs_change+0xb/0xb [ 480.048246] INFO: task ceph-osd:3282 blocked for more than 120 seconds. [ 480.065939] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 480.083979] ceph-osd D ffffffff8180e9c0 0 3282 1 0x00000004 [ 480.102327] ffff880dbbb9d7b8 0000000000000086 ffff880dbbba0000 0000000000012200 [ 480.121333] ffff880dbbb9dfd8 ffff880dbbb9c010 0000000000012200 0000000000012200 [ 480.140374] ffff880dbbb9dfd8 0000000000012200 ffff881038e69810 ffff880dbbba0000 [ 480.159738] Call Trace: [ 480.178812] [] schedule+0x24/0x70 [ 480.198230] [] schedule_timeout+0x1fd/0x2e0 [ 480.217775] [] ? xfs_iext_bno_to_ext+0x84/0x160 [ 480.237500] [] __down+0x6a/0xb0 [ 480.257254] [] ? _xfs_buf_find+0xe5/0x230 [ 480.277179] [] down+0x3b/0x50 [ 480.297102] [] xfs_buf_lock+0x40/0xe0 [ 480.317000] [] _xfs_buf_find+0xe5/0x230 [ 480.337193] [] xfs_buf_get+0x2f/0x1b0 [ 480.357648] [] ? xfs_icsb_modify_counters+0xa2/0x190 [ 480.378666] [] xfs_buf_read+0x27/0x100 [ 480.399694] [] xfs_trans_read_buf+0x1f7/0x430 [ 480.421093] [] xfs_read_agf+0x61/0x190 [ 480.442728] [] ? kmem_zone_alloc+0x77/0xf0 [ 480.464717] [] xfs_alloc_read_agf+0x34/0xd0 [ 480.486433] [] xfs_alloc_fix_freelist+0x427/0x4a0 [ 480.508220] [] ? xfs_bmap_del_extent+0x45f/0xac0 [ 480.530335] [] ? kmem_cache_free+0x1b/0xf0 [ 480.552300] [] ? xfs_trans_free_item_desc+0x2e/0x30 [ 480.574439] [] ? xfs_perag_get+0x33/0xb0 [ 480.596523] [] ? xfs_free_extent+0x97/0x140 [ 480.618126] [] xfs_free_extent+0xab/0x140 [ 480.639105] [] xfs_bmap_finish+0x15d/0x1a0 [ 480.659636] [] xfs_itruncate_extents+0xf7/0x2a0 [ 480.659645] [] xfs_setattr_size+0x394/0x410 [ 480.659647] [] xfs_vn_setattr+0x32/0x40 [ 480.659650] [] notify_change+0x13a/0x2e0 [ 480.659652] [] do_truncate+0x5f/0x90 [ 480.659654] [] sys_truncate+0x12f/0x150 [ 480.659657] [] system_call_fastpath+0x16/0x1b [ 480.659659] INFO: task ceph-osd:3283 blocked for more than 120 seconds. [ 480.659659] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 480.659661] ceph-osd D 0000000000000001 0 3283 1 0x00000004 [ 480.659663] ffff880dbbb9f678 0000000000000086 ffff880dbbba1810 0000000000012200 [ 480.659666] ffff880dbbb9ffd8 ffff880dbbb9e010 0000000000012200 0000000000012200 [ 480.659669] ffff880dbbb9ffd8 0000000000012200 ffff881036ab1810 ffff880dbbba1810 [ 480.659671] Call Trace: [ 480.659674] [] schedule+0x24/0x70 [ 480.659676] [] schedule_timeout+0x1fd/0x2e0 [ 480.659679] [] ? T.1708+0x44/0x50 [ 480.659681] [] ? try_to_wake_up+0x23f/0x2b0 [ 480.659684] [] wait_for_common+0xc4/0x160 [ 480.659686] [] ? try_to_wake_up+0x2b0/0x2b0 [ 480.659688] [] wait_for_completion+0x18/0x20 [ 480.659691] [] xfs_alloc_vextent+0x6d/0x80 [ 480.659693] [] xfs_bmap_btalloc+0x2a1/0x7d0 [ 480.659695] [] ? check_preempt_curr+0x75/0xa0 [ 480.659698] [] ? __xfs_alloc_vextent+0x5f0/0x5f0 [ 480.659700] [] xfs_bmap_alloc+0x9/0x10 [ 480.659702] [] xfs_bmapi_allocate+0xc8/0x2c0 [ 480.659704] [] xfs_bmapi_write+0x509/0x780 [ 480.659707] [] xfs_iomap_write_allocate+0x134/0x370 [ 480.659709] [] xfs_map_blocks+0x155/0x230 [ 480.659711] [] xfs_vm_writepage+0x197/0x540 [ 480.659713] [] ? xfs_trans_free_item_desc+0x2e/0x30 [ 480.659717] [] __writepage+0x12/0x40 [ 480.659720] [] write_cache_pages+0x236/0x4c0 [ 480.659722] [] ? xfs_log_commit_cil+0x3bf/0x480 [ 480.659724] [] ? set_page_dirty+0x70/0x70 [ 480.659726] [] ? xfs_trans_free+0x5c/0x70 [ 480.659728] [] ? xfs_trans_commit+0xab/0x260 [ 480.659731] [] ? xfs_iunlock+0x7e/0xd0 [ 480.659733] [] generic_writepages+0x4c/0x70 [ 480.659735] [] xfs_vm_writepages+0x4f/0x60 [ 480.659737] [] do_writepages+0x1c/0x40 [ 480.659739] [] __filemap_fdatawrite_range+0x5b/0x60 [ 480.659742] [] filemap_fdatawrite_range+0xe/0x10 [ 480.659745] [] sys_sync_file_range+0x14e/0x180 [ 480.659747] [] ? filp_close+0x60/0x90 [ 480.659749] [] system_call_fastpath+0x16/0x1b [ 480.659756] INFO: task flush-8:64:3184 blocked for more than 120 seconds. [ 480.659757] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 480.659758] flush-8:64 D ffffffff8180e9c0 0 3184 2 0x00000000 [ 480.659761] ffff880dbcf31920 0000000000000046 ffff880dbdbcc830 0000000000012200 [ 480.659763] ffff880dbcf31fd8 ffff880dbcf30010 0000000000012200 0000000000012200 [ 480.659766] ffff880dbcf31fd8 0000000000012200 ffff881038e89810 ffff880dbdbcc830 [ 480.659768] Call Trace: [ 480.659772] [] ? __lock_page+0x70/0x70 [ 480.659774] [] schedule+0x24/0x70 [ 480.659777] [] io_schedule+0x87/0xd0 [ 480.659779] [] sleep_on_page+0x9/0x10 [ 480.659781] [] __wait_on_bit_lock+0x52/0xb0 [ 480.659783] [] ? find_get_pages_tag+0xcd/0x180 [ 480.659785] [] __lock_page+0x62/0x70 [ 480.659789] [] ? autoremove_wake_function+0x40/0x40 [ 480.659791] [] ? pagevec_lookup_tag+0x20/0x30 [ 480.659793] [] write_cache_pages+0x376/0x4c0 [ 480.659796] [] ? set_page_dirty+0x70/0x70 [ 480.659798] [] generic_writepages+0x4c/0x70 [ 480.659800] [] xfs_vm_writepages+0x4f/0x60 [ 480.659802] [] do_writepages+0x1c/0x40 [ 480.659805] [] writeback_single_inode+0x18f/0x400 [ 480.659807] [] writeback_sb_inodes+0x196/0x260 [ 480.659809] [] __writeback_inodes_wb+0x96/0xc0 [ 480.659811] [] wb_writeback+0x1eb/0x2c0 [ 480.659814] [] ? try_to_del_timer_sync+0x81/0xe0 [ 480.659816] [] wb_do_writeback+0x1f9/0x250 [ 480.659818] [] bdi_writeback_thread+0x9a/0x230 [ 480.659820] [] ? wb_do_writeback+0x250/0x250 [ 480.659822] [] ? wb_do_writeback+0x250/0x250 [ 480.659824] [] kthread+0x96/0xa0 [ 480.659827] [] kernel_thread_helper+0x4/0x10 [ 480.659829] [] ? kthread_worker_fn+0x130/0x130 [ 480.659832] [] ? gs_change+0xb/0xb Am 26.06.2012 19:49, schrieb Stefan Priebe: > I would but both XFS and btrfs are crashing after a short period. > > XFS crashes with this one: > [ 479.732636] INFO: task ceph-osd:3217 blocked for more than 120 seconds. > [ 479.747724] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" > disables this message. > [ 479.763534] ceph-osd D ffffffff8180e9c0 0 3217 1 > 0x00000000 > [ 479.779837] ffff880bc4321bd8 0000000000000082 ffff880bc5694830 > 0000000000012200 > [ 479.779840] ffff880bc4321fd8 ffff880bc4320010 0000000000012200 > 0000000000012200 > [ 479.779841] ffff880bc4321fd8 0000000000012200 ffff880e40ea9810 > ffff880bc5694830 > [ 479.779843] Call Trace: > [ 479.779850] [] schedule+0x24/0x70 > [ 479.779853] [] xlog_wait+0x69/0x90 > [ 479.779856] [] ? try_to_wake_up+0x2b0/0x2b0 > [ 479.779858] [] xlog_cil_push+0x343/0x3c0 > [ 479.779861] [] ? xfs_buf_unlock+0x19/0x70 > [ 479.779862] [] xlog_cil_force_lsn+0x101/0x110 > [ 479.779864] [] ? xfs_trans_free_item_desc+0x2e/0x30 > [ 479.779865] [] ? xfs_trans_free_items+0x87/0xb0 > [ 479.779867] [] _xfs_log_force_lsn+0x48/0x290 > [ 479.779871] [] ? kmem_cache_free+0x1b/0xf0 > [ 479.779872] [] xfs_trans_commit+0x24b/0x260 > [ 479.779875] [] xfs_fs_log_dummy+0x5d/0x90 > [ 479.779877] [] ? xfs_log_need_covered+0x7c/0xc0 > [ 479.779879] [] xfs_quiesce_data+0x88/0x90 > [ 479.779881] [] xfs_fs_sync_fs+0x28/0x60 > [ 479.779884] [] __sync_filesystem+0x5e/0x90 > [ 479.779885] [] sync_filesystem+0x43/0x60 > [ 479.779887] [] sys_syncfs+0x48/0x80 > [ 479.779890] [] system_call_fastpath+0x16/0x1b > > > Am 26.06.2012 18:59, schrieb Mark Nelson: >> On 06/26/2012 11:43 AM, Stefan Priebe wrote: >>> Am 26.06.2012 18:29, schrieb Mark Nelson: >>>> On 06/26/2012 11:15 AM, Stefan Priebe wrote: >>>> Hi Stefan, >>>> >>>> If you can, it would be really interesting to see the blktrace results >>>> during these tests for both xfs and btrfs. blktrace is in the ubuntu >>>> repositories and can be run quite easily from the command line during >>>> your test. >>> >>> Sure any special parameters? I have 4 SSDs per OSD Server. >>> >>> Or just blktrace -o file? >>> >>> Stefan >> >> For each device you run it on you'll get one file per core. There may >> be some performance impact if you run blktrace on every device per node. >> If your data is well distributed, even a trace for one OSD (per test) >> would be interesting. >> >> so blktrace -o -d where device is your first >> OSD or something. If you can do it for both btrfs and xfs and maybe run >> each test for a couple of minutes that might be enough. >> >> Mark >> -- >> To unsubscribe from this list: send the line "unsubscribe ceph-devel" in >> the body of a message to majordomo@vger.kernel.org >> More majordomo info at http://vger.kernel.org/majordomo-info.html > > -- > To unsubscribe from this list: send the line "unsubscribe ceph-devel" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html