From mboxrd@z Thu Jan 1 00:00:00 1970 From: Wu Fengguang Subject: Re: [Bug 18632] "INFO: task" dpkg "blocked for more than 120 seconds. Date: Fri, 10 Jun 2011 11:21:49 +0800 Message-ID: <20110610032149.GA7114@localhost> References: <20110608150241.8412a63d.akpm@linux-foundation.org> <20110609033217.GA10741@localhost> <20110609035426.GA12061@localhost> <20110609082718.GA10335@infradead.org> <20110609090906.GA19186@localhost> <20110609110214.GA9017@infradead.org> <20110609121117.GA5768@localhost> <20110609121742.GA29569@infradead.org> <20110609124315.GA8789@localhost> <20110609132357.GA9928@infradead.org> Mime-Version: 1.0 Content-Type: text/plain; charset=us-ascii Cc: Carlos Alberto Lopez Perez , Dave Chinner , Andrew Morton , Jan Kara , "linux-fsdevel@vger.kernel.org" , "bugzilla-daemon@bugzilla.kernel.org" , "daaugusto@gmail.com" , "kernel-bugzilla@cygnusx-1.org" , "listposter@gmail.com" , "justincase@yopmail.com" , Tao Ma , Jens Axboe , Vivek Goyal To: Christoph Hellwig Return-path: Received: from mga01.intel.com ([192.55.52.88]:60941 "EHLO mga01.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751517Ab1FJDVx (ORCPT ); Thu, 9 Jun 2011 23:21:53 -0400 Content-Disposition: inline In-Reply-To: <20110609132357.GA9928@infradead.org> Sender: linux-fsdevel-owner@vger.kernel.org List-ID: On Thu, Jun 09, 2011 at 09:23:57PM +0800, Christoph Hellwig wrote: > Here is a patch that removes the XFS-internal writing of data in sync > and instead trusts the VFS code to get it right, and also does the > ioend_wait under a lock, to protect against new pending I/Os showing up. > > It passed a single xfsqa run so far, so at least it won't blow up > completely. > > Can you check if that helps with your livelock test case? (And yes, I > should go and try it myself. Will do as soon as I'll get a bit more > time). My pleasure. It's easier to compare results on the same test bed and I've made it very convenient to test patches :) The patch does improve the sync time. before patch after patch -------------------------------------------------- sync time: 18 sync time: 18 sync time: 21 sync time: 15 sync time: 14 sync time: 14 sync time: 20 sync time: 16 sync time: 21 sync time: 14 avg 18.800 avg 15.400 stddev 2.638 stddev 1.497 I also did extra test runs and collect sync(1) call traces by doing "echo t > /proc/sysrq-trigger" randomly. The ones in XFS are: [ 78.395664] sync D 0000000000000002 5296 3024 3003 0x00000000 [ 78.396162] ffff8800b4229de8 0000000000000046 ffffffff819252fa 0000000000000000 [ 78.396850] ffff8800b4228000 00000000001d1a40 00000000001d1a40 ffff8800aefa22f0 [ 78.397519] 00000000001d1a40 ffff8800b4229fd8 00000000001d1a40 ffff8800b4229fd8 [ 78.398189] Call Trace: [ 78.398431] [] ? schedule+0x374/0x992 [ 78.398778] [] ? _xfs_log_force+0x1de/0x226 [ 78.399143] [] ? local_clock+0x41/0x5a [ 78.399493] [] ? lock_release_holdtime+0xa3/0xab [ 78.399875] [] ? _xfs_log_force+0x1de/0x226 [ 78.400239] [] _xfs_log_force+0x1e3/0x226 [ 78.400599] [] ? try_to_wake_up+0x28b/0x28b [ 78.400964] [] ? xfs_inode_ag_iterator+0x6d/0x8f [ 78.401347] [] ? __sync_filesystem+0x7a/0x7a [ 78.401715] [] xfs_log_force+0x18/0x3a [ 78.402064] [] xfs_quiesce_data+0x3a/0x96 [ 78.402424] [] xfs_fs_sync_fs+0x21/0x48 [ 78.402777] [] __sync_filesystem+0x66/0x7a [ 78.403138] [] sync_one_sb+0x16/0x18 [ 78.403482] [] iterate_supers+0x72/0xc8 [ 78.403835] [] sync_filesystems+0x20/0x22 [ 78.404198] [] sys_sync+0x21/0x33 [ 78.404533] [] system_call_fastpath+0x16/0x1b [ 221.106751] sync D 0000000000000007 5296 3136 3116 0x00000000 [ 221.107229] ffff8800aee7bb40 0000000000000046 ffffffff819252fa 0000000000000000 [ 221.107903] ffff8800aee7a000 00000000001d1a40 00000000001d1a40 ffff88009c6a45e0 [ 221.108574] 00000000001d1a40 ffff8800aee7bfd8 00000000001d1a40 ffff8800aee7bfd8 [ 221.109246] Call Trace: [ 221.109486] [] ? schedule+0x374/0x992 [ 221.109831] [] ? rwsem_down_failed_common+0xc7/0x103 [ 221.110225] [] ? local_clock+0x41/0x5a [ 221.110574] [] ? lock_release_holdtime+0xa3/0xab [ 221.110956] [] ? rwsem_down_failed_common+0xc7/0x103 [ 221.111352] [] rwsem_down_failed_common+0xd3/0x103 [ 221.111738] [] rwsem_down_read_failed+0x12/0x14 [ 221.112116] [] call_rwsem_down_read_failed+0x14/0x30 [ 221.112513] [] ? xfs_ilock+0x4d/0xba [ 221.112855] [] ? xfs_ilock+0x30/0xba [ 221.113199] [] ? down_read_nested+0x74/0x89 [ 221.113563] [] ? xfs_ilock+0x4d/0xba [ 221.113904] [] xfs_ilock+0x4d/0xba [ 221.114243] [] xfs_wait_ioend_cb+0x1b/0x36 [ 221.114604] [] xfs_inode_ag_walk+0x1a5/0x288 [ 221.114973] [] ? xfs_inode_ag_walk+0x25f/0x288 [ 221.115348] [] ? xfs_sync_inode_data+0x9d/0x9d [ 221.115722] [] ? lock_release_holdtime+0xa3/0xab [ 221.116103] [] ? xfs_check_sizes+0x160/0x160 [ 221.116473] [] ? xfs_perag_get+0x68/0xd1 [ 221.116828] [] ? xfs_perag_get+0x80/0xd1 [ 221.117185] [] ? xfs_check_sizes+0x160/0x160 [ 221.117553] [] ? xfs_inode_ag_iterator+0x6d/0x8f [ 221.117936] [] ? xfs_sync_inode_data+0x9d/0x9d [ 221.118312] [] xfs_inode_ag_iterator+0x47/0x8f [ 221.118689] [] ? __sync_filesystem+0x7a/0x7a [ 221.119058] [] xfs_quiesce_data+0x23/0x96 [ 221.119417] [] xfs_fs_sync_fs+0x21/0x48 [ 221.119771] [] __sync_filesystem+0x66/0x7a [ 221.120134] [] sync_one_sb+0x16/0x18 [ 221.120477] [] iterate_supers+0x72/0xc8 [ 221.120828] [] sync_filesystems+0x20/0x22 [ 221.121188] [] sys_sync+0x21/0x33 [ 349.972430] sync D 0000000000000007 5296 3312 3292 0x00000000 [ 349.972907] ffff88009c77fc08 0000000000000046 ffffffff819252fa 0000000100000070 [ 349.973575] ffff88009c77e000 00000000001d1a40 00000000001d1a40 ffff8800b56b0000 [ 349.974250] 00000000001d1a40 ffff88009c77ffd8 00000000001d1a40 ffff88009c77ffd8 [ 349.974921] Call Trace: [ 349.975159] [] ? schedule+0x374/0x992 [ 349.975504] [] ? lock_release_holdtime+0xa3/0xab [ 349.975880] [] ? prepare_to_wait+0x6c/0x79 [ 349.976238] [] xfs_ioend_wait+0x87/0x9f [ 349.976588] [] ? wake_up_bit+0x2a/0x2a [ 349.976932] [] ? xfs_ilock+0x4d/0xba [ 349.977272] [] xfs_wait_ioend_cb+0x23/0x36 [ 349.977631] [] xfs_inode_ag_walk+0x1a5/0x288 [ 349.977997] [] ? xfs_inode_ag_walk+0x25f/0x288 [ 349.978371] [] ? xfs_sync_inode_data+0x9d/0x9d [ 349.978748] [] ? lock_release_holdtime+0xa3/0xab [ 349.979129] [] ? xfs_check_sizes+0x160/0x160 [ 349.979499] [] ? xfs_perag_get+0x68/0xd1 [ 349.979856] [] ? xfs_perag_get+0x80/0xd1 [ 349.980211] [] ? xfs_check_sizes+0x160/0x160 [ 349.980582] [] ? xfs_inode_ag_iterator+0x6d/0x8f [ 349.980964] [] ? xfs_sync_inode_data+0x9d/0x9d [ 349.981338] [] xfs_inode_ag_iterator+0x47/0x8f [ 349.981717] [] ? __sync_filesystem+0x7a/0x7a [ 349.982085] [] xfs_quiesce_data+0x23/0x96 [ 349.982443] [] xfs_fs_sync_fs+0x21/0x48 [ 349.982798] [] __sync_filesystem+0x66/0x7a [ 349.983159] [] sync_one_sb+0x16/0x18 [ 349.983504] [] iterate_supers+0x72/0xc8 [ 349.983857] [] sync_filesystems+0x20/0x22 [ 349.984215] [] sys_sync+0x21/0x33 [ 349.984552] [] system_call_fastpath+0x16/0x1b [ 419.417496] sync D 0000000000000006 5296 3412 3392 0x00000000 [ 419.417972] ffff8800aef6dc08 0000000000000046 ffffffff819252fa 0000000100000070 [ 419.418645] ffff8800aef6c000 00000000001d1a40 00000000001d1a40 ffff8800b704a2f0 [ 419.419317] 00000000001d1a40 ffff8800aef6dfd8 00000000001d1a40 ffff8800aef6dfd8 [ 419.419987] Call Trace: [ 419.420228] [] ? schedule+0x374/0x992 [ 419.420576] [] ? lock_release_holdtime+0xa3/0xab [ 419.420957] [] ? prepare_to_wait+0x6c/0x79 [ 419.421321] [] xfs_ioend_wait+0x87/0x9f [ 419.421673] [] ? wake_up_bit+0x2a/0x2a [ 419.422020] [] ? xfs_ilock+0x4d/0xba [ 419.422364] [] xfs_wait_ioend_cb+0x23/0x36 [ 419.422724] [] xfs_inode_ag_walk+0x1a5/0x288 [ 419.423093] [] ? xfs_inode_ag_walk+0x25f/0x288 [ 419.423470] [] ? xfs_sync_inode_data+0x9d/0x9d [ 419.423844] [] ? lock_release_holdtime+0xa3/0xab [ 419.424226] [] ? xfs_check_sizes+0x160/0x160 [ 419.424596] [] ? xfs_perag_get+0x68/0xd1 [ 419.424951] [] ? xfs_perag_get+0x80/0xd1 [ 419.425307] [] ? xfs_check_sizes+0x160/0x160 [ 419.425676] [] ? xfs_inode_ag_iterator+0x6d/0x8f [ 419.426058] [] ? xfs_sync_inode_data+0x9d/0x9d [ 419.426435] [] xfs_inode_ag_iterator+0x47/0x8f [ 419.426812] [] ? __sync_filesystem+0x7a/0x7a [ 419.427180] [] xfs_quiesce_data+0x23/0x96 [ 419.427542] [] xfs_fs_sync_fs+0x21/0x48 [ 419.427895] [] __sync_filesystem+0x66/0x7a [ 419.428258] [] sync_one_sb+0x16/0x18 [ 419.428601] [] iterate_supers+0x72/0xc8 [ 419.428954] [] sync_filesystems+0x20/0x22 [ 419.429315] [] sys_sync+0x21/0x33 [ 419.429648] [] system_call_fastpath+0x16/0x1b Thanks, Fengguang