Re: [BUG] reproducable ubifs reboot assert and corruption

[Andrew Ruder <andy@aeruder.net>]

[-- Attachment #1: Type: text/plain, Size: 8899 bytes --]

On Wed, Jan 29, 2014 at 04:56:04PM +0100, Richard Weinberger wrote:
> Does the issue also happen if you disable preemption?
> i.e. CONFIG_PREEMPT_NONE=y

CONFIG_PREEMPT_NONE=y still breaks.  I suspect that sync_filesystem
has some blocking behavior that allows other processes to schedule.

Another log is attached and the patch I used to create this log is
below.  I think the most interesting part of this patch is the last hunk
that modifies ubifs_remount_ro.  This clearly shows that after the mount
has been marked as read-only we have dirty inodes waiting for the
writeback to come in and hit all the asserts.

Here's some of the important parts of the log:

  170 pre sync_filesystem
# Notice that while we were running sync_filesystem
# a inode (0xd75ab658) snuck in with a sys_rename
  204 inode: d75ab658
  205 ------------[ cut here ]------------
  206 WARNING: CPU: 0 PID: 625 at fs/fs-writeback.c:1213 __mark_inode_dirty+0x2a4/0x2f4()
  207 Modules linked in:
  208 CPU: 0 PID: 625 Comm: fsstress Tainted: G        W    3.12.0-00041-g7f12d39-dirty #250
  209 [<c0014f54>] (unwind_backtrace+0x0/0x128) from [<c001235c>] (show_stack+0x20/0x24)
  210 [<c001235c>] (show_stack+0x20/0x24) from [<c0344574>] (dump_stack+0x20/0x28)
  211 [<c0344574>] (dump_stack+0x20/0x28) from [<c00206d4>] (warn_slowpath_common+0x78/0x98)
  212 [<c00206d4>] (warn_slowpath_common+0x78/0x98) from [<c00207b0>] (warn_slowpath_null+0x2c/0x34)
  213 [<c00207b0>] (warn_slowpath_null+0x2c/0x34) from [<c00f0e8c>] (__mark_inode_dirty+0x2a4/0x2f4)
  214 [<c00f0e8c>] (__mark_inode_dirty+0x2a4/0x2f4) from [<c0136428>] (ubifs_rename+0x4a4/0x600)
  215 [<c0136428>] (ubifs_rename+0x4a4/0x600) from [<c00d9f44>] (vfs_rename+0x280/0x3f4)
  216 [<c00d9f44>] (vfs_rename+0x280/0x3f4) from [<c00dabe4>] (SyS_renameat+0x18c/0x218)
  217 [<c00dabe4>] (SyS_renameat+0x18c/0x218) from [<c00dac9c>] (SyS_rename+0x2c/0x30)
  218 [<c00dac9c>] (SyS_rename+0x2c/0x30) from [<c000e820>] (ret_fast_syscall+0x0/0x2c)
  219 ---[ end trace 35ebec8569a53526 ]---
  754 post sync_filesystem
  755 pre prepare_remount
  756 post prepare_remount
  757 prepare_remount success
  758 UBIFS: background thread "ubifs_bgt0_0" stops
  759 we are now a read-only mount
  760 bdi.work_list = d7ac4110, .next = d7ac4110
# WE HAVE DIRTY I/O (notice the .next != &b_dirty)
  761 bdi.wb.b_dirty = d7ac40d8, .next = d75accd0
  762 bdi.wb.b_io = d7ac40e0, .next = d7ac40e0
  763 bdi.wb.b_more_io = d7ac40e8, .next = d7ac40e8
  764 do_remount_sb success
# And now our friend (inode 0xd75ab658) comes in with a writeback after
# we are read-only triggering the assert
  779 inode: d75ab658
  780 UBIFS assert failed in reserve_space at 125 (pid 11)
  781 CPU: 0 PID: 11 Comm: kworker/u2:1 Tainted: G        W    3.12.0-00041-g7f12d39-dirty #250
  782 Workqueue: writeback bdi_writeback_workfn (flush-ubifs_0_0)
  783 [<c0014f54>] (unwind_backtrace+0x0/0x128) from [<c001235c>] (show_stack+0x20/0x24)
  784 [<c001235c>] (show_stack+0x20/0x24) from [<c0344574>] (dump_stack+0x20/0x28)
  785 [<c0344574>] (dump_stack+0x20/0x28) from [<c012f90c>] (make_reservation+0x8c/0x560)
  786 [<c012f90c>] (make_reservation+0x8c/0x560) from [<c0130c60>] (ubifs_jnl_write_inode+0xbc/0x214)
  787 [<c0130c60>] (ubifs_jnl_write_inode+0xbc/0x214) from [<c0137ce0>] (ubifs_write_inode+0xec/0x17c)
  788 [<c0137ce0>] (ubifs_write_inode+0xec/0x17c) from [<c00f0a00>] (__writeback_single_inode+0x1fc/0x308)
  789 [<c00f0a00>] (__writeback_single_inode+0x1fc/0x308) from [<c00f1780>] (writeback_sb_inodes+0x1f8/0x3c4)
  790 [<c00f1780>] (writeback_sb_inodes+0x1f8/0x3c4) from [<c00f19cc>] (__writeback_inodes_wb+0x80/0xc0)
  791 [<c00f19cc>] (__writeback_inodes_wb+0x80/0xc0) from [<c00f1ba4>] (wb_writeback+0x198/0x310)
  792 [<c00f1ba4>] (wb_writeback+0x198/0x310) from [<c00f2328>] (bdi_writeback_workfn+0x15c/0x454)
  793 [<c00f2328>] (bdi_writeback_workfn+0x15c/0x454) from [<c0038348>] (process_one_work+0x280/0x420)
  794 [<c0038348>] (process_one_work+0x280/0x420) from [<c00389e4>] (worker_thread+0x240/0x380)
  795 [<c00389e4>] (worker_thread+0x240/0x380) from [<c003dff8>] (kthread+0xbc/0xc8)
  796 [<c003dff8>] (kthread+0xbc/0xc8) from [<c000e8b0>] (ret_from_fork+0x14/0x20)

- Andy




--- patch ---
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 9f4935b..48e4272 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -93,6 +93,10 @@ static void bdi_queue_work(struct backing_dev_info *bdi,
 {
 	trace_writeback_queue(bdi, work);
 
+	do {
+		extern bool ubifs_enable_debug;
+		WARN_ON(ubifs_enable_debug);
+	} while (0);
 	spin_lock_bh(&bdi->wb_lock);
 	list_add_tail(&work->list, &bdi->work_list);
 	spin_unlock_bh(&bdi->wb_lock);
@@ -194,6 +198,11 @@ static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
 		if (time_before(inode->dirtied_when, tail->dirtied_when))
 			inode->dirtied_when = jiffies;
 	}
+	do {
+		extern bool ubifs_enable_debug;
+		if (ubifs_enable_debug) pr_info("inode: %p\n", inode);
+		WARN_ON(ubifs_enable_debug);
+	} while (0);
 	list_move(&inode->i_wb_list, &wb->b_dirty);
 }
 
@@ -204,6 +213,11 @@ static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
 {
 	assert_spin_locked(&wb->list_lock);
 	list_move(&inode->i_wb_list, &wb->b_more_io);
+	do {
+		extern bool ubifs_enable_debug;
+		if (ubifs_enable_debug) pr_info("inode: %p\n", inode);
+		WARN_ON(ubifs_enable_debug);
+	} while (0);
 }
 
 static void inode_sync_complete(struct inode *inode)
@@ -437,6 +451,8 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
 	unsigned dirty;
 	int ret;
 
+	extern bool ubifs_enable_debug;
+	if (ubifs_enable_debug) pr_info("inode: %p\n", inode);
 	WARN_ON(!(inode->i_state & I_SYNC));
 
 	trace_writeback_single_inode_start(inode, wbc, nr_to_write);
@@ -1191,6 +1207,11 @@ void __mark_inode_dirty(struct inode *inode, int flags)
 
 			inode->dirtied_when = jiffies;
 			list_move(&inode->i_wb_list, &bdi->wb.b_dirty);
+	do {
+		extern bool ubifs_enable_debug;
+		if (ubifs_enable_debug) pr_info("inode: %p\n", inode);
+		WARN_ON(ubifs_enable_debug);
+	} while (0);
 			spin_unlock(&bdi->wb.list_lock);
 
 			if (wakeup_bdi)
diff --git a/fs/super.c b/fs/super.c
index 0225c20..29afc68 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -737,6 +737,12 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force)
 	int retval;
 	int remount_ro;
 
+	extern bool ubifs_enable_debug;
+	if (flags & MS_RDONLY) {
+		ubifs_enable_debug = true;
+	}
+	WARN_ON(ubifs_enable_debug);
+
 	if (sb->s_writers.frozen != SB_UNFROZEN)
 		return -EBUSY;
 
@@ -747,8 +753,11 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force)
 
 	if (flags & MS_RDONLY)
 		acct_auto_close(sb);
+	pr_info("pre shrink_dcache_sb\n");
 	shrink_dcache_sb(sb);
+	pr_info("pre sync_filesystem\n");
 	sync_filesystem(sb);
+	pr_info("post sync_filesystem\n");
 
 	remount_ro = (flags & MS_RDONLY) && !(sb->s_flags & MS_RDONLY);
 
@@ -758,9 +767,12 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force)
 		if (force) {
 			mark_files_ro(sb);
 		} else {
+			pr_info("pre prepare_remount\n");
 			retval = sb_prepare_remount_readonly(sb);
+			pr_info("post prepare_remount\n");
 			if (retval)
 				return retval;
+			pr_info("prepare_remount success\n");
 		}
 	}
 
@@ -789,6 +801,7 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force)
 	 */
 	if (remount_ro && sb->s_bdev)
 		invalidate_bdev(sb->s_bdev);
+	pr_info("do_remount_sb success\n");
 	return 0;
 
 cancel_readonly:
diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c
index 123c79b..c9f2d5d 100644
--- a/fs/ubifs/file.c
+++ b/fs/ubifs/file.c
@@ -902,6 +902,8 @@ static int do_writepage(struct page *page, int len)
 	struct inode *inode = page->mapping->host;
 	struct ubifs_info *c = inode->i_sb->s_fs_info;
 
+	WARN_ON(c->ro_mount);
+
 #ifdef UBIFS_DEBUG
 	spin_lock(&ui->ui_lock);
 	ubifs_assert(page->index <= ui->synced_i_size << PAGE_CACHE_SIZE);
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
index 3e4aa72..8cbb731 100644
--- a/fs/ubifs/super.c
+++ b/fs/ubifs/super.c
@@ -38,6 +38,8 @@
 #include <linux/writeback.h>
 #include "ubifs.h"
 
+bool ubifs_enable_debug;
+
 /*
  * Maximum amount of memory we may 'kmalloc()' without worrying that we are
  * allocating too much.
@@ -1756,6 +1758,11 @@ static void ubifs_remount_ro(struct ubifs_info *c)
 	err = dbg_check_space_info(c);
 	if (err)
 		ubifs_ro_mode(c, err);
+	pr_info("we are now a read-only mount\n");
+	pr_info("bdi.work_list = %p, .next = %p\n", &c->bdi.work_list, c->bdi.work_list.next);
+	pr_info("bdi.wb.b_dirty = %p, .next = %p\n", &c->bdi.wb.b_dirty, c->bdi.wb.b_dirty.next);
+	pr_info("bdi.wb.b_io = %p, .next = %p\n", &c->bdi.wb.b_io, c->bdi.wb.b_io.next);
+	pr_info("bdi.wb.b_more_io = %p, .next = %p\n", &c->bdi.wb.b_more_io, c->bdi.wb.b_more_io.next);
 	mutex_unlock(&c->umount_mutex);
 }
 

[-- Attachment #2: experiments11.small.txt.gz --]
[-- Type: application/x-gunzip, Size: 5974 bytes --]