[PATCHv1 0/4] fstests: ext4: Add fast_commit related tests

[PATCHv1 0/4] fstests: ext4: Add fast_commit related tests

[Ritesh Harjani]

This series mainly aims at adding some of those test cases for testing
fast_commit. Kernel fix for Patch-1 is still under review [1].

RFC -> PATCHv1:
================
1. Patch-1 mostly remains the same.
2. Added ext4/057 & ext4/058 in the test bucket for fast_commit which are based
   on recent kernel fixes done by Xin Yin.

[RFC]: https://lore.kernel.org/all/b834b83720f61a6aaab11fe20c48b75007be0a46.1643642943.git.riteshh@linux.ibm.com/
[1]: https://lore.kernel.org/all/53596bdf7bd7aed66020db98d903b1653a1dbc7a.1644062450.git.riteshh@linux.ibm.com/

Ritesh Harjani (4):
  ext4/056: Add fast_commit regression test causing data abort exception
  common/punch: Add block_size argument to _filter_fiemap_**
  ext4/057: Add crash test to check unwritten extents tracking with fast_commit
  ext4/058: Add shutdown recovery test with fast_commit

 common/punch       |  12 +++---
 tests/ext4/056     | 101 +++++++++++++++++++++++++++++++++++++++++++++
 tests/ext4/056.out |   3 ++
 tests/ext4/057     |  62 ++++++++++++++++++++++++++++
 tests/ext4/057.out |   6 +++
 tests/ext4/058     |  66 +++++++++++++++++++++++++++++
 tests/ext4/058.out |   7 ++++
 7 files changed, 252 insertions(+), 5 deletions(-)
 create mode 100755 tests/ext4/056
 create mode 100644 tests/ext4/056.out
 create mode 100755 tests/ext4/057
 create mode 100644 tests/ext4/057.out
 create mode 100755 tests/ext4/058
 create mode 100644 tests/ext4/058.out

--
2.31.1

[PATCHv1 1/4] ext4/056: Add fast_commit regression test causing data abort exception

[Ritesh Harjani]

This adds a targeted regression test which could cause data abort
exception to hit on latest kernel with fast_commit enabled.

ext4_mb_mark_bb() does not takes care of block boundary overflow of a given
block group while doing set/clear in buffer_head bitmap during
fast_commit recovery (after a sudden shutdown).

Signed-off-by: Ritesh Harjani <riteshh@linux.ibm.com>
---
 tests/ext4/056     | 101 +++++++++++++++++++++++++++++++++++++++++++++
 tests/ext4/056.out |   3 ++
 2 files changed, 104 insertions(+)
 create mode 100755 tests/ext4/056
 create mode 100644 tests/ext4/056.out

diff --git a/tests/ext4/056 b/tests/ext4/056
new file mode 100755
index 00000000..588e8af8
--- /dev/null
+++ b/tests/ext4/056
@@ -0,0 +1,101 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2022 IBM Corporation.  All Rights Reserved.
+#
+# FS QA Test 056
+#
+# regression testing to test kernel crash with fast_commit feature.
+# (based on example from tests/generic/468)
+# commit: https://patchwork.ozlabs.org/patch/1586856
+#
+
+. ./common/preamble
+_begin_fstest auto quick log
+
+# Override the default cleanup function.
+_cleanup()
+{
+   cd /
+   rm -r -f $tmp.*
+   _scratch_unmount > /dev/null 2>&1
+}
+
+# Import common functions.
+. ./common/filter
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs ext4
+_require_scratch
+_require_scratch_shutdown
+_require_scratch_ext4_feature "fast_commit"
+
+testfile1=$SCRATCH_MNT/testfile1
+testfile2=$SCRATCH_MNT/testfile2
+blocksize=4096
+
+$MKFS_EXT4_PROG -F -b $blocksize -O fast_commit $SCRATCH_DEV 5G >> $seqres.full 2>&1 ||
+	_notrun "Require 5G disk space"
+
+_scratch_mount >> $seqres.full 2>&1
+
+check_stat_before_after()
+{
+	before="$1"
+	after="$2"
+
+	if [ "$before" != "$after" ]; then
+		echo "Before: $before"
+		echo "After : $after"
+	fi
+	echo "Before: $before" >> $seqres.full
+	echo "After : $after" >> $seqres.full
+}
+
+#
+# Idea is to create a file of 5M & write 5M of data followed by fsync.
+# Then falloc extra blocks (with -k) such that the blocks extend the block group
+# boundary considerably, followed by fsync, followed by shutdown.
+# This will trigger fast_commit recovery and will help test kernel crash w/o
+# fix.
+#
+fact=10
+echo "Test-1: fsync shutdown recovery test"
+$XFS_IO_PROG -f -c "truncate $((5*1024*1024))" \
+			-c "pwrite 0 $((5*1024*1024))" \
+			-c "fsync"  \
+			-c "falloc -k $((5*1024*1024)) $((32768*$blocksize*$fact))" \
+			$testfile1 >> $seqres.full 2>&1
+
+stat_opt='-c "b: %b s: %s a: %x m: %y c: %z"'
+before=$(stat "$stat_opt" "$testfile1")
+
+$XFS_IO_PROG -c "fsync" $testfile1
+_scratch_shutdown -v >> $seqres.full 2>&1
+_scratch_cycle_mount >> $seqres.full 2>&1
+
+after=$(stat "$stat_opt" $testfile1)
+
+check_stat_before_after "$before" "$after"
+
+echo "Test-2: fdatasync shutdown recovery test"
+$XFS_IO_PROG -f -c "truncate $((5*1024*1024))" \
+			-c "pwrite 0 $((5*1024*1024))" \
+			-c "fsync"  \
+			-c "falloc -k $((5*1024*1024)) $((32768*$blocksize*$fact))" \
+			$testfile2 >> $seqres.full 2>&1
+
+stat_opt='-c "b: %b s: %s"'
+before=$(stat "$stat_opt" $testfile2)
+
+$XFS_IO_PROG -c "fdatasync" $testfile2
+_scratch_shutdown -v >> $seqres.full 2>&1
+_scratch_cycle_mount >> $seqres.full 2>&1
+
+after=$(stat "$stat_opt" "$testfile2")
+check_stat_before_after "$before" "$after"
+
+# success, all done
+status=0
+exit
diff --git a/tests/ext4/056.out b/tests/ext4/056.out
new file mode 100644
index 00000000..0a793a4c
--- /dev/null
+++ b/tests/ext4/056.out
@@ -0,0 +1,3 @@
+QA output created by 056
+Test-1: fsync shutdown recovery test
+Test-2: fdatasync shutdown recovery test
-- 
2.31.1

[PATCHv1 2/4] common/punch: Add block_size argument to _filter_fiemap_**

[Ritesh Harjani]

From: Ritesh Harjani <ritesh.harjani@gmail.com>

Add block_size paramter to _filter_fiemap_flags() and
_filter_hole_fiemap(). This is used in next patches

Also this fixes some of the end of line whitespace issues while we are
at it.

Signed-off-by: Ritesh Harjani <ritesh.harjani@gmail.com>
---
 common/punch | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/common/punch b/common/punch
index b6b8a0b9..f99e21ad 100644
--- a/common/punch
+++ b/common/punch
@@ -109,6 +109,7 @@ _filter_fiemap()
 
 _filter_fiemap_flags()
 {
+	block_size=$1
 	$AWK_PROG '
 		$3 ~ /hole/ {
 			print $1, $2, $3;
@@ -135,23 +136,24 @@ _filter_fiemap_flags()
 			}
 			print $1, $2, flag_str
 		}' |
-	_coalesce_extents
+	_coalesce_extents $block_size
 }
 
-# Filters fiemap output to only print the 
+# Filters fiemap output to only print the
 # file offset column and whether or not
 # it is an extent or a hole
 _filter_hole_fiemap()
 {
+	block_size=$1
 	$AWK_PROG '
 		$3 ~ /hole/ {
-			print $1, $2, $3; 
+			print $1, $2, $3;
 			next;
-		}   
+		}
 		$5 ~ /0x[[:xdigit:]]+/ {
 			print $1, $2, "extent";
 		}' |
-	_coalesce_extents
+	_coalesce_extents $block_size
 }
 
 #     10000 Unwritten preallocated extent
-- 
2.31.1

[PATCHv1 3/4] ext4/057: Add crash test to check unwritten extents tracking with fast_commit

[Ritesh Harjani]

fast_commit in certain cases as discussed in the corresponding kernel
commit may miss to track the unwritten extents. This patch add a test
case around that (based on test case shared in kernel commit)

5e4d0eba1ccaf19f
ext4: fix fast commit may miss tracking range for FALLOC_FL_ZERO_RANGE

Signed-off-by: Ritesh Harjani <riteshh@linux.ibm.com>
---
 tests/ext4/057     | 62 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/ext4/057.out |  6 +++++
 2 files changed, 68 insertions(+)
 create mode 100755 tests/ext4/057
 create mode 100644 tests/ext4/057.out

diff --git a/tests/ext4/057 b/tests/ext4/057
new file mode 100755
index 00000000..9d008b65
--- /dev/null
+++ b/tests/ext4/057
@@ -0,0 +1,62 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2022 IBM.  All Rights Reserved.
+#
+# FS QA Test 057
+#
+# Test for ext4 fast_commit crash recovery test fixed by below kernel commit.
+#
+# commit 5e4d0eba1ccaf19f
+# ext4: fix fast commit may miss tracking range for FALLOC_FL_ZERO_RANGE
+#
+. ./common/preamble
+_begin_fstest auto quick log shutdown
+
+# Override the default cleanup function.
+_cleanup()
+{
+	cd /
+	rm -r -f $tmp.*
+}
+
+# Import common functions.
+. ./common/filter
+. ./common/punch
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs ext4
+_require_scratch
+_require_scratch_ext4_feature "fast_commit"
+
+t1=$SCRATCH_MNT/t1
+
+$MKFS_EXT4_PROG -F -O 64bit,fast_commit $SCRATCH_DEV 512m >> $seqres.full 2>&1
+_scratch_mount >> $seqres.full 2>&1
+
+bs=$(_get_block_size $SCRATCH_MNT)
+
+# create and write data to t1
+$XFS_IO_PROG -f -c "pwrite 0 $((100*$bs))" $t1 | _filter_xfs_io
+
+# fsync t1
+$XFS_IO_PROG -c "fsync" $t1
+
+# fzero certain range in between
+$XFS_IO_PROG -c "fzero -k  $((40*$bs)) $((20*$bs))" $t1 | _filter_xfs_io
+
+# fsync t1
+$XFS_IO_PROG -c "fsync" $t1
+
+# shutdown FS now for replay of FC to kick in next mount
+_scratch_shutdown -v >> $seqres.full 2>&1
+
+_scratch_cycle_mount
+
+# check fiemap reported is valid or not
+$XFS_IO_PROG -c "fiemap -v" $t1 | _filter_fiemap_flags $bs
+
+# success, all done
+status=0
+exit
diff --git a/tests/ext4/057.out b/tests/ext4/057.out
new file mode 100644
index 00000000..fc905de6
--- /dev/null
+++ b/tests/ext4/057.out
@@ -0,0 +1,6 @@
+QA output created by 057
+wrote 409600/409600 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+0: [0..39]: none
+1: [40..59]: unwritten
+2: [60..99]: nonelast
-- 
2.31.1

[PATCHv1 4/4] ext4/058: Add shutdown recovery test with fast_commit

[Ritesh Harjani]

In certain cases fast_commit may not delete the range during replay
phase (after sudden FS shutdown) due to some operations which depends on
inode->i_size (which during replay of an inode with fast_commit could be
0 for sometime). This fstest tests such scenario with fast_commit in
ext4.

This test case is based on the test case shared via Xin Yin.

Signed-off-by: Ritesh Harjani <riteshh@linux.ibm.com>
---
 tests/ext4/058     | 66 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/ext4/058.out |  7 +++++
 2 files changed, 73 insertions(+)
 create mode 100755 tests/ext4/058
 create mode 100644 tests/ext4/058.out

diff --git a/tests/ext4/058 b/tests/ext4/058
new file mode 100755
index 00000000..4d70f483
--- /dev/null
+++ b/tests/ext4/058
@@ -0,0 +1,66 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2022 IBM.  All Rights Reserved.
+#
+# FS QA Test 058
+#
+# Tests fast_commit feature of ext4 as fixed in below commit
+# 0b5b5a62b945a141: ext4: use ext4_ext_remove_space() for fast commit replay delete range
+# (Based on test case shared by Xin Yin <yinxin.x@bytedance.com>)
+#
+. ./common/preamble
+_begin_fstest auto shutdown quick log recoveryloop
+
+# Override the default cleanup function.
+_cleanup()
+{
+	cd /
+	rm -r -f $tmp.*
+   _scratch_unmount > /dev/null 2>&1
+}
+
+# Import common functions.
+. ./common/filter
+. ./common/punch
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs ext4
+_require_scratch
+_require_scratch_ext4_feature "fast_commit"
+
+t1=$SCRATCH_MNT/foo
+t2=$SCRATCH_MNT/bar
+
+$MKFS_EXT4_PROG -F -O 64bit,fast_commit $SCRATCH_DEV 512m >> $seqres.full 2>&1
+_scratch_mount >> $seqres.full 2>&1
+
+bs=$(_get_block_size $SCRATCH_MNT)
+
+# create and write data to t1
+$XFS_IO_PROG -f -c "pwrite 0 $((100*$bs))" $t1 | _filter_xfs_io
+
+# fzero certain range in between with -k
+$XFS_IO_PROG -c "fzero -k  $((40*$bs)) $((20*$bs))" $t1 | _filter_xfs_io
+
+# create and fsync a new file t2
+$XFS_IO_PROG -f -c "fsync" $t2
+
+# fpunch within the i_size of a file
+$XFS_IO_PROG -c "fpunch $((30*$bs)) $((20*$bs))" $t1
+
+# fsync t1 to trigger fast_commit operation
+$XFS_IO_PROG -c "fsync" $t1
+
+# shutdown FS now for replay of FC to kick in next mount
+_scratch_shutdown -v >> $seqres.full 2>&1
+
+_scratch_cycle_mount
+
+# check fiemap reported is valid or not
+$XFS_IO_PROG -c "fiemap -v" $t1 | _filter_fiemap_flags $bs
+
+# success, all done
+status=0
+exit
diff --git a/tests/ext4/058.out b/tests/ext4/058.out
new file mode 100644
index 00000000..98359e01
--- /dev/null
+++ b/tests/ext4/058.out
@@ -0,0 +1,7 @@
+QA output created by 058
+wrote 409600/409600 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+0: [0..29]: none
+1: [30..49]: hole
+2: [50..59]: unwritten
+3: [60..99]: nonelast
-- 
2.31.1