[PATCH 0/8] xfstests: Stress tests improments v3

[PATCH 0/8] xfstests: Stress tests improments v3

[Dmitry Monakhov]

This is third version of the patch series, previous version
can be found here: http://oss.sgi.com/archives/xfs/2012-09/msg00328.html

xfstests-add-fio-requirement-V2
xfstests-get-rid-of-hardcoded-sbin-mkfs
xfstest-add-configurable-load-factors
xfstest-allow-fsstress-to-use-load-factor-where-appr
run-check-to-common_rc
add-fallocate-truncate-vs-AIO-DIO-stress-test
add-fallocate-punch_hole-vs-AIO-DIO-stress-test
add-defragmentation-stress-tests-for-ext4

Changes since V2

Mostly code style cleanups accodring to Dave's comments
- add _scale_fsstress_args function
- move run_check function to common.rc
- Add more comments
- split combined e4defrag testcase in to disicated tescases.

[PATCH 1/8] xfstests: add fio requirement V2

[Dmitry Monakhov]

FIO is very flexible io generator, I would call it IO swiss knife.
Currently we have tonns of hardcoded application which reproduces
some predefined scenario. This approach has obvious dissadvantages
1) Lack of flexability: one written it is hard to modify it in future
2) Code base is large, many routines written again and again

At the same time add new fio based tast is just add simle INI file.
This greatly simplify code review. I do beleve that some day we will
replace most of hardcoded io binaries with fio.

One who is planning to run $FIO_PROG should first check that system
contains appropriate version which is able to handle jobfile
for example:  _require_fio 286-job.fio

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
Reviewed-by: Dave Chinner <dchinner@redhat.com>
---
 common.config |    1 +
 common.rc     |   15 +++++++++++++++
 2 files changed, 16 insertions(+), 0 deletions(-)

diff --git a/common.config b/common.config
index 7bed1c5..d5c8956 100644
--- a/common.config
+++ b/common.config
@@ -158,6 +158,7 @@ export XFS_QUOTA_PROG="`set_prog_path xfs_quota`"
 export KILLALL_PROG="`set_prog_path killall`"
 export INDENT_PROG="`set_prog_path indent`"
 export XFS_COPY_PROG="`set_prog_path xfs_copy`"
+export FIO_PROG="`set_prog_path fio`"
 
 # Generate a comparable xfsprogs version number in the form of
 # major * 10000 + minor * 100 + release
diff --git a/common.rc b/common.rc
index 602513a..f5027e7 100644
--- a/common.rc
+++ b/common.rc
@@ -1758,6 +1758,21 @@ _require_btrfs()
 	[ $? -eq 0 ] || _notrun "$BTRFS_UTIL_PROG too old (must support $cmd)"
 }
 
+# Check that fio is present, and it is able to execute given jobfile
+_require_fio()
+{
+	job=$1
+
+	_require_command $FIO_PROG
+	if [ -z "$1" ]; then
+		return 1;
+	fi
+	
+	$FIO_PROG --warnings-fatal --showcmd $job >/dev/null 2>&1
+	[ $? -eq 0 ] || _notrun "$FIO_PROG too old"
+}
+
+
 # arg 1 is dev to remove and is output of the below eg.
 # ls -l /sys/class/block/sdd | rev | cut -d "/" -f 3 | rev
 _devmgt_remove()
-- 
1.7.1

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs

[PATCH 2/8] xfstests: get rid of hardcoded /sbin/mkfs

[Dmitry Monakhov]

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 common.rc |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.rc b/common.rc
index f5027e7..196619d 100644
--- a/common.rc
+++ b/common.rc
@@ -334,7 +334,7 @@ _scratch_mkfs()
         $MKFS_BTRFS_PROG $MKFS_OPTIONS $* $SCRATCH_DEV > /dev/null
 	;;
     *)
-	/sbin/mkfs -t $FSTYP -- $MKFS_OPTIONS $* $SCRATCH_DEV
+	$MKFS_PROG -t $FSTYP -- $MKFS_OPTIONS $* $SCRATCH_DEV
 	;;
     esac
 }
@@ -353,7 +353,7 @@ _scratch_mkfs_sized()
 	_scratch_mkfs_xfs -d size=$fssize -b size=$blocksize
 	;;
     ext2|ext3|ext4|ext4dev)
-	/sbin/mkfs.$FSTYP $MKFS_OPTIONS -b $blocksize $SCRATCH_DEV $blocks
+	$MKFS_PROG -t $FSTYP $MKFS_OPTIONS -b $blocksize $SCRATCH_DEV $blocks
 	;;
     btrfs)
 	$MKFS_BTRFS_PROG $MKFS_OPTIONS -b $fssize $SCRATCH_DEV
-- 
1.7.1

[PATCH 3/8] xfstest: add configurable load factors

[Dmitry Monakhov]

Most stress test has probable behaviour, the longer test run the
larger corner cases will be cover. It is reasonable to allow
user to provide some sort of system load factor.
This patch introduce two global variables
LOAD_FACTOR: Usually means factor number of running tasks
TIME_FACTOR: Usually means factor of run time, or number of operations
If not speficied both variables defined to 1, so original behaviour
preserved.

TODO: Change all stress tests to use this variables

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 common.config |    2 ++
 group         |    7 ++++++-
 2 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/common.config b/common.config
index d5c8956..a956a46 100644
--- a/common.config
+++ b/common.config
@@ -73,6 +73,8 @@ export HOST_OPTIONS=${HOST_OPTIONS:=local.config}
 export CHECK_OPTIONS=${CHECK_OPTIONS:="-g auto"}
 export BENCH_PASSES=${BENCH_PASSES:=5}
 export XFS_MKFS_OPTIONS=${XFS_MKFS_OPTIONS:=-bsize=4096}
+export TIME_FACTOR=${TIME_FACTOR:=1}
+export LOAD_FACTOR=${LOAD_FACTOR:=1}
 
 export PWD=`pwd`
 #export DEBUG=${DEBUG:=...} # arbitrary CFLAGS really.
diff --git a/group b/group
index 697269b..810c938 100644
--- a/group
+++ b/group
@@ -113,7 +113,12 @@ dangerous
 # on current systems
 deprecated
 
-#
+# Stress test with probable behaviour, the longer test run the
+# larger corner cases will be covered. Configurable variables:
+# LOAD_FACTOR: Usually means factor number of running tasks
+# TIME_FACTOR: Usually means factor of run time, or number of operations
+stress
+
 # test-group association ... one line per test
 #
 001 rw dir udf auto quick
-- 
1.7.1

[PATCH 4/8] xfstest: allow fsstress to use load factor where appropriate

[Dmitry Monakhov]

1) Add _scale_fsstress_args function which transform argumets according
   to load factors
2) Let all non deterministic fsstress tests to use scaled arguments

I've able to trigger OOPS on xfs see:https://gist.github.com/dmonakhov/4762653

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 017       |    3 ++-
 068       |    4 ++--
 070       |    5 +++--
 076       |    5 +++--
 083       |    4 ++--
 087       |    3 ++-
 104       |    4 +++-
 114       |    6 +++---
 167       |    5 +++--
 232       |    4 ++--
 232.out   |    1 -
 233       |    6 +++---
 233.out   |    1 -
 269       |    2 +-
 270       |    2 +-
 common.rc |   14 ++++++++++++++
 group     |   26 +++++++++++++-------------
 17 files changed, 57 insertions(+), 38 deletions(-)

diff --git a/017 b/017
index 9ca0e72..7db4667 100755
--- a/017
+++ b/017
@@ -67,7 +67,8 @@ echo "*** test"
 for l in 0 1 2 3 4
 do
         echo "    *** test $l"
-        $FSSTRESS_PROG -d $SCRATCH_MNT -n 1000 $FSSTRESS_AVOID >>$seq.full
+	FSSTRESS_ARGS=`_scale_fsstress_args -d $SCRATCH_MNT -n 1000 $FSSTRESS_AVOID`
+        $FSSTRESS_PROG  $FSSTRESS_ARGS >>$seq.full
 
         _scratch_mount -o remount,ro \
             || _fail "remount ro failed"
diff --git a/068 b/068
index b595d1d..5ee899a 100755
--- a/068
+++ b/068
@@ -81,8 +81,8 @@ touch $tmp.running
       do
       # We do both read & write IO - not only is this more realistic,
       # but it also potentially tests atime updates
-      $FSSTRESS_PROG -d $STRESS_DIR -p $procs -n $nops $FSSTRESS_AVOID \
-	  > /dev/null 2>&1
+      FSSTRESS_ARGS=`_scale_fsstress_args -d $STRESS_DIR -p $procs -n $nops $FSSTRESS_AVOID`
+      $FSSTRESS_PROG $FSSTRESS_ARGS > /dev/null 2>&1
     done
 
     rm -r $STRESS_DIR/*
diff --git a/070 b/070
index f48c33c..334cce7 100755
--- a/070
+++ b/070
@@ -52,7 +52,7 @@ _require_attrs
 
 _setup_testdir
 
-$FSSTRESS_PROG \
+FSSTRESS_ARGS=`_scale_fsstress_args \
 	-d $testdir/fsstress \
 	-f allocsp=0 \
 	-f freesp=0 \
@@ -62,7 +62,8 @@ $FSSTRESS_PROG \
 	-f unresvsp=0 \
 	-f attr_set=100 \
 	-f attr_remove=100 \
-        -p 1 -n 10000 -S c >$seq.full 2>&1
+        -p 1 -n 10000 -S c`
+$FSSTRESS_PROG $FSSTRESS_ARGS >$seq.full 2>&1
 
 status=$?
 exit
diff --git a/076 b/076
index e472b26..793b869 100755
--- a/076
+++ b/076
@@ -74,8 +74,9 @@ echo "*** test concurrent block/fs access"
 cat $SCRATCH_DEV >/dev/null &
 pid=$!
 
-$FSSTRESS_PROG -d $SCRATCH_MNT -p 2 -n 2000 $FSSTRESS_AVOID >>$seq.full
-
+FSSTRESS_ARGS=`_scale_fsstress_args -p 2 -n 2000 $FSSTRESS_AVOID`
+echo "run fsstress with args: $FSSTRESS_ARGS" >>$seq.full
+$FSSTRESS_PROG $FSSTRESS_ARGS >>$seq.full
 _lets_get_pidst
 _check_scratch_fs
 
diff --git a/083 b/083
index e0670b9..f5349a9 100755
--- a/083
+++ b/083
@@ -84,8 +84,8 @@ workout()
 		|| _fail "mount failed"
 
 	# -w ensures that the only ops are ones which cause write I/O
-	$FSSTRESS_PROG -d $SCRATCH_MNT -w -p $procs -n $nops $FSSTRESS_AVOID \
-		>>$seq.full
+	FSSTRESS_ARGS=`_scale_fsstress_args -d $SCRATCH_MNT -w -p $procs -n $nops $FSSTRESS_AVOID`
+	$FSSTRESS_PROG $FSSTRESS_ARGS >>$seq.full
 	_check_scratch_fs
 }
 
diff --git a/087 b/087
index 48e5eaa..e7be9a0 100755
--- a/087
+++ b/087
@@ -47,7 +47,8 @@ _do_meta()
     param="-p 4 -z -f rmdir=10 -f link=10 -f creat=10 -f mkdir=10 \
            -f rename=30 -f stat=30 -f unlink=30 -f truncate=20"
     _echofull "calling fsstress $param -m8 -n $count"
-    if ! $FSSTRESS_PROG $param $FSSTRESS_AVOID -m 8 -n $count -d $out >>$seq.full 2>&1
+    FSSTRESS_ARGS=`_scale_fsstress_args $param $FSSTRESS_AVOID -m 8 -n $count -d $out`
+    if ! $FSSTRESS_PROG $FSSTRESS_ARGS >>$seq.full 2>&1
     then
 	_echofull "fsstress failed"
     fi
diff --git a/104 b/104
index 14f2669..8db6d88 100755
--- a/104
+++ b/104
@@ -64,7 +64,9 @@ _stress_scratch()
 	procs=3
 	nops=1000
 	# -w ensures that the only ops are ones which cause write I/O
-	$FSSTRESS_PROG -d $SCRATCH_MNT -w -p $procs -n $nops $FSSTRESS_AVOID > /dev/null &
+	FSSTRESS_ARGS=`_scale_fsstress_args -d $SCRATCH_MNT -w -p $procs \
+	    -n $nops $FSSTRESS_AVOID`
+	$FSSTRESS_PROG $FSSTRESS_ARGS >> $seq.full &
 }
 
 # real QA test starts here
diff --git a/114 b/114
index 7679222..411f079 100755
--- a/114
+++ b/114
@@ -246,14 +246,14 @@ _test_fsstress()
 
 	out=$SCRATCH_MNT/fsstress.$$
 	count=1000
-	args="-z \
+	args=`_scale_fsstress_args -z \
 -f rmdir=10 -f link=10 -f creat=10 \
 -f mkdir=10 -f rename=30 -f unlink=10 \
 -f symlink=10 \
--n $count -d $out -p 3"
+-n $count -d $out -p 3`
 
 	echo "fsstress $args" | sed -e "s#$out#outdir#"
-	if ! $FSSTRESS_PROG $args | _filter_num
+	if ! $FSSTRESS_PROG $FSSTRESS_ARGS | _filter_num
 	then
 		echo "    fsstress $args returned $?"
 		cat $tmp.out | tee -a $here/$seq.full
diff --git a/167 b/167
index ccb6c2a..5fb95e8 100755
--- a/167
+++ b/167
@@ -44,8 +44,9 @@ workout()
 {
 	procs=100
 	nops=15000
-	$FSSTRESS_PROG -d $SCRATCH_MNT -p $procs -n $nops $FSSTRESS_AVOID \
-		>>$seq.full &
+	FSSTRESS_ARGS=`_scale_fsstress_args -d $SCRATCH_MNT -p $procs -n $nops \
+	    $FSSTRESS_AVOID`
+	$FSSTRESS_PROG $FSSTRESS_ARGS >> $seq.full &
 	sleep 2
 }
 
diff --git a/232 b/232
index 2795da7..d915d42 100755
--- a/232
+++ b/232
@@ -54,9 +54,9 @@ _fsstress()
 
 	out=$SCRATCH_MNT/fsstress.$$
 	count=2000
-	args="-n $count -d $out -p 7"
+	args=`_scale_fsstress_args -d $out -n $count -p 7`
 
-	echo "fsstress $args" | tee -a $here/$seq.full | sed -e "s#$out#outdir#"
+	echo "fsstress $args" >> tee -a $here/$seq.full
 	if ! $FSSTRESS_PROG $args | tee -a $here/$seq.full | _filter_num
 	then
 		echo "    fsstress $args returned $?"
diff --git a/232.out b/232.out
index ef82a89..5da53d4 100644
--- a/232.out
+++ b/232.out
@@ -2,7 +2,6 @@ QA output created by 232
 
 Testing fsstress
 
-fsstress -n 2000 -d outdir -p 7
 seed = S
 Comparing user usage
 Comparing group usage
diff --git a/233 b/233
index 28e6ac7..649de51 100755
--- a/233
+++ b/233
@@ -58,12 +58,12 @@ _fsstress()
 
 	out=$SCRATCH_MNT/fsstress.$$
 	count=5000
-	args="-z \
+	args=`_scale_fsstress_args -z \
 -f rmdir=20 -f link=10 -f creat=10 -f mkdir=10 -f unlink=20 -f symlink=10 \
 -f rename=10 -f fsync=2 -f write=15 -f dwrite=15 \
--n $count -d $out -p 7"
+-n $count -d $out -p 7`
 
-	echo "fsstress $args" | tee -a $here/$seq.full | sed -e "s#$out#outdir#"
+	echo "fsstress $args" >> tee -a $here/$seq.full
 	if ! su $qa_user -c "$FSSTRESS_PROG $args" | tee -a $here/$seq.full | _filter_num
 	then
 		echo "    fsstress $args returned $?"
diff --git a/233.out b/233.out
index fa36ca3..91c1a30 100644
--- a/233.out
+++ b/233.out
@@ -2,7 +2,6 @@ QA output created by 233
 
 Testing fsstress
 
-fsstress -z -f rmdir=20 -f link=10 -f creat=10 -f mkdir=10 -f unlink=20 -f symlink=10 -f rename=10 -f fsync=2 -f write=15 -f dwrite=15 -n 5000 -d outdir -p 7
 seed = S
 Comparing user usage
 Comparing group usage
diff --git a/269 b/269
index 7e13ed9..7d63b87 100755
--- a/269
+++ b/269
@@ -45,7 +45,7 @@ _workout()
 	num_iterations=10
 	enospc_time=2
 	out=$SCRATCH_MNT/fsstress.$$
-	args="-p128 -n999999999 -f setattr=1 $FSSTRESS_AVOID -d $out"
+	args=`_scale_fsstress_args -p128 -n999999999 -f setattr=1 $FSSTRESS_AVOID -d $out`
 	echo "fsstress $args" >> $here/$seq.full
 	$FSSTRESS_PROG $args > /dev/null 2>&1 &
 	pid=$!
diff --git a/270 b/270
index b9ada27..b753923 100755
--- a/270
+++ b/270
@@ -48,7 +48,7 @@ _workout()
 	num_iterations=10
 	enospc_time=2
 	out=$SCRATCH_MNT/fsstress.$$
-	args="-p128 -n999999999 -f setattr=1 $FSSTRESS_AVOID -d $out"
+	args=`_scale_fsstress_args -p128 -n999999999 -f setattr=1 $FSSTRESS_AVOID -d $out`
 	echo "fsstress $args" >> $here/$seq.full
 	# Grant chown capability 
 	cp $FSSTRESS_PROG  $tmp.fsstress.bin
diff --git a/common.rc b/common.rc
index 196619d..ea5c477 100644
--- a/common.rc
+++ b/common.rc
@@ -1793,6 +1793,20 @@ _devmgt_add()
 	echo ${tdl} >  /sys/class/scsi_host/host${h}/scan || _fail "Add disk failed"
 }
 
+_scale_fsstress_args()
+{
+    args=""                                                                                                         
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            -n) args="$args $1 $(($2 * $TIME_FACTOR))"; shift ;;
+            -p) args="$args $1 $(($2 * $LOAD_FACTOR))"; shift ;;
+            *) args="$args $1" ;;
+        esac                                                                                                    
+        shift
+    done
+    echo $args                                                                                            
+}
+  
 ################################################################################
 
 if [ "$iam" != new -a "$iam" != bench ]
diff --git a/group b/group
index 810c938..c6630c9 100644
--- a/group
+++ b/group
@@ -137,7 +137,7 @@ stress
 014 rw udf auto quick
 015 other auto quick
 016 rw auto quick
-017 mount auto quick
+017 mount auto quick stress
 018 deprecated # log logprint v2log
 019 mkfs auto quick
 020 metadata attr udf auto quick
@@ -188,26 +188,26 @@ stress
 065 dump auto
 066 dump ioctl auto quick
 067 acl attr auto quick
-068 other auto freeze dangerous
+068 other auto freeze dangerous stress
 069 rw udf auto quick
-070 attr udf auto quick
+070 attr udf auto quick stress
 071 rw auto
 072 rw auto prealloc quick
 073 copy auto
 074 rw udf auto
 075 rw udf auto quick
-076 metadata rw udf auto quick
+076 metadata rw udf auto quick stress
 077 acl attr auto enospc
 078 growfs auto quick
 079 acl attr ioctl metadata auto quick
 080 rw ioctl
 081 deprecated # log logprint quota
 082 deprecated # log logprint v2log
-083 rw auto
+083 rw auto enospc stress
 084 ioctl rw auto
 085 log auto quick
 086 log v2log auto
-087 log v2log auto quota
+087 log v2log auto quota stress
 088 perms auto quick
 089 metadata auto
 090 rw auto
@@ -224,7 +224,7 @@ stress
 101 udf
 102 udf
 103 metadata dir ioctl auto quick
-104 growfs ioctl prealloc auto
+104 growfs ioctl prealloc auto stress
 105 acl auto quick
 106 quota
 107 quota
@@ -234,7 +234,7 @@ stress
 111 ioctl
 112 rw aio auto quick
 113 rw aio auto quick
-114 parent attr
+114 parent attr stress
 115 parent attr
 116 quota auto quick
 117 attr auto quick
@@ -287,7 +287,7 @@ stress
 164 rw pattern auto prealloc quick
 165 rw pattern auto prealloc quick
 166 rw metadata auto quick
-167 rw metadata auto
+167 rw metadata auto stress
 168 dmapi auto
 169 rw metadata auto quick
 170 rw filestreams auto quick
@@ -355,8 +355,8 @@ stress
 229 auto rw
 230 auto quota quick
 231 auto quota
-232 auto quota
-233 auto quota
+232 auto quota stress
+233 auto quota stress
 234 auto quota
 235 auto quota quick
 236 auto quick metadata
@@ -392,8 +392,8 @@ stress
 266 dump ioctl auto quick
 267 dump ioctl tape
 268 dump ioctl tape
-269 auto rw prealloc ioctl enospc
-270 auto quota rw prealloc ioctl enospc
+269 auto rw prealloc ioctl enospc stress
+270 auto quota rw prealloc ioctl enospc stress
 271 auto rw quick
 272 auto enospc rw
 273 auto rw
-- 
1.7.1

[PATCH 5/8] xfstests: Move run_check to common.rc

[Dmitry Monakhov]

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 276       |    6 ------
 common.rc |    8 +++++++-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/276 b/276
index 082f943..c1ce8ac 100755
--- a/276
+++ b/276
@@ -165,12 +165,6 @@ _btrfs_inspect_check()
 	return $ret
 }
 
-run_check()
-{
-	echo "# $@" >> $seq.full 2>&1
-	"$@" >> $seq.full 2>&1 || _fail "failed: '$@'"
-}
-
 workout()
 {
 	fsz=$1
diff --git a/common.rc b/common.rc
index ea5c477..5c3dda1 100644
--- a/common.rc
+++ b/common.rc
@@ -1806,7 +1806,13 @@ _scale_fsstress_args()
     done
     echo $args                                                                                            
 }
-  
+
+run_check()
+{
+	echo "# $@" >> $seq.full 2>&1
+	"$@" >> $seq.full 2>&1 || _fail "failed: '$@'"
+}
+
 ################################################################################
 
 if [ "$iam" != new -a "$iam" != bench ]
-- 
1.7.1

[PATCH 6/8] xfstests: add fallocate/truncate vs AIO/DIO stress test

[Dmitry Monakhov]

Run  DIO, fallocate and truncate threads on a common file in parallel.
If race exist old dio request may rewrite blocks after it was allocated
to another file, we will catch that by verifying blocks content.

this patch known to catch deadlock for ext4
http://lists.openwall.net/linux-ext4/2012/09/06/3

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 286     |  157 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 286.out |    5 ++
 group   |    1 +
 3 files changed, 163 insertions(+), 0 deletions(-)
 create mode 100755 286
 create mode 100644 286.out

diff --git a/286 b/286
new file mode 100755
index 0000000..588387c
--- /dev/null
+++ b/286
@@ -0,0 +1,157 @@
+#! /bin/bash
+# FSQA Test No. 286
+#
+# AIO/DIO stress test
+# Run random AIO/DIO activity and fallocate/truncate simultaneously
+# Test will operate on huge sparsed files so ENOSPC is expected.
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
+owner=dmonakhov@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_need_to_be_root
+_require_scratch
+
+NUM_JOBS=$((4*LOAD_FACTOR))
+BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
+FILE_SIZE=$((BLK_DEV_SIZE * 512))
+
+cat >$tmp-$seq.fio <<EOF
+###########
+# $seq test fio activity
+# Filenames derived from jobsname and jobid like follows:
+# ${JOB_NAME}.${JOB_ID}.${ITERATION_ID}
+[global]
+ioengine=libaio
+bs=128k
+directory=${SCRATCH_MNT}
+filesize=${FILE_SIZE}
+size=999G
+iodepth=128*${LOAD_FACTOR}
+continue_on_error=write
+ignore_error=,ENOSPC
+error_dump=0
+create_on_open=1
+fallocate=none
+exitall=1
+
+## Perform direct aio, to files which may be truncated
+## by external task
+[direct_aio]
+direct=1
+buffered=0
+numjobs=${NUM_JOBS}
+rw=randwrite
+runtime=100*${TIME_FACTOR}
+time_based
+
+# Perform direct aio and verify data
+# This test case should check use-after-free issues
+[aio-dio-verifier]
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=4
+verifysort=1
+direct=1
+bs=4k
+rw=randrw
+filename=aio-dio-verifier
+
+# Perform buffered aio and verify data
+# This test case should check use-after-free issues
+[buffered-aio-verifier]
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=4
+verifysort=1
+direct=0
+buffered=1
+bs=4k
+rw=randrw
+filename=buffered-aio-verifier
+EOF
+
+_require_fio $tmp-$seq.fio
+
+_workout()
+{
+	echo ""
+	echo "Run fio with random aio-dio pattern"
+	echo ""
+	cat $tmp-$seq.fio >>  $seq.full
+	run_check $FIO_PROG $tmp-$seq.fio &
+	pid=$!
+	echo "Start fallocate/truncate loop"
+	
+	for ((i=0; ; i++))
+	do
+	    for ((k=1; k <= NUM_JOBS; k++))
+	    do
+		fallocate -l $FILE_SIZE $SCRATCH_MNT/direct_aio.$k.0 \
+		    	>> $seq.full 2>&1
+	    done
+	    for ((k=1; k <= NUM_JOBS; k++))
+	    do
+		truncate -s 0 $SCRATCH_MNT/direct_aio.$k.0
+	    done
+	    # Following like will check that pid is still run.
+	    # Once fio exit we can stop fallocate/truncate loop
+	    kill -0 $pid > /dev/null 2>&1 || break
+	done
+	wait $pid
+}
+
+_scratch_mkfs >> $seq.full 2>&1
+_scratch_mount
+
+if ! _workout; then
+	umount $SCRATCH_DEV 2>/dev/null
+	exit
+fi
+
+if ! _scratch_unmount; then
+	echo "failed to umount"
+	status=1
+	exit
+fi
+_check_scratch_fs
+status=$?
+exit
diff --git a/286.out b/286.out
new file mode 100644
index 0000000..d721996
--- /dev/null
+++ b/286.out
@@ -0,0 +1,5 @@
+QA output created by 286
+
+Run fio with random aio-dio pattern
+
+Start fallocate/truncate loop
diff --git a/group b/group
index c6630c9..01fb35b 100644
--- a/group
+++ b/group
@@ -409,3 +409,4 @@ stress
 283 dump ioctl auto quick
 284 auto
 285 auto dump quota quick
+286 auto rw enospc aio stress
-- 
1.7.1

Re: [PATCH 6/8] xfstests: add fallocate/truncate vs AIO/DIO stress test

[Dave Chinner]

On Wed, Feb 13, 2013 at 12:50:01PM +0400, Dmitry Monakhov wrote:
> Run  DIO, fallocate and truncate threads on a common file in parallel.
> If race exist old dio request may rewrite blocks after it was allocated
> to another file, we will catch that by verifying blocks content.
> 
> this patch known to catch deadlock for ext4
> http://lists.openwall.net/linux-ext4/2012/09/06/3
> 
> Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
> ---
>  286     |  157 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  286.out |    5 ++
>  group   |    1 +
>  3 files changed, 163 insertions(+), 0 deletions(-)
>  create mode 100755 286
>  create mode 100644 286.out
> 
> diff --git a/286 b/286
> new file mode 100755
> index 0000000..588387c
> --- /dev/null
> +++ b/286
> @@ -0,0 +1,157 @@
> +#! /bin/bash
> +# FSQA Test No. 286
> +#
> +# AIO/DIO stress test
> +# Run random AIO/DIO activity and fallocate/truncate simultaneously
> +# Test will operate on huge sparsed files so ENOSPC is expected.
> +#
> +#-----------------------------------------------------------------------
> +# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.

I think you need to put the correct copyright statements in these
new tests ;)

Cheers,

Dave.
-- 
Dave Chinner
david@fromorbit.com

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs

[PATCH 7/8] xfstests: add fallocate/punch_hole vs AIO/DIO stress test

[Dmitry Monakhov]

Run  random AIO/DIO activity (fio's job:direct_aio_raicer)
random fallocate activity(fio's job:falloc_raicer)
and random punch_hole activity(punch_hole_raicer) on a common
file in parallel. If race exist old dio request may rewrite
punched block after it was allocated to another file, we will
catch that by verifier fio's job: "aio-dio-verifier".

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 287     |  153 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 287.out |    4 ++
 group   |    1 +
 3 files changed, 158 insertions(+), 0 deletions(-)
 create mode 100755 287
 create mode 100644 287.out

diff --git a/287 b/287
new file mode 100755
index 0000000..877073f
--- /dev/null
+++ b/287
@@ -0,0 +1,153 @@
+#! /bin/bash
+# FSQA Test No. 287
+#
+# AIO/DIO stress test
+# Run random AIO/DIO activity and fallocate/punch_hole simultaneously
+# Test will operate on huge sparsed file so ENOSPC is expected.
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
+owner=dmonakhov@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_need_to_be_root
+_require_scratch
+
+NUM_JOBS=$((4*LOAD_FACTOR))
+BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
+if [ $((BLK_DEV_SIZE)) -gt 1048576 ]
+then
+    BLK_DEV_SIZE=1048576
+fi 
+FS_SIZE=$((BLK_DEV_SIZE * 512))
+
+cat >$tmp-$seq.fio <<EOF
+###########
+# $seq test fio activity
+# Run  DIO, fallocate and punch_hole threads on a single in parallel 
+#
+# If race exist old dio request may rewrite punched block after it was
+# allocated to another file, we will catch that by verifying blocks content
+#
+[global]
+directory=${SCRATCH_MNT}
+filesize=${FS_SIZE}
+size=999G
+continue_on_error=write
+ignore_error=,ENOSPC
+error_dump=0
+
+create_on_open=1
+fallocate=none
+exitall=1
+
+## Perform direct aio, to files which may be truncated
+## by external task
+[direct_aio_raicer]
+ioengine=libaio
+iodepth=128*${LOAD_FACTOR} 
+bs=128k 
+direct=1
+numjobs=${NUM_JOBS}
+rw=randwrite
+runtime=100*${TIME_FACTOR}
+time_based
+filename=racer
+
+# Run falloc and punch_hole threads in parallel
+# After activity file will be highly fragmented
+[falloc_raicer]
+ioengine=falloc
+runtime=100*${TIME_FACTOR}
+iodepth=1
+bssplit=128k/80:512k/10:32k/10
+rw=randwrite
+numjobs=1
+filename=racer
+
+[punch_hole_raicer]
+ioengine=falloc
+runtime=100*${TIME_FACTOR}
+bs=4k
+time_based=10
+rw=randtrim
+numjobs=2
+filename=racer
+time_based
+
+# Verifier thread continiously write to newly allcated blocks
+# and veryfy written content
+[aio-dio-verifier]
+ioengine=libaio 
+iodepth=128*${LOAD_FACTOR}
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=4
+verifysort=1
+direct=1
+bs=4k
+rw=randwrite
+filename=aio-dio-verifier
+EOF
+
+_workout()
+{
+	echo ""
+	echo "Run fio with random aio-dio pattern"
+	echo ""
+	cat $tmp-$seq.fio >>  $seq.full
+	run_check $FIO_PROG $tmp-$seq.fio
+}
+
+_require_fio $tmp-$seq.fio
+
+_scratch_mkfs_sized $FS_SIZE >> $seq.full 2>&1
+_scratch_mount
+
+if ! _workout; then
+	umount $SCRATCH_DEV 2>/dev/null
+	exit
+fi
+
+if ! _scratch_unmount; then
+	echo "failed to umount"
+	status=1
+	exit
+fi
+_check_scratch_fs
+status=$?
+exit
diff --git a/287.out b/287.out
new file mode 100644
index 0000000..d22e097
--- /dev/null
+++ b/287.out
@@ -0,0 +1,4 @@
+QA output created by 287
+
+Run fio with random aio-dio pattern
+
diff --git a/group b/group
index 01fb35b..fcd9174 100644
--- a/group
+++ b/group
@@ -410,3 +410,4 @@ stress
 284 auto
 285 auto dump quota quick
 286 auto rw enospc aio stress
+287 auto rw enospc aio prealloc stress
-- 
1.7.1

[PATCH 8/8] xfstests: add defragmentation stress tests for ext4

[Dmitry Monakhov]

Perform various regression tests for ext4defrag  subsystem

 288'th Test1: Defragment file while other task does direct AIO
 289'th Test2: Perform defragmentation on file under buffered AIO
 	while third task does direct AIO to donor file
 290'th Test3: Two defrag tasks use common donor file.
 291'th Test4: Stress defragmentation. Several threads pefrorm
 	fragmentation at random position use inplace=1 will
	allocate and free blocks inside defrag event improve
	load pressure.

This tests are considered dengerous because 289'th and 290'th are known
to trigger OOPS on recent kernels see:https://gist.github.com/dmonakhov/4770294

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 288           |  126 +++++++++++++++++++++++++++++++++++++++++++++++
 288.out       |    4 ++
 289           |  143 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 289.out       |    4 ++
 290           |  153 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 290.out       |    4 ++
 291           |  130 ++++++++++++++++++++++++++++++++++++++++++++++++
 291.out       |    4 ++
 common.defrag |    4 +-
 group         |    4 ++
 10 files changed, 574 insertions(+), 2 deletions(-)
 create mode 100755 288
 create mode 100644 288.out
 create mode 100755 289
 create mode 100644 289.out
 create mode 100755 290
 create mode 100644 290.out
 create mode 100755 291
 create mode 100644 291.out

diff --git a/288 b/288
new file mode 100755
index 0000000..fbe4753
--- /dev/null
+++ b/288
@@ -0,0 +1,126 @@
+#! /bin/bash
+# FSQA Test No. 288
+#
+# Ext4 defragmentation stress test
+# Defragment file while other task does direct io
+#-----------------------------------------------------------------------
+# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
+owner=dmonakhov@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.defrag
+
+# real QA test starts here
+_supported_os Linux
+_supported_fs ext4
+_need_to_be_root
+_require_scratch
+_require_defrag
+
+BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
+# We need space for 2 files (test file, and donor one)
+# reserve 30% in order to avoid ENOSPC 
+FILE_SIZE=$((BLK_DEV_SIZE * (512 / (2 + 1))))
+
+cat >$tmp-$seq.fio <<EOF
+# Common e4defrag regression tests
+[global]
+ioengine=ioe_e4defrag
+iodepth=1
+directory=${SCRATCH_MNT}
+filesize=${FILE_SIZE}
+size=999G
+buffered=0
+fadvise_hint=0
+group_reporting
+
+#################################
+# Test1
+# Defragment file while other task does direct io
+
+# Continious sequential defrag activity
+[defrag-4k]
+ioengine=e4defrag
+iodepth=1
+bs=128k
+donorname=test1.def
+filename=test1
+inplace=0
+rw=write
+numjobs=${NUM_JOBS}
+runtime=30*${TIME_FACTOR}
+time_based
+
+# Verifier
+[aio-dio-verifier]
+ioengine=libaio 
+iodepth=128*${LOAD_FACTOR}
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=1
+verifysort=1
+direct=1
+bs=64k
+rw=randwrite
+filename=test1
+runtime=30*${TIME_FACTOR}
+time_based
+EOF
+
+_workout()
+{
+	echo ""
+	echo " Start defragment activity "
+	echo ""
+	cat $tmp-$seq.fio >>  $seq.full
+	run_check $FIO_PROG $tmp-$seq.fio
+}
+
+_require_fio $tmp-$seq.fio
+
+_scratch_mkfs  >> $seq.full 2>&1
+_scratch_mount
+
+if ! _workout; then
+	umount $SCRATCH_DEV 2>/dev/null
+	exit
+fi
+
+if ! _scratch_unmount; then
+	echo "failed to umount"
+	status=1
+	exit
+fi
+_check_scratch_fs
+status=$?
+exit diff --git a/288.out b/288.out new file mode 100644 index 0000000..b215a3f
diff --git a/288.out b/288.out
new file mode 100644
index 0000000..b215a3f
--- /dev/null
+++ b/288.out
@@ -0,0 +1,4 @@
+QA output created by 288
+
+ Start defragment activity 
+
diff --git a/289 b/289
new file mode 100755
index 0000000..8a6c746
--- /dev/null
+++ b/289
@@ -0,0 +1,143 @@
+#! /bin/bash
+# FSQA Test No. 289
+#
+# Ext4 defragmentation stress test
+# Perform defragmentation on file under buffered io
+# while third task does direct io to donor file
+#-----------------------------------------------------------------------
+# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
+owner=dmonakhov@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.defrag
+
+# real QA test starts here
+_supported_os Linux
+_supported_fs ext4
+_need_to_be_root
+_require_scratch
+_require_defrag
+
+BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
+# We need space for 2 files (test file, and donor one)
+# reserve 30% in order to avoid ENOSPC 
+FILE_SIZE=$((BLK_DEV_SIZE * (512 / (2 + 1))))
+
+cat >$tmp-$seq.fio <<EOF
+# Common e4defrag regression tests
+[global]
+ioengine=ioe_e4defrag
+iodepth=1
+directory=${SCRATCH_MNT}
+filesize=${FILE_SIZE}
+size=999G
+buffered=0
+fadvise_hint=0
+group_reporting
+
+##########################################
+# Test2 
+# Perform defragmentation on file under buffered io
+# while third task does direct io to donor file
+#
+# Continious sequential defrag activity
+[defrag-4k]
+stonewall
+ioengine=e4defrag
+iodepth=1
+bs=128k
+donorname=test2.def
+filename=test2
+inplace=0
+rw=write
+numjobs=${LOAD_FACTOR}
+runtime=30*${TIME_FACTOR}
+time_based
+
+# Run DIO/AIO for donor file
+[donor-file-fuzzer]
+ioengine=libaio 
+iodepth=128*${LOAD_FACTOR}
+numjobs=${LOAD_FACTOR}
+verify=0
+direct=1
+bs=64k
+rw=randwrite
+filename=test2.def
+runtime=30*${TIME_FACTOR}
+time_based
+
+# Verifier thread
+[aio-dio-verifier]
+ioengine=libaio 
+iodepth=128*${LOAD_FACTOR}
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=1
+verifysort=1
+buffered=1
+bs=64k
+rw=randrw
+filename=test2
+runtime=30*${TIME_FACTOR}
+time_based
+
+EOF
+
+_workout()
+{
+	echo ""
+	echo " Start defragment activity "
+	echo ""
+	cat $tmp-$seq.fio >>  $seq.full
+	run_check $FIO_PROG $tmp-$seq.fio
+}
+
+_require_fio $tmp-$seq.fio
+
+_scratch_mkfs  >> $seq.full 2>&1
+_scratch_mount
+
+if ! _workout; then
+	umount $SCRATCH_DEV 2>/dev/null
+	exit
+fi
+
+if ! _scratch_unmount; then
+	echo "failed to umount"
+	status=1
+	exit
+fi
+_check_scratch_fs
+status=$?
+exit
diff --git a/289.out b/289.out
new file mode 100644
index 0000000..51b1118
--- /dev/null
+++ b/289.out
@@ -0,0 +1,4 @@
+QA output created by 289
+
+ Start defragment activity 
+
diff --git a/290 b/290
new file mode 100755
index 0000000..68ab4c8
--- /dev/null
+++ b/290
@@ -0,0 +1,153 @@
+#! /bin/bash
+# FSQA Test No. 288
+#
+# Ext4 defragmentation stress test
+# Two defrag tasks use common donor file
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
+owner=dmonakhov@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.defrag
+
+# real QA test starts here
+_supported_os Linux
+_supported_fs ext4
+_need_to_be_root
+_require_scratch
+_require_defrag
+
+BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
+# We need space for 3 files (one donor file and two test files)
+# Reserve space for 4 files in order to avoid ENOSPC
+FILE_SIZE=$((BLK_DEV_SIZE * (512 / (3+1))))
+
+cat >$tmp-$seq.fio <<EOF
+# Common e4defrag regression tests
+[global]
+ioengine=ioe_e4defrag
+iodepth=1
+directory=${SCRATCH_MNT}
+filesize=${FILE_SIZE}
+size=999G
+buffered=0
+fadvise_hint=0
+group_reporting
+
+#################################
+# Test3
+# Two defrag tasks use common donor file
+[defrag-1]
+ioengine=e4defrag
+iodepth=1
+bs=128k
+donorname=test3.def
+filename=test31
+inplace=0
+rw=write
+numjobs=${LOAD_FACTOR}
+runtime=30*${TIME_FACTOR}
+time_based
+
+[defrag-2]
+ioengine=e4defrag
+iodepth=1
+bs=128k
+donorname=test3.def
+filename=test32
+inplace=0
+rw=write
+numjobs=${LOAD_FACTOR}
+runtime=30*${TIME_FACTOR}
+time_based
+
+[aio-dio-verifier-1]
+ioengine=libaio 
+iodepth=128*${LOAD_FACTOR}
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=1
+verifysort=1
+direct=1
+bs=64k
+rw=write
+filename=test31
+runtime=30*${TIME_FACTOR}
+time_based
+
+[aio-buffer-verifier-2]
+ioengine=libaio 
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=1
+verifysort=1
+buffered=1
+bs=64k
+rw=randrw
+filename=test32
+runtime=30*${TIME_FACTOR}
+time_based
+
+EOF
+
+_workout()
+{
+	echo ""
+	echo " Start defragment activity "
+	echo ""
+	cat $tmp-$seq.fio >>  $seq.full
+	run_check $FIO_PROG $tmp-$seq.fio
+}
+
+_require_fio $tmp-$seq.fio
+
+_scratch_mkfs  >> $seq.full 2>&1
+_scratch_mount
+
+if ! _workout; then
+	umount $SCRATCH_DEV 2>/dev/null
+	exit
+fi
+
+if ! _scratch_unmount; then
+	echo "failed to umount"
+	status=1
+	exit
+fi
+_check_scratch_fs
+status=$?
+exit
diff --git a/290.out b/290.out
new file mode 100644
index 0000000..abc8756
--- /dev/null
+++ b/290.out
@@ -0,0 +1,4 @@
+QA output created by 290
+
+ Start defragment activity 
+
diff --git a/291 b/291
new file mode 100755
index 0000000..8a2e537
--- /dev/null
+++ b/291
@@ -0,0 +1,130 @@
+#! /bin/bash
+# FSQA Test No. 288
+#
+# Ext4 defragmentation stress test
+# Several threads pefrorm defragmentatin at random position
+# using 'inplace' mode (allocate and free blocks inside defrag event)
+# which significantly improve load pressure on block allocator.
+#-----------------------------------------------------------------------
+# Copyright (c) 2006 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#-----------------------------------------------------------------------
+#
+# creator
+owner=dmonakhov@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.defrag
+
+# real QA test starts here
+_supported_os Linux
+_supported_fs ext4
+_need_to_be_root
+_require_scratch
+_require_defrag
+
+BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
+# We need space for 2 files (test file, and donor one)
+# reserve 30% in order to avoid ENOSPC 
+FILE_SIZE=$((BLK_DEV_SIZE * (512 / (2 + 1))))
+
+cat >$tmp-$seq.fio <<EOF
+# Common e4defrag regression tests
+[global]
+ioengine=ioe_e4defrag
+iodepth=1
+directory=${SCRATCH_MNT}
+filesize=${FILE_SIZE}
+size=999G
+buffered=0
+fadvise_hint=0
+group_reporting
+
+#################################
+# Test4
+# Stress test defragmentation engine
+# Several threads pefrorm defragmentatin at random position
+# use inplace=1 will allocate and free blocks inside defrag event
+# which highly increase defragmentation
+[defrag-fuzzer]
+ioengine=e4defrag
+iodepth=1
+bs=8k
+donorname=test4.def
+filename=test4
+inplace=1
+rw=randwrite
+numjobs=4*${LOAD_FACTOR}
+runtime=30*${TIME_FACTOR}
+time_based
+
+[aio-dio-verifier]
+ioengine=libaio 
+iodepth=128
+iomem_align=4k
+numjobs=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=1
+verifysort=1
+direct=1
+bs=64k
+rw=write
+filename=test4
+runtime=30*${TIME_FACTOR}
+time_based
+
+EOF
+
+_workout()
+{
+	echo ""
+	echo " Start defragment activity "
+	echo ""
+	cat $tmp-$seq.fio >>  $seq.full
+	run_check $FIO_PROG $tmp-$seq.fio
+}
+
+_require_fio $tmp-$seq.fio
+
+_scratch_mkfs  >> $seq.full 2>&1
+_scratch_mount
+
+if ! _workout; then
+	umount $SCRATCH_DEV 2>/dev/null
+	exit
+fi
+
+if ! _scratch_unmount; then
+	echo "failed to umount"
+	status=1
+	exit
+fi
+_check_scratch_fs
+status=$?
+exit
diff --git a/291.out b/291.out
new file mode 100644
index 0000000..bd6b959
--- /dev/null
+++ b/291.out
@@ -0,0 +1,4 @@
+QA output created by 291
+
+ Start defragment activity 
+
diff --git a/common.defrag b/common.defrag
index ea6c14c..84b1e65 100644
--- a/common.defrag
+++ b/common.defrag
@@ -24,10 +24,10 @@ _require_defrag()
 {
     case "$FSTYP" in
     xfs)
-        DEFRAG_PROG=/usr/sbin/xfs_fsr
+        DEFRAG_PROG="`set_prog_path xfs_fsr`"
 	;;
     ext4|ext4dev)
-        DEFRAG_PROG=/usr/bin/e4defrag
+        DEFRAG_PROG="`set_prog_path e4defrag`"
 	;;
     btrfs)
 	DEFRAG_PROG="$BTRFS_UTIL_PROG filesystem defragment"
diff --git a/group b/group
index fcd9174..b962a33 100644
--- a/group
+++ b/group
@@ -411,3 +411,7 @@ stress
 285 auto dump quota quick
 286 auto rw enospc aio stress
 287 auto rw enospc aio prealloc stress
+288 auto aio dangerous ioctl rw stress
+289 auto aio dangerous ioctl rw stress
+290 auto aio dangerous ioctl rw stress
+291 auto aio dangerous ioctl rw stress
-- 
1.7.1