[PATCH] xfstests: btrfs/047: check btrfs-convert with extent and non-extent source

[PATCH] xfstests: btrfs/047: check btrfs-convert with extent and non-extent source

[Lakshmipathi.G]

This is used to check the source which contains combination of Ext3 files
in non-extent format and  Ext4 extent-files. And validate the file md5sums
before and after conversion.

btrfs/012: BTRFS_CONVERT_PROG,E2FSCK_PROG definitions reused from common/config

Signed-off-by: Lakshmipathi.G <Lakshmipathi.G@giis.co.in>
---
 common/config       |   3 ++
 tests/btrfs/012     |   3 --
 tests/btrfs/047     | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/btrfs/047.out |   2 +
 tests/btrfs/group   |   1 +
 5 files changed, 128 insertions(+), 3 deletions(-)
 create mode 100755 tests/btrfs/047
 create mode 100644 tests/btrfs/047.out

diff --git a/common/config b/common/config
index 0706aca..fa89f42 100644
--- a/common/config
+++ b/common/config
@@ -240,11 +240,14 @@ case "$HOSTOS" in
         export DUMP_F2FS_PROG="`set_prog_path dump.f2fs`"
         export BTRFS_UTIL_PROG="`set_prog_path btrfs`"
         export BTRFS_SHOW_SUPER_PROG="`set_prog_path btrfs-show-super`"
+	export BTRFS_CONVERT_PROG="`set_prog_path btrfs-convert`"
         export XFS_FSR_PROG="`set_prog_path xfs_fsr`"
         export MKFS_NFS_PROG="false"
         export MKFS_CIFS_PROG="false"
         export MKFS_OVERLAY_PROG="false"
         export MKFS_REISER4_PROG="`set_prog_path mkfs.reiser4`"
+	export E2FSCK_PROG="`set_prog_path e2fsck`"
+	export TUNE2FS_PROG="`set_prog_path tune2fs`"
         ;;
 esac
 
diff --git a/tests/btrfs/012 b/tests/btrfs/012
index 6a3cb81..85c82f0 100755
--- a/tests/btrfs/012
+++ b/tests/btrfs/012
@@ -54,9 +54,6 @@ _supported_fs btrfs
 _supported_os Linux
 _require_scratch_nocheck
 
-BTRFS_CONVERT_PROG="`set_prog_path btrfs-convert`"
-E2FSCK_PROG="`set_prog_path e2fsck`"
-
 _require_command "$BTRFS_CONVERT_PROG" btrfs-convert
 _require_command "$MKFS_EXT4_PROG" mkfs.ext4
 _require_command "$E2FSCK_PROG" e2fsck
diff --git a/tests/btrfs/047 b/tests/btrfs/047
new file mode 100755
index 0000000..d349d12
--- /dev/null
+++ b/tests/btrfs/047
@@ -0,0 +1,122 @@
+#! /bin/bash
+# FS QA Test 047
+#
+# Test btrfs-convert
+#
+# 1) create ext3 filesystem & populate it.
+# 2) upgrade ext3 filesystem to ext4.
+# 3) populate data.
+# 4) source has combination of non-extent and extent files.
+# 5) convert it to btrfs, mount and verify contents.
+#-----------------------------------------------------------------------
+# Copyright (c) 2017 Lakshmipathi.G  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch_nocheck
+
+_require_command "$BTRFS_CONVERT_PROG" btrfs-convert
+_require_command "$MKFS_EXT4_PROG" mkfs.ext4
+_require_command "$E2FSCK_PROG" e2fsck
+_require_command "$TUNE2FS_PROG" tune2fs
+
+rm -f $seqres.full
+
+BLOCK_SIZE=`_get_block_size $TEST_DIR`
+EXT_MD5SUM="$tmp.ext43"
+BTRFS_MD5SUM="$tmp.btrfs"
+
+_populate_data(){
+	data_path=$1
+	mkdir -p $data_path
+	args=`_scale_fsstress_args -p 20 -n 100 $FSSTRESS_AVOID -d $data_path`
+	echo "Run fsstress $args" >>$seqres.full
+	$FSSTRESS_PROG $args >/dev/null 2>&1 &
+	fsstress_pid=$!
+	wait $fsstress_pid
+}
+
+# Create & populate an ext3 filesystem
+$MKFS_EXT4_PROG -F -t ext3 -b $BLOCK_SIZE $SCRATCH_DEV > $seqres.full 2>&1 || \
+	_notrun "Could not create ext3 filesystem"
+
+# mount and populate non-extent file
+mount -t ext3 $SCRATCH_DEV $SCRATCH_MNT
+_populate_data "$SCRATCH_MNT/ext3_ext4_data/ext3"
+_scratch_unmount
+
+# Upgrade it to ext4.
+$TUNE2FS_PROG -O extents,uninit_bg,dir_index $SCRATCH_DEV >> $seqres.full 2>&1
+# After Conversion, its highly recommended to run e2fsck.
+$E2FSCK_PROG -fyD $SCRATCH_DEV >> $seqres.full 2>&1
+
+# mount and populate extent file
+mount -t ext4 $SCRATCH_DEV $SCRATCH_MNT
+_populate_data "$SCRATCH_MNT/ext3_ext4_data/ext4"
+
+# Compute md5 of ext3,ext4 files.
+find "$SCRATCH_MNT/ext3_ext4_data" -type f -fprint "$EXT_MD5SUM"
+sort "$EXT_MD5SUM" -o "$EXT_MD5SUM"
+_scratch_unmount
+
+# Convert non-extent & extent data to btrfs, mount it, verify the data
+$BTRFS_CONVERT_PROG $SCRATCH_DEV >> $seqres.full 2>&1 || \
+	_fail "btrfs-convert failed"
+_scratch_mount || _fail "Could not mount new btrfs fs"
+
+# Compute md5 for converted files.
+find "$SCRATCH_MNT/ext3_ext4_data" -type f -fprint "$BTRFS_MD5SUM"
+sort "$BTRFS_MD5SUM" -o "$BTRFS_MD5SUM"
+
+# Compare two md5sum files.
+btrfs_perm=`md5sum "$BTRFS_MD5SUM" | cut -f1 -d' '`
+ext_perm=`md5sum "$EXT_MD5SUM" | cut -f1 -d' '`
+
+if [ "$btrfs_perm" != "$ext_perm" ];then
+       echo "md5sum mismatch"
+fi
+
+# success, all done
+echo "Silence is golden"
+status=0
+exit
diff --git a/tests/btrfs/047.out b/tests/btrfs/047.out
new file mode 100644
index 0000000..48db9bf
--- /dev/null
+++ b/tests/btrfs/047.out
@@ -0,0 +1,2 @@
+QA output created by 047
+Silence is golden
diff --git a/tests/btrfs/group b/tests/btrfs/group
index 3fbf706..224a082 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -49,6 +49,7 @@
 044 auto quick send
 045 auto quick send
 046 auto quick send
+047 auto convert
 048 auto quick
 049 auto quick
 050 auto quick send
-- 
2.7.4

[PATCH] xfstests: btrfs/047: check btrfs-convert with extent and non-extent source

[Lakshmipathi.G]

Signed-off-by: Lakshmipathi.G <Lakshmipathi.G@giis.co.in>
---
 tests/btrfs/047     | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/btrfs/047.out |   1 +
 tests/btrfs/group   |   1 +
 3 files changed, 110 insertions(+)
 create mode 100755 tests/btrfs/047
 create mode 100644 tests/btrfs/047.out

diff --git a/tests/btrfs/047 b/tests/btrfs/047
new file mode 100755
index 0000000..0c4b2c7
--- /dev/null
+++ b/tests/btrfs/047
@@ -0,0 +1,108 @@
+#! /bin/bash
+# FS QA Test 047
+#
+# Test btrfs-convert
+# 
+# 1) create ext3 filesystem & populate it.
+# 2) update ext3 filesystem to ext4.
+# 3) populate data.
+# 4) source has combination of non-extent and extent files.
+# 5) convert it btrfs, mount and verify contents.
+#-----------------------------------------------------------------------
+# Copyright (c) 2017 Lakshmipathi.G  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch_nocheck
+
+BTRFS_CONVERT_PROG="`set_prog_path btrfs-convert`"
+E2FSCK_PROG="`set_prog_path e2fsck`"
+TUNE2FS_PROG="`set_prog_path tune2fs`"
+
+_require_command "$BTRFS_CONVERT_PROG" btrfs-convert
+_require_command "$MKFS_EXT4_PROG" mkfs.ext4
+_require_command "$E2FSCK_PROG" e2fsck
+_require_command "$TUNE2FS_PROG" tune2fs
+
+rm -f $seqres.full
+
+BLOCK_SIZE=`_get_block_size $TEST_DIR`
+
+# Create & populate an ext3 filesystem
+$MKFS_EXT4_PROG -t ext3 -b $BLOCK_SIZE $SCRATCH_DEV > $seqres.full 2>&1 || \
+	_notrun "Could not create ext3 filesystem"
+
+# mount and populate non-extent file
+mount -t ext3 $SCRATCH_DEV $SCRATCH_MNT
+dd if=/dev/urandom of=$SCRATCH_MNT/f1.txt bs=1MB count=10 >> $seqres.full 2>&1
+NON_EXTENT_MD5=`md5sum $SCRATCH_MNT/f1.txt  | awk '{print $1}' `
+_scratch_unmount
+
+# Upgrade it to ext4.
+$TUNE2FS_PROG -O extents,uninit_bg,dir_index $SCRATCH_DEV >> $seqres.full 2>&1
+$E2FSCK_PROG -fyD $SCRATCH_DEV >> $seqres.full 2>&1
+
+# mount and populate extent file
+mount -t ext4 $SCRATCH_DEV $SCRATCH_MNT
+dd if=/dev/urandom of=$SCRATCH_MNT/f2.txt bs=1MB count=10 >> $seqres.full 2>&1
+EXTENT_MD5=`md5sum $SCRATCH_MNT/f2.txt  | awk '{print $1}'`
+_scratch_unmount
+
+# Convert non-extent & extent data to btrfs, mount it, verify the data
+$BTRFS_CONVERT_PROG $SCRATCH_DEV >> $seqres.full 2>&1 || \
+	_fail "btrfs-convert failed"
+_scratch_mount || _fail "Could not mount new btrfs fs"
+
+F1_MD5=`md5sum $SCRATCH_MNT/f1.txt  | awk '{print $1}'`
+F2_MD5=`md5sum $SCRATCH_MNT/f2.txt  | awk '{print $1}'`
+if [ $NON_EXTENT_MD5 != $F1_MD5 ] ; then 
+_fail "ext3 file mismatch."
+fi
+
+if [ $EXTENT_MD5 != $F2_MD5 ] ; then 
+_fail "ext4 file mismatch."
+fi
+   
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/047.out b/tests/btrfs/047.out
new file mode 100644
index 0000000..58e2353
--- /dev/null
+++ b/tests/btrfs/047.out
@@ -0,0 +1 @@
+QA output created by 047
diff --git a/tests/btrfs/group b/tests/btrfs/group
index 3fbf706..224a082 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -49,6 +49,7 @@
 044 auto quick send
 045 auto quick send
 046 auto quick send
+047 auto convert
 048 auto quick
 049 auto quick
 050 auto quick send
-- 
2.7.4

Re: [PATCH] xfstests: btrfs/047: check btrfs-convert with extent and non-extent source

[Eryu Guan]

On Wed, Jan 18, 2017 at 07:17:02AM +0530, Lakshmipathi.G wrote:
> Signed-off-by: Lakshmipathi.G <Lakshmipathi.G@giis.co.in>

Need detailed test description in commit log too.

> ---
>  tests/btrfs/047     | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/btrfs/047.out |   1 +
>  tests/btrfs/group   |   1 +
>  3 files changed, 110 insertions(+)
>  create mode 100755 tests/btrfs/047
>  create mode 100644 tests/btrfs/047.out
> 
> diff --git a/tests/btrfs/047 b/tests/btrfs/047
> new file mode 100755
> index 0000000..0c4b2c7
> --- /dev/null
> +++ b/tests/btrfs/047
> @@ -0,0 +1,108 @@
> +#! /bin/bash
> +# FS QA Test 047
> +#
> +# Test btrfs-convert
> +# 

Trailing whitespace in above line.

> +# 1) create ext3 filesystem & populate it.
> +# 2) update ext3 filesystem to ext4.
> +# 3) populate data.
> +# 4) source has combination of non-extent and extent files.
> +# 5) convert it btrfs, mount and verify contents.
> +#-----------------------------------------------------------------------
> +# Copyright (c) 2017 Lakshmipathi.G  All Rights Reserved.
> +#
> +# This program is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU General Public License as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it would be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write the Free Software Foundation,
> +# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
> +#-----------------------------------------------------------------------
> +#
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1	# failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +
> +# remove previous $seqres.full before test
> +rm -f $seqres.full
> +
> +# real QA test starts here
> +
> +# Modify as appropriate.
> +_supported_fs btrfs
> +_supported_os Linux
> +_require_scratch_nocheck
> +
> +BTRFS_CONVERT_PROG="`set_prog_path btrfs-convert`"
> +E2FSCK_PROG="`set_prog_path e2fsck`"
> +TUNE2FS_PROG="`set_prog_path tune2fs`"

These should go to common/config. Can you please update btrfs/012 as
well to move such defines to common/config?

> +
> +_require_command "$BTRFS_CONVERT_PROG" btrfs-convert
> +_require_command "$MKFS_EXT4_PROG" mkfs.ext4
> +_require_command "$E2FSCK_PROG" e2fsck
> +_require_command "$TUNE2FS_PROG" tune2fs
> +
> +rm -f $seqres.full
> +
> +BLOCK_SIZE=`_get_block_size $TEST_DIR`
> +
> +# Create & populate an ext3 filesystem
> +$MKFS_EXT4_PROG -t ext3 -b $BLOCK_SIZE $SCRATCH_DEV > $seqres.full 2>&1 || \
> +	_notrun "Could not create ext3 filesystem"

Better to add "-F" option to mkfs to force mkfs so it won't stop when
there's an existing fs on SCRATCH_DEV.

> +
> +# mount and populate non-extent file
> +mount -t ext3 $SCRATCH_DEV $SCRATCH_MNT
> +dd if=/dev/urandom of=$SCRATCH_MNT/f1.txt bs=1MB count=10 >> $seqres.full 2>&1
> +NON_EXTENT_MD5=`md5sum $SCRATCH_MNT/f1.txt  | awk '{print $1}' `

Better to have different files with different file sizes and different
types, e.g. run fsstress to create such a fs structure.

> +_scratch_unmount
> +
> +# Upgrade it to ext4.
> +$TUNE2FS_PROG -O extents,uninit_bg,dir_index $SCRATCH_DEV >> $seqres.full 2>&1
> +$E2FSCK_PROG -fyD $SCRATCH_DEV >> $seqres.full 2>&1

Why is this e2fsck needed? Add some comments? Or it just can be removed?

> +
> +# mount and populate extent file
> +mount -t ext4 $SCRATCH_DEV $SCRATCH_MNT
> +dd if=/dev/urandom of=$SCRATCH_MNT/f2.txt bs=1MB count=10 >> $seqres.full 2>&1
> +EXTENT_MD5=`md5sum $SCRATCH_MNT/f2.txt  | awk '{print $1}'`
> +_scratch_unmount
> +
> +# Convert non-extent & extent data to btrfs, mount it, verify the data
> +$BTRFS_CONVERT_PROG $SCRATCH_DEV >> $seqres.full 2>&1 || \
> +	_fail "btrfs-convert failed"
> +_scratch_mount || _fail "Could not mount new btrfs fs"
> +
> +F1_MD5=`md5sum $SCRATCH_MNT/f1.txt  | awk '{print $1}'`
> +F2_MD5=`md5sum $SCRATCH_MNT/f2.txt  | awk '{print $1}'`
> +if [ $NON_EXTENT_MD5 != $F1_MD5 ] ; then 

Trailing whitespace in above line.

> +_fail "ext3 file mismatch."

No need to _fail, just echo this message to break golden image.

And need indention inside "if-then-fi"

> +fi
> +
> +if [ $EXTENT_MD5 != $F2_MD5 ] ; then 

Trailing whitespace.

> +_fail "ext4 file mismatch."

Same here. Use echo and indention.

> +fi
> +   

Trailing whitespace.

> +# success, all done
> +status=0
> +exit
> diff --git a/tests/btrfs/047.out b/tests/btrfs/047.out
> new file mode 100644
> index 0000000..58e2353
> --- /dev/null
> +++ b/tests/btrfs/047.out
> @@ -0,0 +1 @@
> +QA output created by 047

Usually we print a message "Silence is golden" to indicate that this
test doesn't expect any output.

Thanks,
Eryu

> diff --git a/tests/btrfs/group b/tests/btrfs/group
> index 3fbf706..224a082 100644
> --- a/tests/btrfs/group
> +++ b/tests/btrfs/group
> @@ -49,6 +49,7 @@
>  044 auto quick send
>  045 auto quick send
>  046 auto quick send
> +047 auto convert
>  048 auto quick
>  049 auto quick
>  050 auto quick send
> -- 
> 2.7.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe fstests" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] xfstests: btrfs/047: check btrfs-convert with extent and non-extent source

[Lakshmipathi.G]

On Wed, Jan 18, 2017 at 02:39:53PM +0800, Eryu Guan wrote:
> On Wed, Jan 18, 2017 at 07:17:02AM +0530, Lakshmipathi.G wrote:
> > Signed-off-by: Lakshmipathi.G <Lakshmipathi.G@giis.co.in>
> 
> Need detailed test description in commit log too.

Okay will include them.

> 
> 
> Trailing whitespace in above line.
> 

> > +BTRFS_CONVERT_PROG="`set_prog_path btrfs-convert`"
> > +E2FSCK_PROG="`set_prog_path e2fsck`"
> > +TUNE2FS_PROG="`set_prog_path tune2fs`"
> 
> These should go to common/config. Can you please update btrfs/012 as
> well to move such defines to common/config?
> 

> > +# Create & populate an ext3 filesystem
> > +$MKFS_EXT4_PROG -t ext3 -b $BLOCK_SIZE $SCRATCH_DEV > $seqres.full 2>&1 || \
> > +	_notrun "Could not create ext3 filesystem"
> 
> Better to add "-F" option to mkfs to force mkfs so it won't stop when
> there's an existing fs on SCRATCH_DEV.
> 
> > +
> > +# mount and populate non-extent file
> > +mount -t ext3 $SCRATCH_DEV $SCRATCH_MNT
> > +dd if=/dev/urandom of=$SCRATCH_MNT/f1.txt bs=1MB count=10 >> $seqres.full 2>&1
> > +NON_EXTENT_MD5=`md5sum $SCRATCH_MNT/f1.txt  | awk '{print $1}' `
> 
> Better to have different files with different file sizes and different
> types, e.g. run fsstress to create such a fs structure.
> 

Will look into fsstress tool or else simply copy files from hosts machine. (ex: /usr/lib)

> > +_scratch_unmount
> > +
> > +# Upgrade it to ext4.
> > +$TUNE2FS_PROG -O extents,uninit_bg,dir_index $SCRATCH_DEV >> $seqres.full 2>&1
> > +$E2FSCK_PROG -fyD $SCRATCH_DEV >> $seqres.full 2>&1
> 
> Why is this e2fsck needed? Add some comments? Or it just can be removed?
> 
It recommended to run e2fsck after conversion as per https://ext4.wiki.kernel.org/index.php/UpgradeToExt4

> > +
> > +if [ $NON_EXTENT_MD5 != $F1_MD5 ] ; then 
> 
> Trailing whitespace in above line.
> 
> > +_fail "ext3 file mismatch."
> 
> No need to _fail, just echo this message to break golden image.
> 
> And need indention inside "if-then-fi"
> 
> > +fi
> > +
> > +if [ $EXTENT_MD5 != $F2_MD5 ] ; then 
> 
> Trailing whitespace.
> 
> > +_fail "ext4 file mismatch."
> 
> Same here. Use echo and indention.
> 
> > +fi
> > +   
> 
> Trailing whitespace.
> 
> > +# success, all done
> > +status=0
> > +exit
> > diff --git a/tests/btrfs/047.out b/tests/btrfs/047.out
> > new file mode 100644
> > index 0000000..58e2353
> > --- /dev/null
> > +++ b/tests/btrfs/047.out
> > @@ -0,0 +1 @@
> > +QA output created by 047
> 
> Usually we print a message "Silence is golden" to indicate that this
> test doesn't expect any output.
> 
> Thanks,
> Eryu
> 
Will modify the script to reflect above review comments.

Cheers.
Lakshmipathi.G