* [RFC PATCH 1/4] fstests: Add support to check btrfs sysfs features
2016-02-02 2:44 [RFC PATCH 0/4] fstests for basic btrfs inband de-duplication Qu Wenruo
@ 2016-02-02 2:44 ` Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 2/4] fstests: btrfs: Add basic test for btrfs in-band de-duplication Qu Wenruo
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Qu Wenruo @ 2016-02-02 2:44 UTC (permalink / raw)
To: linux-btrfs
Btrfs has its sysfs interface showing what features current kernel/btrfs
module support.
Add _require_btrfs_kernel_feature() to check such interface.
Also rename _require_btrfs() to _require_btrfs_subcommand() to avoid
confusion.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
common/rc | 12 +++++++++++-
tests/btrfs/004 | 2 +-
tests/btrfs/048 | 2 +-
tests/btrfs/059 | 2 +-
4 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/common/rc b/common/rc
index 5135260..f95cd34 100644
--- a/common/rc
+++ b/common/rc
@@ -2583,7 +2583,7 @@ _require_deletable_scratch_dev_pool()
}
# We check for btrfs and (optionally) features of the btrfs command
-_require_btrfs()
+_require_btrfs_subcommand()
{
cmd=$1
_require_command "$BTRFS_UTIL_PROG" btrfs
@@ -2594,6 +2594,16 @@ _require_btrfs()
[ $? -eq 0 ] || _notrun "$BTRFS_UTIL_PROG too old (must support $cmd)"
}
+# We check if the kernel support given btrfs feature from its sysfs interface
+_require_btrfs_kernel_feature()
+{
+ feat=$1
+ # Use /dev/btrfs-control to ensure btrfs is loaded
+ touch /dev/btrfs-control
+ [[ ! -f /sys/fs/btrfs/features/$feat ]] && \
+ _notrun "kernel does not support $feat feature"
+}
+
# Check that fio is present, and it is able to execute given jobfile
_require_fio()
{
diff --git a/tests/btrfs/004 b/tests/btrfs/004
index d588c5b..b5686ec 100755
--- a/tests/btrfs/004
+++ b/tests/btrfs/004
@@ -52,7 +52,7 @@ _supported_fs btrfs
_supported_os Linux
_require_scratch
_require_no_large_scratch_dev
-_require_btrfs inspect-internal
+_require_btrfs_subcommand inspect-internal
_require_command "/usr/sbin/filefrag" filefrag
rm -f $seqres.full
diff --git a/tests/btrfs/048 b/tests/btrfs/048
index dc7386d..9ddd23f 100755
--- a/tests/btrfs/048
+++ b/tests/btrfs/048
@@ -47,7 +47,7 @@ _supported_fs btrfs
_supported_os Linux
_require_test
_require_scratch
-_require_btrfs "property"
+_require_btrfs_subcommand "property"
_need_to_be_root
send_files_dir=$TEST_DIR/btrfs-test-$seq
diff --git a/tests/btrfs/059 b/tests/btrfs/059
index 3379ead..21d246c 100755
--- a/tests/btrfs/059
+++ b/tests/btrfs/059
@@ -50,7 +50,7 @@ _supported_fs btrfs
_supported_os Linux
_require_test
_require_scratch
-_require_btrfs "property"
+_require_btrfs_subcommand "property"
_need_to_be_root
rm -f $seqres.full
--
2.7.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [RFC PATCH 2/4] fstests: btrfs: Add basic test for btrfs in-band de-duplication
2016-02-02 2:44 [RFC PATCH 0/4] fstests for basic btrfs inband de-duplication Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 1/4] fstests: Add support to check btrfs sysfs features Qu Wenruo
@ 2016-02-02 2:44 ` Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 3/4] fstests: btrfs: Add testcase for btrfs dedup enable disable race test Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 4/4] fstests: btrfs: Add per inode dedup flag test Qu Wenruo
3 siblings, 0 replies; 5+ messages in thread
From: Qu Wenruo @ 2016-02-02 2:44 UTC (permalink / raw)
To: linux-btrfs
Add basic test for btrfs in-band de-duplication, including:
1) Enable
2) Re-enable
3) On disk extents are refering to same bytenr
4) Disable
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
common/defrag | 8 ++++
tests/btrfs/200 | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tests/btrfs/200.out | 17 ++++++++
tests/btrfs/group | 1 +
4 files changed, 138 insertions(+)
create mode 100755 tests/btrfs/200
create mode 100644 tests/btrfs/200.out
diff --git a/common/defrag b/common/defrag
index d2b137e..46175f5 100644
--- a/common/defrag
+++ b/common/defrag
@@ -47,6 +47,14 @@ _extent_count()
$XFS_IO_PROG -c "fiemap" $1 | tail -n +2 | grep -v hole | wc -l| $AWK_PROG '{print $1}'
}
+_uniq_extent_count()
+{
+ file=$1
+ $XFS_IO_PROG -c "fiemap" $file >> $seqres.full 2>&1
+ $XFS_IO_PROG -c "fiemap" $file | tail -n +2 | grep -v hole |\
+ $AWK_PROG '{print $3}' | sort | uniq | wc -l
+}
+
_check_extent_count()
{
min=$1
diff --git a/tests/btrfs/200 b/tests/btrfs/200
new file mode 100755
index 0000000..856e1fb
--- /dev/null
+++ b/tests/btrfs/200
@@ -0,0 +1,112 @@
+#! /bin/bash
+# FS QA Test 200
+#
+# Basic btrfs inband dedup test, including:
+# 1) Enable
+# 2) Uniq file extent number
+# 3) Re-enable
+# 4) Disable
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2016 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/defrag
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_need_to_be_root
+_require_btrfs_subcommand dedup
+_require_btrfs_kernel_feature dedup
+_require_btrfs_mkfs_feature dedup
+
+# File size is twice the maximum file extent of btrfs
+# So even fallbacked to non-dedup, it will have at least 2 extents
+file_size=$(( 256 * 1024 * 1024 ))
+
+_scratch_mkfs "-O dedup" >> $seqres.full 2>&1
+_scratch_mount
+
+do_dedup_test()
+{
+ backend=$1
+ dedup_bs=$2
+ _run_btrfs_util_prog dedup enable -s $backend -b $dedup_bs $SCRATCH_MNT
+ $XFS_IO_PROG -f -c "pwrite -b $dedup_bs 0 $dedup_bs" \
+ $SCRATCH_MNT/initial_block | _filter_xfs_io
+
+ # sync to ensure dedup hash is added into dedup pool
+ sync
+ $XFS_IO_PROG -f -c "pwrite -b $dedup_bs 0 $file_size" \
+ $SCRATCH_MNT/real_file | _filter_xfs_io
+ # sync again to ensure data are all rewriten to disk
+ sync
+
+ # Test if real_file is de-duplicated
+ nr_uniq_extents=$(_uniq_extent_count $SCRATCH_MNT/real_file)
+ nr_total_extents=$(_extent_count $SCRATCH_MNT/real_file)
+
+ echo "uniq/total: $nr_uniq_extents/$nr_total_extents" >> $seqres.full
+ # Allow a small amount of dedup miss, as commit interval or
+ # memory pressure may break a dedup_bs block and cause
+ # smalll extent which won't go through dedup routine
+ if [ $nr_uniq_extents -ge $(( $nr_total_extents * 5 / 100 )) ]; then
+ echo "Too high dedup failure rate"
+ fi
+}
+
+# Test inmemory dedup first, use 64K dedup bs to keep compatibility
+# with 64K page size
+do_dedup_test inmemory 64K
+
+# Test ondisk backend, and re-enable function
+do_dedup_test ondisk 64K
+
+# Test 128K(default) dedup bs
+do_dedup_test inmemory 128K
+do_dedup_test ondisk 128K
+
+# Check dedup disable
+_run_btrfs_util_prog dedup disable $SCRATCH_MNT
+
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/200.out b/tests/btrfs/200.out
new file mode 100644
index 0000000..5197dbc
--- /dev/null
+++ b/tests/btrfs/200.out
@@ -0,0 +1,17 @@
+QA output created by 200
+wrote 65536/65536 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/btrfs/group b/tests/btrfs/group
index de628be..b54ee6e 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -117,3 +117,4 @@
114 auto qgroup
115 auto qgroup
116 auto quick metadata
+200 auto dedup
--
2.7.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [RFC PATCH 3/4] fstests: btrfs: Add testcase for btrfs dedup enable disable race test
2016-02-02 2:44 [RFC PATCH 0/4] fstests for basic btrfs inband de-duplication Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 1/4] fstests: Add support to check btrfs sysfs features Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 2/4] fstests: btrfs: Add basic test for btrfs in-band de-duplication Qu Wenruo
@ 2016-02-02 2:44 ` Qu Wenruo
2016-02-02 2:44 ` [RFC PATCH 4/4] fstests: btrfs: Add per inode dedup flag test Qu Wenruo
3 siblings, 0 replies; 5+ messages in thread
From: Qu Wenruo @ 2016-02-02 2:44 UTC (permalink / raw)
To: linux-btrfs
Add test case to check btrfs dedup enable/disable race.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
tests/btrfs/201 | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++
tests/btrfs/201.out | 1 +
tests/btrfs/group | 1 +
3 files changed, 99 insertions(+)
create mode 100755 tests/btrfs/201
create mode 100644 tests/btrfs/201.out
diff --git a/tests/btrfs/201 b/tests/btrfs/201
new file mode 100755
index 0000000..9cbded4
--- /dev/null
+++ b/tests/btrfs/201
@@ -0,0 +1,97 @@
+#! /bin/bash
+# FS QA Test 201
+#
+# Basic btrfs inband dedup enable/disable race test
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2016 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_need_to_be_root
+_require_btrfs_subcommand dedup
+_require_btrfs_kernel_feature dedup
+_require_btrfs_mkfs_feature dedup
+
+# Use 64K dedup size to keep compatibility for 64K page size
+dedup_bs=64K
+
+_scratch_mkfs "-O dedup" >> $seqres.full 2>&1
+_scratch_mount
+
+mkdir -p $SCRATCH_MNT/stressdir
+
+fsstress_work()
+{
+ $FSSTRESS_PROG $(_scale_fsstress_args -p 8 -n 5000) $FSSTRESS_AVOID \
+ -d $SCRATCH_MNT/stressdir > /dev/null 2>&1
+}
+
+trigger_work()
+{
+ for i in $(seq 1 300); do
+ _run_btrfs_util_prog dedup enable -s inmemory \
+ -b $dedup_bs $SCRATCH_MNT
+ sleep 5
+ _run_btrfs_util_prog dedup disable $SCRATCH_MNT
+ sleep 5
+ _run_btrfs_util_prog dedup enable -s ondisk \
+ -b $dedup_bs $SCRATCH_MNT
+ sleep 5
+ _run_btrfs_util_prog dedup disable $SCRATCH_MNT
+ sleep 5
+ done
+}
+
+fsstress_work &
+fsstress_pid=$!
+
+trigger_work &
+trigger_pid=$!
+
+wait $fsstress_pid
+kill $trigger_pid
+
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/201.out b/tests/btrfs/201.out
new file mode 100644
index 0000000..b8969af
--- /dev/null
+++ b/tests/btrfs/201.out
@@ -0,0 +1 @@
+QA output created by 201
diff --git a/tests/btrfs/group b/tests/btrfs/group
index b54ee6e..8159148 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -118,3 +118,4 @@
115 auto qgroup
116 auto quick metadata
200 auto dedup
+201 auto dedup
--
2.7.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [RFC PATCH 4/4] fstests: btrfs: Add per inode dedup flag test
2016-02-02 2:44 [RFC PATCH 0/4] fstests for basic btrfs inband de-duplication Qu Wenruo
` (2 preceding siblings ...)
2016-02-02 2:44 ` [RFC PATCH 3/4] fstests: btrfs: Add testcase for btrfs dedup enable disable race test Qu Wenruo
@ 2016-02-02 2:44 ` Qu Wenruo
3 siblings, 0 replies; 5+ messages in thread
From: Qu Wenruo @ 2016-02-02 2:44 UTC (permalink / raw)
To: linux-btrfs
This test will check per inode dedup flag.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
tests/btrfs/202 | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tests/btrfs/202.out | 15 +++++++
tests/btrfs/group | 1 +
3 files changed, 133 insertions(+)
create mode 100755 tests/btrfs/202
create mode 100644 tests/btrfs/202.out
diff --git a/tests/btrfs/202 b/tests/btrfs/202
new file mode 100755
index 0000000..92143fb
--- /dev/null
+++ b/tests/btrfs/202
@@ -0,0 +1,117 @@
+#! /bin/bash
+# FS QA Test 202
+#
+# Btrfs per inode dedup flag test
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2016 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/defrag
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_need_to_be_root
+_require_btrfs_subcommand dedup
+_require_btrfs_subcommand property
+_require_btrfs_kernel_feature dedup
+_require_btrfs_mkfs_feature dedup
+
+# File size is twice the maximum file extent of btrfs
+# So even fallbacked to non-dedup, it will have at least 2 extents
+file_size=$(( 256 * 1024 * 1024 ))
+dedup_bs=$(( 64 * 1024 ))
+
+_scratch_mkfs "-O dedup" >> $seqres.full 2>&1
+_scratch_mount
+
+# Return 0 for not deduped at all , return 1 for part or full deduped
+test_file_deduped () {
+ file=$1
+
+ nr_uniq_extents=$(_uniq_extent_count $file)
+ nr_total_extents=$(_extent_count $file)
+
+ if [ $nr_uniq_extents -eq $nr_total_extents ]; then
+ echo "not de-duplicated"
+ else
+ echo "de-duplicated"
+ fi
+}
+
+dedup_write_file () {
+ file=$1
+ size=$2
+
+ $XFS_IO_PROG -f -c "pwrite -b $dedup_bs 0 $size" $file | _filter_xfs_io
+}
+
+print_result () {
+ file=$1
+
+ echo "$file: $(test_file_deduped $file)"
+}
+_run_btrfs_util_prog dedup enable -b $dedup_bs $SCRATCH_MNT
+touch $SCRATCH_MNT/dedup_file
+touch $SCRATCH_MNT/no_dedup_file
+mkdir $SCRATCH_MNT/dedup_dir
+mkdir $SCRATCH_MNT/no_dedup_dir
+
+_run_btrfs_util_prog property set $SCRATCH_MNT/no_dedup_file dedup disable
+_run_btrfs_util_prog property set $SCRATCH_MNT/no_dedup_dir dedup disable
+
+dedup_write_file $SCRATCH_MNT/tmp $dedup_bs
+# sync to ensure hash is added to dedup tree
+sync
+
+dedup_write_file $SCRATCH_MNT/dedup_file $file_size
+dedup_write_file $SCRATCH_MNT/no_dedup_file $file_size
+dedup_write_file $SCRATCH_MNT/dedup_dir/dedup_file $file_size
+dedup_write_file $SCRATCH_MNT/no_dedup_dir/no_dedup_file $file_size
+
+print_result $SCRATCH_MNT/dedup_file
+print_result $SCRATCH_MNT/no_dedup_file
+print_result $SCRATCH_MNT/dedup_dir/dedup_file
+print_result $SCRATCH_MNT/no_dedup_dir/no_dedup_file
+
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/202.out b/tests/btrfs/202.out
new file mode 100644
index 0000000..cc18013
--- /dev/null
+++ b/tests/btrfs/202.out
@@ -0,0 +1,15 @@
+QA output created by 202
+wrote 65536/65536 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+/mnt/scratch/dedup_file: de-duplicated
+/mnt/scratch/no_dedup_file: not de-duplicated
+/mnt/scratch/dedup_dir/dedup_file: de-duplicated
+/mnt/scratch/no_dedup_dir/no_dedup_file: not de-duplicated
diff --git a/tests/btrfs/group b/tests/btrfs/group
index 8159148..62bb414 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -119,3 +119,4 @@
116 auto quick metadata
200 auto dedup
201 auto dedup
+202 auto dedup
--
2.7.0
^ permalink raw reply related [flat|nested] 5+ messages in thread