* [PATCH 1/3] fstests: btrfs: Add basic test for btrfs in-band de-duplication
2017-03-16 1:50 [PATCH 0/3] Btrfs in-band de-duplication test cases Qu Wenruo
@ 2017-03-16 1:50 ` Qu Wenruo
2017-03-16 8:08 ` Eryu Guan
2017-03-16 1:50 ` [PATCH] fstests: generic: Test space allocation when there is only fragmented space Qu Wenruo
` (3 subsequent siblings)
4 siblings, 1 reply; 11+ messages in thread
From: Qu Wenruo @ 2017-03-16 1:50 UTC (permalink / raw)
To: linux-btrfs; +Cc: fstests
Add basic test for btrfs in-band de-duplication(inmemory backend), including:
1) Enable
3) Dedup rate
4) File correctness
5) Disable
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
common/defrag | 13 ++++++
tests/btrfs/200 | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tests/btrfs/200.out | 22 ++++++++++
tests/btrfs/group | 2 +
4 files changed, 153 insertions(+)
create mode 100755 tests/btrfs/200
create mode 100644 tests/btrfs/200.out
diff --git a/common/defrag b/common/defrag
index d279382f..0a41714f 100644
--- a/common/defrag
+++ b/common/defrag
@@ -59,6 +59,19 @@ _extent_count()
$XFS_IO_PROG -c "fiemap" $1 | tail -n +2 | grep -v hole | wc -l| $AWK_PROG '{print $1}'
}
+# Get the number of unique file extents
+# Unique file extents means they have different ondisk bytenr
+# Some filesystem supports reflinkat() or in-band de-dup can create
+# a file whose all file extents points to the same ondisk bytenr
+# this can be used to test if such reflinkat() or in-band de-dup works
+_extent_count_uniq()
+{
+ file=$1
+ $XFS_IO_PROG -c "fiemap" $file >> $seqres.full 2>&1
+ $XFS_IO_PROG -c "fiemap" $file | tail -n +2 | grep -v hole |\
+ $AWK_PROG '{print $3}' | sort | uniq | wc -l
+}
+
_check_extent_count()
{
min=$1
diff --git a/tests/btrfs/200 b/tests/btrfs/200
new file mode 100755
index 00000000..1b3e46fd
--- /dev/null
+++ b/tests/btrfs/200
@@ -0,0 +1,116 @@
+#! /bin/bash
+# FS QA Test 200
+#
+# Basic btrfs inband dedupe test for inmemory backend
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2016 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/defrag
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_require_btrfs_command dedupe
+_require_btrfs_fs_feature dedupe
+
+# File size is twice the maximum file extent of btrfs
+# So even fallbacked to non-dedupe, it will have at least 2 extents
+file_size=256m
+
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+do_dedupe_test()
+{
+ dedupe_bs=$1
+
+ echo "Testing inmemory dedupe backend with block size $dedupe_bs"
+ _run_btrfs_util_prog dedupe enable -f -s inmemory -b $dedupe_bs \
+ $SCRATCH_MNT
+ # do sync write to ensure dedupe hash is added into dedupe pool
+ $XFS_IO_PROG -f -c "pwrite -b $dedupe_bs 0 $dedupe_bs" -c "fsync"\
+ $SCRATCH_MNT/initial_block | _filter_xfs_io
+
+ # do sync write to ensure we can get stable fiemap later
+ $XFS_IO_PROG -f -c "pwrite -b $dedupe_bs 0 $file_size" -c "fsync"\
+ $SCRATCH_MNT/real_file | _filter_xfs_io
+
+ # Test if real_file is de-duplicated
+ nr_uniq_extents=$(_extent_count_uniq $SCRATCH_MNT/real_file)
+ nr_total_extents=$(_extent_count $SCRATCH_MNT/real_file)
+ nr_deduped_extents=$(($nr_total_extents - $nr_uniq_extents))
+
+ echo "deduped/total: $nr_deduped_extents/$nr_total_extents" \
+ >> $seqres.full
+ # Allow a small amount of dedupe miss, as commit interval or
+ # memory pressure may break a dedupe_bs block and cause
+ # small extent which won't go through dedupe routine
+ _within_tolerance "number of deduped extents" $nr_deduped_extents \
+ $nr_total_extents 5% -v
+
+ # Also check the md5sum to ensure data is not corrupted
+ md5=$(_md5_checksum $SCRATCH_MNT/real_file)
+ echo "md5sum: $md5"
+}
+
+# Test inmemory dedupe first, use 64K dedupe bs to keep compatibility
+# with 64K page size
+do_dedupe_test 64K
+
+# Test 128K(default) dedupe bs
+do_dedupe_test 128K
+
+# Test 1M dedupe bs
+do_dedupe_test 1M
+
+# Check dedupe disable
+_run_btrfs_util_prog dedupe disable $SCRATCH_MNT
+
+# success, all done
+status=0
+exit
+# Check dedupe disable
+_run_btrfs_util_prog dedupe disable $SCRATCH_MNT
+
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/200.out b/tests/btrfs/200.out
new file mode 100644
index 00000000..e09e5733
--- /dev/null
+++ b/tests/btrfs/200.out
@@ -0,0 +1,22 @@
+QA output created by 200
+Testing inmemory dedupe backend with block size 64K
+wrote 65536/65536 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+number of deduped extents is in range
+md5sum: a30e0f3f1b0884081de11d4357811c2e
+Testing inmemory dedupe backend with block size 128K
+wrote 131072/131072 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+number of deduped extents is in range
+md5sum: a30e0f3f1b0884081de11d4357811c2e
+Testing inmemory dedupe backend with block size 1M
+wrote 1048576/1048576 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 268435456/268435456 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+number of deduped extents is in range
+md5sum: a30e0f3f1b0884081de11d4357811c2e
diff --git a/tests/btrfs/group b/tests/btrfs/group
index 76a1181e..bf001d3c 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -125,6 +125,7 @@
120 auto quick snapshot metadata
121 auto quick snapshot qgroup
122 auto quick snapshot qgroup
+<<<<<<< HEAD
123 auto quick qgroup
124 auto replace
125 auto replace
@@ -141,3 +142,4 @@
136 auto convert
137 auto quick send
138 auto compress
+200 auto ib-dedupe
--
2.12.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 1/3] fstests: btrfs: Add basic test for btrfs in-band de-duplication
2017-03-16 1:50 ` [PATCH 1/3] fstests: btrfs: Add basic test for btrfs in-band de-duplication Qu Wenruo
@ 2017-03-16 8:08 ` Eryu Guan
2017-03-16 8:33 ` Qu Wenruo
0 siblings, 1 reply; 11+ messages in thread
From: Eryu Guan @ 2017-03-16 8:08 UTC (permalink / raw)
To: Qu Wenruo; +Cc: linux-btrfs, fstests
On Thu, Mar 16, 2017 at 09:50:25AM +0800, Qu Wenruo wrote:
> Add basic test for btrfs in-band de-duplication(inmemory backend), including:
> 1) Enable
> 3) Dedup rate
> 4) File correctness
> 5) Disable
>
> Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
I haven't looked into this patchset closely, this may need more help
from other btrfs developers. Some comments from my first round
eyeballing review :)
> ---
> common/defrag | 13 ++++++
> tests/btrfs/200 | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> tests/btrfs/200.out | 22 ++++++++++
> tests/btrfs/group | 2 +
> 4 files changed, 153 insertions(+)
> create mode 100755 tests/btrfs/200
> create mode 100644 tests/btrfs/200.out
>
> diff --git a/common/defrag b/common/defrag
> index d279382f..0a41714f 100644
> --- a/common/defrag
> +++ b/common/defrag
> @@ -59,6 +59,19 @@ _extent_count()
> $XFS_IO_PROG -c "fiemap" $1 | tail -n +2 | grep -v hole | wc -l| $AWK_PROG '{print $1}'
> }
>
> +# Get the number of unique file extents
> +# Unique file extents means they have different ondisk bytenr
> +# Some filesystem supports reflinkat() or in-band de-dup can create
> +# a file whose all file extents points to the same ondisk bytenr
> +# this can be used to test if such reflinkat() or in-band de-dup works
> +_extent_count_uniq()
> +{
> + file=$1
Declare local vars as "local".
> + $XFS_IO_PROG -c "fiemap" $file >> $seqres.full 2>&1
> + $XFS_IO_PROG -c "fiemap" $file | tail -n +2 | grep -v hole |\
> + $AWK_PROG '{print $3}' | sort | uniq | wc -l
> +}
> +
> _check_extent_count()
> {
> min=$1
> diff --git a/tests/btrfs/200 b/tests/btrfs/200
> new file mode 100755
> index 00000000..1b3e46fd
> --- /dev/null
> +++ b/tests/btrfs/200
> @@ -0,0 +1,116 @@
> +#! /bin/bash
> +# FS QA Test 200
> +#
> +# Basic btrfs inband dedupe test for inmemory backend
> +#
> +#-----------------------------------------------------------------------
> +# Copyright (c) 2016 Fujitsu. All Rights Reserved.
> +#
> +# This program is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU General Public License as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it would be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write the Free Software Foundation,
> +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
> +#-----------------------------------------------------------------------
> +#
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1 # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> + cd /
> + rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +. ./common/defrag
> +
> +# remove previous $seqres.full before test
> +rm -f $seqres.full
> +
> +# real QA test starts here
> +
> +_supported_fs btrfs
> +_supported_os Linux
> +_require_scratch
> +_require_btrfs_command dedupe
> +_require_btrfs_fs_feature dedupe
> +
> +# File size is twice the maximum file extent of btrfs
> +# So even fallbacked to non-dedupe, it will have at least 2 extents
> +file_size=256m
> +
> +_scratch_mkfs >> $seqres.full 2>&1
> +_scratch_mount
> +
> +do_dedupe_test()
> +{
> + dedupe_bs=$1
> +
> + echo "Testing inmemory dedupe backend with block size $dedupe_bs"
> + _run_btrfs_util_prog dedupe enable -f -s inmemory -b $dedupe_bs \
> + $SCRATCH_MNT
> + # do sync write to ensure dedupe hash is added into dedupe pool
> + $XFS_IO_PROG -f -c "pwrite -b $dedupe_bs 0 $dedupe_bs" -c "fsync"\
> + $SCRATCH_MNT/initial_block | _filter_xfs_io
> +
> + # do sync write to ensure we can get stable fiemap later
> + $XFS_IO_PROG -f -c "pwrite -b $dedupe_bs 0 $file_size" -c "fsync"\
> + $SCRATCH_MNT/real_file | _filter_xfs_io
> +
> + # Test if real_file is de-duplicated
> + nr_uniq_extents=$(_extent_count_uniq $SCRATCH_MNT/real_file)
> + nr_total_extents=$(_extent_count $SCRATCH_MNT/real_file)
> + nr_deduped_extents=$(($nr_total_extents - $nr_uniq_extents))
> +
> + echo "deduped/total: $nr_deduped_extents/$nr_total_extents" \
> + >> $seqres.full
> + # Allow a small amount of dedupe miss, as commit interval or
> + # memory pressure may break a dedupe_bs block and cause
> + # small extent which won't go through dedupe routine
> + _within_tolerance "number of deduped extents" $nr_deduped_extents \
> + $nr_total_extents 5% -v
> +
> + # Also check the md5sum to ensure data is not corrupted
> + md5=$(_md5_checksum $SCRATCH_MNT/real_file)
> + echo "md5sum: $md5"
> +}
> +
> +# Test inmemory dedupe first, use 64K dedupe bs to keep compatibility
> +# with 64K page size
> +do_dedupe_test 64K
> +
> +# Test 128K(default) dedupe bs
> +do_dedupe_test 128K
> +
> +# Test 1M dedupe bs
> +do_dedupe_test 1M
> +
> +# Check dedupe disable
> +_run_btrfs_util_prog dedupe disable $SCRATCH_MNT
> +
> +# success, all done
> +status=0
> +exit
> +# Check dedupe disable
> +_run_btrfs_util_prog dedupe disable $SCRATCH_MNT
> +
> +# success, all done
> +status=0
> +exit
Double "dedupe disable" and "exit".
> diff --git a/tests/btrfs/200.out b/tests/btrfs/200.out
> new file mode 100644
> index 00000000..e09e5733
> --- /dev/null
> +++ b/tests/btrfs/200.out
> @@ -0,0 +1,22 @@
> +QA output created by 200
> +Testing inmemory dedupe backend with block size 64K
> +wrote 65536/65536 bytes at offset 0
> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +wrote 268435456/268435456 bytes at offset 0
> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +number of deduped extents is in range
> +md5sum: a30e0f3f1b0884081de11d4357811c2e
> +Testing inmemory dedupe backend with block size 128K
> +wrote 131072/131072 bytes at offset 0
> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +wrote 268435456/268435456 bytes at offset 0
> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +number of deduped extents is in range
> +md5sum: a30e0f3f1b0884081de11d4357811c2e
> +Testing inmemory dedupe backend with block size 1M
> +wrote 1048576/1048576 bytes at offset 0
> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +wrote 268435456/268435456 bytes at offset 0
> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> +number of deduped extents is in range
> +md5sum: a30e0f3f1b0884081de11d4357811c2e
> diff --git a/tests/btrfs/group b/tests/btrfs/group
> index 76a1181e..bf001d3c 100644
> --- a/tests/btrfs/group
> +++ b/tests/btrfs/group
> @@ -125,6 +125,7 @@
> 120 auto quick snapshot metadata
> 121 auto quick snapshot qgroup
> 122 auto quick snapshot qgroup
> +<<<<<<< HEAD
Leftover from resolving conflicts?
Thanks,
Eryu
> 123 auto quick qgroup
> 124 auto replace
> 125 auto replace
> @@ -141,3 +142,4 @@
> 136 auto convert
> 137 auto quick send
> 138 auto compress
> +200 auto ib-dedupe
> --
> 2.12.0
>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe fstests" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 1/3] fstests: btrfs: Add basic test for btrfs in-band de-duplication
2017-03-16 8:08 ` Eryu Guan
@ 2017-03-16 8:33 ` Qu Wenruo
0 siblings, 0 replies; 11+ messages in thread
From: Qu Wenruo @ 2017-03-16 8:33 UTC (permalink / raw)
To: Eryu Guan; +Cc: linux-btrfs, fstests
At 03/16/2017 04:08 PM, Eryu Guan wrote:
> On Thu, Mar 16, 2017 at 09:50:25AM +0800, Qu Wenruo wrote:
>> Add basic test for btrfs in-band de-duplication(inmemory backend), including:
>> 1) Enable
>> 3) Dedup rate
>> 4) File correctness
>> 5) Disable
>>
>> Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
>
> I haven't looked into this patchset closely, this may need more help
> from other btrfs developers. Some comments from my first round
> eyeballing review :)
Thanks for the review.
I'll update the patchset.
Thanks,
Qu
>
>> ---
>> common/defrag | 13 ++++++
>> tests/btrfs/200 | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>> tests/btrfs/200.out | 22 ++++++++++
>> tests/btrfs/group | 2 +
>> 4 files changed, 153 insertions(+)
>> create mode 100755 tests/btrfs/200
>> create mode 100644 tests/btrfs/200.out
>>
>> diff --git a/common/defrag b/common/defrag
>> index d279382f..0a41714f 100644
>> --- a/common/defrag
>> +++ b/common/defrag
>> @@ -59,6 +59,19 @@ _extent_count()
>> $XFS_IO_PROG -c "fiemap" $1 | tail -n +2 | grep -v hole | wc -l| $AWK_PROG '{print $1}'
>> }
>>
>> +# Get the number of unique file extents
>> +# Unique file extents means they have different ondisk bytenr
>> +# Some filesystem supports reflinkat() or in-band de-dup can create
>> +# a file whose all file extents points to the same ondisk bytenr
>> +# this can be used to test if such reflinkat() or in-band de-dup works
>> +_extent_count_uniq()
>> +{
>> + file=$1
>
> Declare local vars as "local".
>
>> + $XFS_IO_PROG -c "fiemap" $file >> $seqres.full 2>&1
>> + $XFS_IO_PROG -c "fiemap" $file | tail -n +2 | grep -v hole |\
>> + $AWK_PROG '{print $3}' | sort | uniq | wc -l
>> +}
>> +
>> _check_extent_count()
>> {
>> min=$1
>> diff --git a/tests/btrfs/200 b/tests/btrfs/200
>> new file mode 100755
>> index 00000000..1b3e46fd
>> --- /dev/null
>> +++ b/tests/btrfs/200
>> @@ -0,0 +1,116 @@
>> +#! /bin/bash
>> +# FS QA Test 200
>> +#
>> +# Basic btrfs inband dedupe test for inmemory backend
>> +#
>> +#-----------------------------------------------------------------------
>> +# Copyright (c) 2016 Fujitsu. All Rights Reserved.
>> +#
>> +# This program is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU General Public License as
>> +# published by the Free Software Foundation.
>> +#
>> +# This program is distributed in the hope that it would be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write the Free Software Foundation,
>> +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
>> +#-----------------------------------------------------------------------
>> +#
>> +
>> +seq=`basename $0`
>> +seqres=$RESULT_DIR/$seq
>> +echo "QA output created by $seq"
>> +
>> +here=`pwd`
>> +tmp=/tmp/$$
>> +status=1 # failure is the default!
>> +trap "_cleanup; exit \$status" 0 1 2 3 15
>> +
>> +_cleanup()
>> +{
>> + cd /
>> + rm -f $tmp.*
>> +}
>> +
>> +# get standard environment, filters and checks
>> +. ./common/rc
>> +. ./common/filter
>> +. ./common/defrag
>> +
>> +# remove previous $seqres.full before test
>> +rm -f $seqres.full
>> +
>> +# real QA test starts here
>> +
>> +_supported_fs btrfs
>> +_supported_os Linux
>> +_require_scratch
>> +_require_btrfs_command dedupe
>> +_require_btrfs_fs_feature dedupe
>> +
>> +# File size is twice the maximum file extent of btrfs
>> +# So even fallbacked to non-dedupe, it will have at least 2 extents
>> +file_size=256m
>> +
>> +_scratch_mkfs >> $seqres.full 2>&1
>> +_scratch_mount
>> +
>> +do_dedupe_test()
>> +{
>> + dedupe_bs=$1
>> +
>> + echo "Testing inmemory dedupe backend with block size $dedupe_bs"
>> + _run_btrfs_util_prog dedupe enable -f -s inmemory -b $dedupe_bs \
>> + $SCRATCH_MNT
>> + # do sync write to ensure dedupe hash is added into dedupe pool
>> + $XFS_IO_PROG -f -c "pwrite -b $dedupe_bs 0 $dedupe_bs" -c "fsync"\
>> + $SCRATCH_MNT/initial_block | _filter_xfs_io
>> +
>> + # do sync write to ensure we can get stable fiemap later
>> + $XFS_IO_PROG -f -c "pwrite -b $dedupe_bs 0 $file_size" -c "fsync"\
>> + $SCRATCH_MNT/real_file | _filter_xfs_io
>> +
>> + # Test if real_file is de-duplicated
>> + nr_uniq_extents=$(_extent_count_uniq $SCRATCH_MNT/real_file)
>> + nr_total_extents=$(_extent_count $SCRATCH_MNT/real_file)
>> + nr_deduped_extents=$(($nr_total_extents - $nr_uniq_extents))
>> +
>> + echo "deduped/total: $nr_deduped_extents/$nr_total_extents" \
>> + >> $seqres.full
>> + # Allow a small amount of dedupe miss, as commit interval or
>> + # memory pressure may break a dedupe_bs block and cause
>> + # small extent which won't go through dedupe routine
>> + _within_tolerance "number of deduped extents" $nr_deduped_extents \
>> + $nr_total_extents 5% -v
>> +
>> + # Also check the md5sum to ensure data is not corrupted
>> + md5=$(_md5_checksum $SCRATCH_MNT/real_file)
>> + echo "md5sum: $md5"
>> +}
>> +
>> +# Test inmemory dedupe first, use 64K dedupe bs to keep compatibility
>> +# with 64K page size
>> +do_dedupe_test 64K
>> +
>> +# Test 128K(default) dedupe bs
>> +do_dedupe_test 128K
>> +
>> +# Test 1M dedupe bs
>> +do_dedupe_test 1M
>> +
>> +# Check dedupe disable
>> +_run_btrfs_util_prog dedupe disable $SCRATCH_MNT
>> +
>> +# success, all done
>> +status=0
>> +exit
>> +# Check dedupe disable
>> +_run_btrfs_util_prog dedupe disable $SCRATCH_MNT
>> +
>> +# success, all done
>> +status=0
>> +exit
>
> Double "dedupe disable" and "exit".
>
>> diff --git a/tests/btrfs/200.out b/tests/btrfs/200.out
>> new file mode 100644
>> index 00000000..e09e5733
>> --- /dev/null
>> +++ b/tests/btrfs/200.out
>> @@ -0,0 +1,22 @@
>> +QA output created by 200
>> +Testing inmemory dedupe backend with block size 64K
>> +wrote 65536/65536 bytes at offset 0
>> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> +wrote 268435456/268435456 bytes at offset 0
>> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> +number of deduped extents is in range
>> +md5sum: a30e0f3f1b0884081de11d4357811c2e
>> +Testing inmemory dedupe backend with block size 128K
>> +wrote 131072/131072 bytes at offset 0
>> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> +wrote 268435456/268435456 bytes at offset 0
>> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> +number of deduped extents is in range
>> +md5sum: a30e0f3f1b0884081de11d4357811c2e
>> +Testing inmemory dedupe backend with block size 1M
>> +wrote 1048576/1048576 bytes at offset 0
>> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> +wrote 268435456/268435456 bytes at offset 0
>> +XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> +number of deduped extents is in range
>> +md5sum: a30e0f3f1b0884081de11d4357811c2e
>> diff --git a/tests/btrfs/group b/tests/btrfs/group
>> index 76a1181e..bf001d3c 100644
>> --- a/tests/btrfs/group
>> +++ b/tests/btrfs/group
>> @@ -125,6 +125,7 @@
>> 120 auto quick snapshot metadata
>> 121 auto quick snapshot qgroup
>> 122 auto quick snapshot qgroup
>> +<<<<<<< HEAD
>
> Leftover from resolving conflicts?
>
> Thanks,
> Eryu
>
>> 123 auto quick qgroup
>> 124 auto replace
>> 125 auto replace
>> @@ -141,3 +142,4 @@
>> 136 auto convert
>> 137 auto quick send
>> 138 auto compress
>> +200 auto ib-dedupe
>> --
>> 2.12.0
>>
>>
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe fstests" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH] fstests: generic: Test space allocation when there is only fragmented space
2017-03-16 1:50 [PATCH 0/3] Btrfs in-band de-duplication test cases Qu Wenruo
2017-03-16 1:50 ` [PATCH 1/3] fstests: btrfs: Add basic test for btrfs in-band de-duplication Qu Wenruo
@ 2017-03-16 1:50 ` Qu Wenruo
2017-03-16 1:50 ` [PATCH 2/3] fstests: btrfs: Add testcase for btrfs dedupe and metadata balance race test Qu Wenruo
` (2 subsequent siblings)
4 siblings, 0 replies; 11+ messages in thread
From: Qu Wenruo @ 2017-03-16 1:50 UTC (permalink / raw)
To: linux-btrfs; +Cc: fstests, Filipe Manana, Liu Bo
This test case will test if file system works well when handling large
write while available space are all fragmented.
This can expose a bug in a btrfs unmerged patch, which wrongly modified
the delayed allocation code, to exit before allocating all space, and
cause hang when unmounting.
The wrong patch is:
[PATCH v6 1/2] btrfs: Fix metadata underflow caused by btrfs_reloc_clone_csum error
The test case will:
1) Fill small filesystem with page sized small files
All these files has a sequential number as file name
2) Remove files with odd number as file name
This will free almost half of the space
3) Try to write a file which takes 1/8 of the file system
The method to create fragmented fs may not be generic enough, but should
work for most extent based fs.
Unless one file system will allocate extents from both end of its free
space.
Cc: Filipe Manana <fdmanana@kernel.org>
Cc: Liu Bo <bo.li.liu@oracle.com>
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
tests/generic/416 | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++
tests/generic/416.out | 3 ++
tests/generic/group | 1 +
3 files changed, 103 insertions(+)
create mode 100755 tests/generic/416
create mode 100644 tests/generic/416.out
diff --git a/tests/generic/416 b/tests/generic/416
new file mode 100755
index 0000000..925524b
--- /dev/null
+++ b/tests/generic/416
@@ -0,0 +1,99 @@
+#! /bin/bash
+# FS QA Test 416
+#
+# Test fs behavior when large write request can't be met by one single extent
+#
+# Inspired by a bug in a btrfs fix, which doesn't get exposed by current test
+# cases
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2017 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+# Modify as appropriate.
+_supported_fs generic
+_supported_os IRIX Linux
+_require_scratch
+
+fs_size=$((128 * 1024 * 1024))
+page_size=$(get_page_size)
+
+# We will never reach this number though
+nr_files=$(($fs_size / $page_size))
+
+# Use small fs to make the fill more faster
+_scratch_mkfs_sized $fs_size >> $seqres.full 2>&1
+
+_scratch_mount
+
+fill_fs()
+{
+ dir=$1
+ for i in $(seq -w $nr_files); do
+ # xfs_io can't return correct value when it hit ENOSPC, use
+ # dd here to detect ENOSPC
+ dd if=/dev/zero of=$SCRATCH_MNT/$i bs=$page_size count=1 \
+ &> /dev/null
+ if [ $? -ne 0 ]; then
+ break
+ fi
+ done
+}
+
+fill_fs $SCRATCH_MNT
+
+# remount to sync every thing into fs, and drop all cache
+_scratch_remount
+
+# remove all files with odd file names, which should free near half
+# of the space
+rm $SCRATCH_MNT/*[13579]
+sync
+
+# We should be able to write at least 1/8 of the whole fs size
+# The number 1/8 is for btrfs, which only has about 47M for data.
+# And half of the 47M is already taken up, so only 1/8 is safe here
+$XFS_IO_PROG -f -c "pwrite 0 $(($fs_size / 8))" $SCRATCH_MNT/large_file | \
+ _filter_xfs_io
+
+# success, all done
+status=0
+exit
diff --git a/tests/generic/416.out b/tests/generic/416.out
new file mode 100644
index 0000000..8d2ffac
--- /dev/null
+++ b/tests/generic/416.out
@@ -0,0 +1,3 @@
+QA output created by 416
+wrote 16777216/16777216 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/generic/group b/tests/generic/group
index b510d41..59f94f9 100644
--- a/tests/generic/group
+++ b/tests/generic/group
@@ -418,3 +418,4 @@
413 auto quick
414 auto quick clone
415 auto clone
+416 auto enospc
--
2.9.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 2/3] fstests: btrfs: Add testcase for btrfs dedupe and metadata balance race test
2017-03-16 1:50 [PATCH 0/3] Btrfs in-band de-duplication test cases Qu Wenruo
2017-03-16 1:50 ` [PATCH 1/3] fstests: btrfs: Add basic test for btrfs in-band de-duplication Qu Wenruo
2017-03-16 1:50 ` [PATCH] fstests: generic: Test space allocation when there is only fragmented space Qu Wenruo
@ 2017-03-16 1:50 ` Qu Wenruo
2017-03-16 8:10 ` Eryu Guan
2017-03-16 1:50 ` [PATCH 3/3] fstests: btrfs: Test inband dedupe with data balance Qu Wenruo
2017-03-21 7:23 ` [PATCH 0/3] Btrfs in-band de-duplication test cases Eryu Guan
4 siblings, 1 reply; 11+ messages in thread
From: Qu Wenruo @ 2017-03-16 1:50 UTC (permalink / raw)
To: linux-btrfs; +Cc: fstests
Btrfs balance with inband dedupe enable/disable will expose a lot of
hidden dedupe bug:
1) Enable/disable race bug
2) Btrfs dedupe tree balance corrupted delayed_ref
3) Btrfs disable and balance will cause balance BUG_ON
Reported-by: Satoru Takeuchi <takeuchi_satoru@jp.fujitsu.com>
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
tests/btrfs/201 | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tests/btrfs/201.out | 2 +
tests/btrfs/group | 1 +
3 files changed, 115 insertions(+)
create mode 100755 tests/btrfs/201
create mode 100644 tests/btrfs/201.out
diff --git a/tests/btrfs/201 b/tests/btrfs/201
new file mode 100755
index 00000000..d6913c13
--- /dev/null
+++ b/tests/btrfs/201
@@ -0,0 +1,112 @@
+#! /bin/bash
+# FS QA Test 201
+#
+# Btrfs inband dedup enable/disable race with metadata balance
+#
+# This tests will test the following bugs exposed in development:
+# 1) enable/disable race
+# 2) tree balance cause delayed ref corruption
+# 3) disable and balance cause BUG_ON
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2016 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ rm -f $tmp.*
+ killall $FSSTRESS_PROG &> /dev/null
+ kill $trigger_pid &> /dev/null
+ kill $balance_pid &> /dev/null
+ wait
+
+ # See comment later
+ $BTRFS_UTIL_PROG balance cancel $SCRATCH_MNT &> /dev/null
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_require_btrfs_command dedupe
+_require_btrfs_fs_feature dedupe
+
+# Use 64K dedupe size to keep compatibility for 64K page size
+dedupe_bs=64K
+
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+mkdir -p $SCRATCH_MNT/stressdir
+
+runtime=$((60 * $TIME_FACTOR))
+
+trigger_work()
+{
+ while true; do
+ _run_btrfs_util_prog dedupe enable -s inmemory \
+ -b $dedupe_bs $SCRATCH_MNT
+ sleep 1
+ _run_btrfs_util_prog dedupe disable $SCRATCH_MNT
+ sleep 1
+ done
+}
+
+# redirect all output, as error output like 'balance cancelled by user'
+# will populuate the golden output.
+_btrfs_stress_balance -m $SCRATCH_MNT &> /dev/null &
+balance_pid=$!
+
+$FSSTRESS_PROG $(_scale_fsstress_args -p 1 -n 10000000) $FSSTRESS_AVOID \
+ -d $SCRATCH_MNT/stressdir > /dev/null 2>&1 &
+
+trigger_work &
+trigger_pid=$!
+
+sleep $runtime
+killall $FSSTRESS_PROG &> /dev/null
+kill $trigger_pid &> /dev/null
+kill $balance_pid &> /dev/null
+wait
+
+# Manually stop balance as it's possible balance is still running for a short
+# time. And we don't want to populate $seqres.full, so call $BTRFS_UTIL_PROG
+# directly
+$BTRFS_UTIL_PROG balance cancel $SCRATCH_MNT &> /dev/null
+
+echo "Silence is golden"
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/201.out b/tests/btrfs/201.out
new file mode 100644
index 00000000..5ac973f5
--- /dev/null
+++ b/tests/btrfs/201.out
@@ -0,0 +1,2 @@
+QA output created by 201
+Silence is golden
diff --git a/tests/btrfs/group b/tests/btrfs/group
index bf001d3c..f87d995c 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -143,3 +143,4 @@
137 auto quick send
138 auto compress
200 auto ib-dedupe
+201 auto ib-dedupe
--
2.12.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 2/3] fstests: btrfs: Add testcase for btrfs dedupe and metadata balance race test
2017-03-16 1:50 ` [PATCH 2/3] fstests: btrfs: Add testcase for btrfs dedupe and metadata balance race test Qu Wenruo
@ 2017-03-16 8:10 ` Eryu Guan
0 siblings, 0 replies; 11+ messages in thread
From: Eryu Guan @ 2017-03-16 8:10 UTC (permalink / raw)
To: Qu Wenruo; +Cc: linux-btrfs, fstests
On Thu, Mar 16, 2017 at 09:50:27AM +0800, Qu Wenruo wrote:
> Btrfs balance with inband dedupe enable/disable will expose a lot of
> hidden dedupe bug:
>
> 1) Enable/disable race bug
> 2) Btrfs dedupe tree balance corrupted delayed_ref
> 3) Btrfs disable and balance will cause balance BUG_ON
>
> Reported-by: Satoru Takeuchi <takeuchi_satoru@jp.fujitsu.com>
> Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
> ---
> tests/btrfs/201 | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> tests/btrfs/201.out | 2 +
> tests/btrfs/group | 1 +
> 3 files changed, 115 insertions(+)
> create mode 100755 tests/btrfs/201
> create mode 100644 tests/btrfs/201.out
>
> diff --git a/tests/btrfs/201 b/tests/btrfs/201
> new file mode 100755
> index 00000000..d6913c13
> --- /dev/null
> +++ b/tests/btrfs/201
> @@ -0,0 +1,112 @@
> +#! /bin/bash
> +# FS QA Test 201
> +#
> +# Btrfs inband dedup enable/disable race with metadata balance
> +#
> +# This tests will test the following bugs exposed in development:
> +# 1) enable/disable race
> +# 2) tree balance cause delayed ref corruption
> +# 3) disable and balance cause BUG_ON
> +#
> +#-----------------------------------------------------------------------
> +# Copyright (c) 2016 Fujitsu. All Rights Reserved.
^^^^ 2017, all these three patches need update too.
> +#
> +# This program is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU General Public License as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it would be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write the Free Software Foundation,
> +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
> +#-----------------------------------------------------------------------
> +#
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1 # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> + cd /
> + rm -f $tmp.*
> + killall $FSSTRESS_PROG &> /dev/null
$KILLALL_PROG and
_require_command "$KILLALL_PROG" killall
Thanks,
Eryu
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH 3/3] fstests: btrfs: Test inband dedupe with data balance.
2017-03-16 1:50 [PATCH 0/3] Btrfs in-band de-duplication test cases Qu Wenruo
` (2 preceding siblings ...)
2017-03-16 1:50 ` [PATCH 2/3] fstests: btrfs: Add testcase for btrfs dedupe and metadata balance race test Qu Wenruo
@ 2017-03-16 1:50 ` Qu Wenruo
2017-03-21 7:23 ` [PATCH 0/3] Btrfs in-band de-duplication test cases Eryu Guan
4 siblings, 0 replies; 11+ messages in thread
From: Qu Wenruo @ 2017-03-16 1:50 UTC (permalink / raw)
To: linux-btrfs; +Cc: fstests
Btrfs balance will reloate date extent, but its hash is removed too late
at run_delayed_ref() time, which will cause extent ref increased
during balance, cause either find_data_references() gives WARN_ON()
or even run_delayed_refs() fails and cause transaction abort.
Add such concurrency test for inband dedupe and data balance.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
tests/btrfs/203 | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tests/btrfs/203.out | 3 ++
tests/btrfs/group | 1 +
3 files changed, 113 insertions(+)
create mode 100755 tests/btrfs/203
create mode 100644 tests/btrfs/203.out
diff --git a/tests/btrfs/203 b/tests/btrfs/203
new file mode 100755
index 00000000..aea756cb
--- /dev/null
+++ b/tests/btrfs/203
@@ -0,0 +1,109 @@
+#! /bin/bash
+# FS QA Test 203
+#
+# Btrfs inband dedupe with balance concurrency test
+#
+# This can spot inband dedupe error which will increase delayed ref on
+# an data extent inside RO block group
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2016 Fujitsu. All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#-----------------------------------------------------------------------
+#
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+ cd /
+ kill $populate_pid &> /dev/null
+ kill $balance_pid &> /dev/null
+ wait
+ # Check later comment for reason
+ $BTRFS_UTIL_PROG balance cancel $SCRATCH_MNT &> /dev/null
+ rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+. ./common/reflink
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# real QA test starts here
+
+_supported_fs btrfs
+_supported_os Linux
+_require_scratch
+_require_cp_reflink
+_require_btrfs_command dedupe
+_require_btrfs_fs_feature dedupe
+
+dedupe_bs=128k
+file_size_in_kilo=4096
+init_file=$SCRATCH_MNT/foo
+run_time=$((60 * $TIME_FACTOR))
+
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+do_dedupe_balance_test()
+{
+ _run_btrfs_util_prog dedupe enable -b $dedupe_bs -s inmemory $SCRATCH_MNT
+
+ # create the initial file and fill hash pool
+ $XFS_IO_PROG -f -c "pwrite -S 0x0 -b $dedupe_bs 0 $dedupe_bs" -c "fsync" \
+ $init_file | _filter_xfs_io
+
+ _btrfs_stress_balance $SCRATCH_MNT >/dev/null 2>&1 &
+ balance_pid=$!
+
+ # Populate fs with all 0 data, to trigger enough in-band dedupe work
+ # to race with balance
+ _populate_fs -n 5 -f 10000000 -d 1 -r $SCRATCH_MNT \
+ -s $file_size_in_kilo &> /dev/null &
+ populate_pid=$!
+
+ sleep $run_time
+
+ kill $populate_pid
+ kill $balance_pid
+ wait
+
+ # Sometimes even we killed $balance_pid and wait returned,
+ # balance may still be running, use balance cancel to wait it.
+ # As this is just a workaround, we don't want it pollute seqres
+ # so call $BTRFS_UTIL_PROG directly
+ $BTRFS_UTIL_PROG balance cancel $SCRATCH_MNT &> /dev/null
+
+ rm $SCRATCH_MNT/* -rf &> /dev/null
+ _run_btrfs_util_prog dedupe disable $SCRATCH_MNT
+}
+
+do_dedupe_balance_test
+
+# success, all done
+status=0
+exit
diff --git a/tests/btrfs/203.out b/tests/btrfs/203.out
new file mode 100644
index 00000000..404394c3
--- /dev/null
+++ b/tests/btrfs/203.out
@@ -0,0 +1,3 @@
+QA output created by 203
+wrote 131072/131072 bytes at offset 0
+XXX Bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/btrfs/group b/tests/btrfs/group
index f87d995c..2ef7a498 100644
--- a/tests/btrfs/group
+++ b/tests/btrfs/group
@@ -144,3 +144,4 @@
138 auto compress
200 auto ib-dedupe
201 auto ib-dedupe
+203 auto ib-dedupe balance
--
2.12.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 0/3] Btrfs in-band de-duplication test cases
2017-03-16 1:50 [PATCH 0/3] Btrfs in-band de-duplication test cases Qu Wenruo
` (3 preceding siblings ...)
2017-03-16 1:50 ` [PATCH 3/3] fstests: btrfs: Test inband dedupe with data balance Qu Wenruo
@ 2017-03-21 7:23 ` Eryu Guan
2017-03-21 7:36 ` Qu Wenruo
4 siblings, 1 reply; 11+ messages in thread
From: Eryu Guan @ 2017-03-21 7:23 UTC (permalink / raw)
To: Qu Wenruo; +Cc: linux-btrfs, fstests
On Thu, Mar 16, 2017 at 09:50:24AM +0800, Qu Wenruo wrote:
> Btrfs in-band de-duplication test cases for in-memory backend, which covers
> the bugs exposed during the development.
Sorry, I'm having trouble enabling inband dedupe in tests, I always get
ioctl failure, $seqres.full shows:
btrfs-progs v4.10-5-g23d4fd5
See https://urldefense.proofpoint.com/v2/url?u=http-3A__btrfs.wiki.kernel.org&d=DwIBAg&c=RoP1YumCXCgaWHvlZYR8PQcxBKCX5YTpkKY057SbK10&r=-yMrTV4jriXR7ieyzzjV-QgHBD0UDw8ixoR77aMeAHE&m=QW8j5ytjvhSULN9SgVewhinVZVh9U-etu9FpCxFlsCc&s=N3fHeukNWyZ1Av8wPL6uQRaIzk0Y6Z90W-7loCAj4J8&e= for more information.
Label: (null)
UUID: 3d2cedd0-64ae-4667-ba09-9c12e28ecadd
Node size: 16384
Sector size: 4096
Filesystem size: 15.00GiB
Block group profiles:
Data: single 8.00MiB
Metadata: DUP 1.00GiB
System: DUP 8.00MiB
SSD detected: no
Incompat features: extref, skinny-metadata
Number of devices: 1
Devices:
ID SIZE PATH
1 15.00GiB /dev/mapper/systemvg-testlv2
# /usr/local/bin/btrfs dedupe enable -f -s inmemory -b 64K /scratch
ERROR: failed to (enable) inband deduplication: Inappropriate ioctl for device
ERROR: unsupported dedupe limit combination: nr: 0, mem: 0
failed: '/usr/local/bin/btrfs dedupe enable -f -s inmemory -b 64K /scratch'
I have no special MKFS_OPTIONS or MOUNT_OPTIONS specified, all defaults.
kernel is built from your test tree. Did I miss anything?
Thanks,
Eryu
--
To unsubscribe from this list: send the line "unsubscribe fstests" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at https://urldefense.proofpoint.com/v2/url?u=http-3A__vger.kernel.org_majordomo-2Dinfo.html&d=DwIBAg&c=RoP1YumCXCgaWHvlZYR8PQcxBKCX5YTpkKY057SbK10&r=-yMrTV4jriXR7ieyzzjV-QgHBD0UDw8ixoR77aMeAHE&m=QW8j5ytjvhSULN9SgVewhinVZVh9U-etu9FpCxFlsCc&s=eaJ-tn7MdocpEA2Dssf3q1tiBAMCKm3MbKiI4S_bkL0&e=
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 0/3] Btrfs in-band de-duplication test cases
2017-03-21 7:23 ` [PATCH 0/3] Btrfs in-band de-duplication test cases Eryu Guan
@ 2017-03-21 7:36 ` Qu Wenruo
2017-03-21 7:48 ` Eryu Guan
0 siblings, 1 reply; 11+ messages in thread
From: Qu Wenruo @ 2017-03-21 7:36 UTC (permalink / raw)
To: Eryu Guan; +Cc: linux-btrfs, fstests
At 03/21/2017 03:23 PM, Eryu Guan wrote:
> On Thu, Mar 16, 2017 at 09:50:24AM +0800, Qu Wenruo wrote:
>> Btrfs in-band de-duplication test cases for in-memory backend, which covers
>> the bugs exposed during the development.
>
> Sorry, I'm having trouble enabling inband dedupe in tests, I always get
> ioctl failure, $seqres.full shows:
>
> btrfs-progs v4.10-5-g23d4fd5
> See https://urldefense.proofpoint.com/v2/url?u=http-3A__btrfs.wiki.kernel.org&d=DwICaQ&c=RoP1YumCXCgaWHvlZYR8PQcxBKCX5YTpkKY057SbK10&r=-yMrTV4jriXR7ieyzzjV-QgHBD0UDw8ixoR77aMeAHE&m=iFkFnL2ACo5og4TuRoLnShXk6V7EGH0SsaV94r1u6cI&s=kY0tWZ91L3lG9Pa3FHOwKX157Xr_Z41QQR9GUGix9U4&e= for more information.
>
> Label: (null)
> UUID: 3d2cedd0-64ae-4667-ba09-9c12e28ecadd
> Node size: 16384
> Sector size: 4096
> Filesystem size: 15.00GiB
> Block group profiles:
> Data: single 8.00MiB
> Metadata: DUP 1.00GiB
> System: DUP 8.00MiB
> SSD detected: no
> Incompat features: extref, skinny-metadata
> Number of devices: 1
> Devices:
> ID SIZE PATH
> 1 15.00GiB /dev/mapper/systemvg-testlv2
>
> # /usr/local/bin/btrfs dedupe enable -f -s inmemory -b 64K /scratch
> ERROR: failed to (enable) inband deduplication: Inappropriate ioctl for device
> ERROR: unsupported dedupe limit combination: nr: 0, mem: 0
What I forgot to mention is that, the inband dedupe feature is an
experimental feature, so CONFIG_BTRFS_DEBUG is needed to use this feature.
Sorry for the inconvenience.
I'll add also add the CONFIG_BTRFS_DEBUG check for
/sys/fs/btrfs/feature/, so it won't export inband dedupe to users.
Thanks,
Qu
> failed: '/usr/local/bin/btrfs dedupe enable -f -s inmemory -b 64K /scratch'
>
> I have no special MKFS_OPTIONS or MOUNT_OPTIONS specified, all defaults.
> kernel is built from your test tree. Did I miss anything?
>
> Thanks,
> Eryu
>
>
--
To unsubscribe from this list: send the line "unsubscribe fstests" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at https://urldefense.proofpoint.com/v2/url?u=http-3A__vger.kernel.org_majordomo-2Dinfo.html&d=DwICaQ&c=RoP1YumCXCgaWHvlZYR8PQcxBKCX5YTpkKY057SbK10&r=-yMrTV4jriXR7ieyzzjV-QgHBD0UDw8ixoR77aMeAHE&m=iFkFnL2ACo5og4TuRoLnShXk6V7EGH0SsaV94r1u6cI&s=IYEZpOyQO99IZosg72WEZfbjJiLw_H2Fn7WFpBbYS04&e=
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH 0/3] Btrfs in-band de-duplication test cases
2017-03-21 7:36 ` Qu Wenruo
@ 2017-03-21 7:48 ` Eryu Guan
0 siblings, 0 replies; 11+ messages in thread
From: Eryu Guan @ 2017-03-21 7:48 UTC (permalink / raw)
To: Qu Wenruo; +Cc: linux-btrfs, fstests
On Tue, Mar 21, 2017 at 03:36:54PM +0800, Qu Wenruo wrote:
>
>
> At 03/21/2017 03:23 PM, Eryu Guan wrote:
> > On Thu, Mar 16, 2017 at 09:50:24AM +0800, Qu Wenruo wrote:
> > > Btrfs in-band de-duplication test cases for in-memory backend, which covers
> > > the bugs exposed during the development.
> >
> > Sorry, I'm having trouble enabling inband dedupe in tests, I always get
> > ioctl failure, $seqres.full shows:
> >
> > btrfs-progs v4.10-5-g23d4fd5
> > See https://urldefense.proofpoint.com/v2/url?u=http-3A__btrfs.wiki.kernel.org&d=DwIBAg&c=RoP1YumCXCgaWHvlZYR8PQcxBKCX5YTpkKY057SbK10&r=-yMrTV4jriXR7ieyzzjV-QgHBD0UDw8ixoR77aMeAHE&m=fSE-SZFHxYOtme73LUGhdAkjjKVeu59aewsxj3KYT8Q&s=wL4wtAJyaJ3YtkcV6849DAoedG_3c8E0ngquYDlE9ns&e= for more information.
> >
> > Label: (null)
> > UUID: 3d2cedd0-64ae-4667-ba09-9c12e28ecadd
> > Node size: 16384
> > Sector size: 4096
> > Filesystem size: 15.00GiB
> > Block group profiles:
> > Data: single 8.00MiB
> > Metadata: DUP 1.00GiB
> > System: DUP 8.00MiB
> > SSD detected: no
> > Incompat features: extref, skinny-metadata
> > Number of devices: 1
> > Devices:
> > ID SIZE PATH
> > 1 15.00GiB /dev/mapper/systemvg-testlv2
> >
> > # /usr/local/bin/btrfs dedupe enable -f -s inmemory -b 64K /scratch
> > ERROR: failed to (enable) inband deduplication: Inappropriate ioctl for device
> > ERROR: unsupported dedupe limit combination: nr: 0, mem: 0
>
> What I forgot to mention is that, the inband dedupe feature is an
> experimental feature, so CONFIG_BTRFS_DEBUG is needed to use this feature.
OK, I'll rebuild with BTRFS_DEBUG and try again, thanks for the info!
But IMHO, giving out experimental message in dmesg seems sufficient,
like what XFS reflink feature does.
>
> Sorry for the inconvenience.
>
> I'll add also add the CONFIG_BTRFS_DEBUG check for /sys/fs/btrfs/feature/,
> so it won't export inband dedupe to users.
I'd suggest that adding a new require rule to check if inband dedupe is
supported by current kernel, by enabling inband dedupe and see the
result, so that we don't need to update the CONFIG_BTRFS_DEBUG check
when this feature is moved out of BTRFS_DEBUG.
Thanks,
Eryu
--
To unsubscribe from this list: send the line "unsubscribe fstests" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at https://urldefense.proofpoint.com/v2/url?u=http-3A__vger.kernel.org_majordomo-2Dinfo.html&d=DwIBAg&c=RoP1YumCXCgaWHvlZYR8PQcxBKCX5YTpkKY057SbK10&r=-yMrTV4jriXR7ieyzzjV-QgHBD0UDw8ixoR77aMeAHE&m=fSE-SZFHxYOtme73LUGhdAkjjKVeu59aewsxj3KYT8Q&s=EWR8R6A6ZhG1pWEZAhSGUJ2opiKV_e5FsB5X3ZMVxLE&e=
^ permalink raw reply [flat|nested] 11+ messages in thread