* [PATCH v8 1/4] btrfs-progs: Basic framework for dedupe-inband command group
2016-06-17 1:02 [PATCH v8 0/4] Inband dedupe for btrfs-progs Qu Wenruo
@ 2016-06-17 1:02 ` Qu Wenruo
2016-06-17 1:02 ` [PATCH v8 2/4] btrfs-progs: dedupe: Add enable command for dedupe " Qu Wenruo
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Qu Wenruo @ 2016-06-17 1:02 UTC (permalink / raw)
To: linux-btrfs
Add basic ioctl header and command group framework for later use.
Alone with basic man page doc.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
Documentation/Makefile.in | 1 +
Documentation/btrfs-dedupe-inband.asciidoc | 40 +++++++++++++++++++++++++
Documentation/btrfs.asciidoc | 4 +++
Makefile.in | 3 +-
btrfs.c | 2 ++
cmds-dedupe-ib.c | 48 ++++++++++++++++++++++++++++++
commands.h | 2 ++
ioctl.h | 21 +++++++++++++
8 files changed, 120 insertions(+), 1 deletion(-)
create mode 100644 Documentation/btrfs-dedupe-inband.asciidoc
create mode 100644 cmds-dedupe-ib.c
diff --git a/Documentation/Makefile.in b/Documentation/Makefile.in
index aea2cb4..7db4164 100644
--- a/Documentation/Makefile.in
+++ b/Documentation/Makefile.in
@@ -28,6 +28,7 @@ MAN8_TXT += btrfs-qgroup.asciidoc
MAN8_TXT += btrfs-replace.asciidoc
MAN8_TXT += btrfs-restore.asciidoc
MAN8_TXT += btrfs-property.asciidoc
+MAN8_TXT += btrfs-dedupe-inband.asciidoc
# Category 5 manual page
MAN5_TXT += btrfs-man5.asciidoc
diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
new file mode 100644
index 0000000..9ee2bc7
--- /dev/null
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -0,0 +1,40 @@
+btrfs-dedupe(8)
+==============
+
+NAME
+----
+btrfs-dedupe-inband - manage in-band (write time) de-duplication of a btrfs
+filesystem
+
+SYNOPSIS
+--------
+*btrfs dedupe-inband* <subcommand> <args>
+
+DESCRIPTION
+-----------
+*btrfs dedupe-inband* is used to enable/disable or show current in-band de-duplication
+status of a btrfs filesystem.
+
+Kernel support for in-band de-duplication starts from 4.8.
+
+WARNING: In-band de-duplication is still an experimental feautre of btrfs,
+use with caution.
+
+SUBCOMMAND
+----------
+Nothing yet
+
+EXIT STATUS
+-----------
+*btrfs dedupe-inband* returns a zero exit status if it succeeds. Non zero is
+returned in case of failure.
+
+AVAILABILITY
+------------
+*btrfs* is part of btrfs-progs.
+Please refer to the btrfs wiki http://btrfs.wiki.kernel.org for
+further details.
+
+SEE ALSO
+--------
+`mkfs.btrfs`(8),
diff --git a/Documentation/btrfs.asciidoc b/Documentation/btrfs.asciidoc
index 6a77a85..8ded842 100644
--- a/Documentation/btrfs.asciidoc
+++ b/Documentation/btrfs.asciidoc
@@ -43,6 +43,10 @@ COMMANDS
Do off-line check on a btrfs filesystem. +
See `btrfs-check`(8) for details.
+*dedupe*::
+ Control btrfs in-band(write time) de-duplication. +
+ See `btrfs-dedupe`(8) for details.
+
*device*::
Manage devices managed by btrfs, including add/delete/scan and so
on. +
diff --git a/Makefile.in b/Makefile.in
index ac6b353..5cf0fc2 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -86,7 +86,8 @@ cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
cmds-quota.o cmds-qgroup.o cmds-replace.o cmds-check.o \
cmds-restore.o cmds-rescue.o chunk-recover.o super-recover.o \
cmds-property.o cmds-fi-usage.o cmds-inspect-dump-tree.o \
- cmds-inspect-dump-super.o cmds-inspect-tree-stats.o cmds-fi-du.o
+ cmds-inspect-dump-super.o cmds-inspect-tree-stats.o cmds-fi-du.o \
+ cmds-dedupe-ib.o
libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \
uuid-tree.o utils-lib.o rbtree-utils.o
libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
diff --git a/btrfs.c b/btrfs.c
index cc70515..06c211b 100644
--- a/btrfs.c
+++ b/btrfs.c
@@ -199,6 +199,8 @@ static const struct cmd_group btrfs_cmd_group = {
{ "receive", cmd_receive, cmd_receive_usage, NULL, 0 },
{ "quota", cmd_quota, NULL, "a_cmd_group, 0 },
{ "qgroup", cmd_qgroup, NULL, &qgroup_cmd_group, 0 },
+ { "dedupe-inband", cmd_dedupe_ib, NULL, &dedupe_ib_cmd_group,
+ 0 },
{ "replace", cmd_replace, NULL, &replace_cmd_group, 0 },
{ "help", cmd_help, cmd_help_usage, NULL, 0 },
{ "version", cmd_version, cmd_version_usage, NULL, 0 },
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
new file mode 100644
index 0000000..160c8ec
--- /dev/null
+++ b/cmds-dedupe-ib.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Fujitsu. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "ctree.h"
+#include "ioctl.h"
+
+#include "commands.h"
+#include "utils.h"
+#include "kerncompat.h"
+#include "dedupe-ib.h"
+
+static const char * const dedupe_ib_cmd_group_usage[] = {
+ "btrfs dedupe-inband <command> [options] <path>",
+ NULL
+};
+
+static const char dedupe_ib_cmd_group_info[] =
+"manage inband(write time) de-duplication";
+
+const struct cmd_group dedupe_ib_cmd_group = {
+ dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
+ NULL_CMD_STRUCT
+ }
+};
+
+int cmd_dedupe_ib(int argc, char **argv)
+{
+ return handle_command_group(&dedupe_ib_cmd_group, argc, argv);
+}
diff --git a/commands.h b/commands.h
index 94229c1..2002c45 100644
--- a/commands.h
+++ b/commands.h
@@ -93,6 +93,7 @@ extern const struct cmd_group inspect_cmd_group;
extern const struct cmd_group property_cmd_group;
extern const struct cmd_group quota_cmd_group;
extern const struct cmd_group qgroup_cmd_group;
+extern const struct cmd_group dedupe_ib_cmd_group;
extern const struct cmd_group replace_cmd_group;
extern const struct cmd_group rescue_cmd_group;
@@ -118,6 +119,7 @@ int cmd_send(int argc, char **argv);
int cmd_receive(int argc, char **argv);
int cmd_quota(int argc, char **argv);
int cmd_qgroup(int argc, char **argv);
+int cmd_dedupe_ib(int argc, char **argv);
int cmd_replace(int argc, char **argv);
int cmd_restore(int argc, char **argv);
int cmd_select_super(int argc, char **argv);
diff --git a/ioctl.h b/ioctl.h
index 5f18bcb..c88bca2 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -503,6 +503,27 @@ struct btrfs_ioctl_get_dev_stats {
__u64 unused[128 - 2 - BTRFS_DEV_STAT_VALUES_MAX]; /* pad to 1k */
};
+/*
+ * de-duplication control modes
+ * For re-config, enable will handle it
+ */
+#define BTRFS_DEDUPE_CTL_ENABLE 1
+#define BTRFS_DEDUPE_CTL_DISABLE 2
+#define BTRFS_DEDUPE_CTL_STATUS 3
+#define BTRFS_DEDUPE_CTL_LAST 4
+struct btrfs_ioctl_dedupe_args {
+ __u16 cmd; /* In: command(see above macro) */
+ __u64 blocksize; /* In/Out: For enable/status */
+ __u64 limit_nr; /* In/Out: For enable/status */
+ __u64 limit_mem; /* In/Out: For enable/status */
+ __u64 current_nr; /* Out: For status output */
+ __u16 backend; /* In/Out: For enable/status */
+ __u16 hash_type; /* In/Out: For enable/status */
+ u8 status; /* Out: For status output */
+ /* pad to 512 bytes */
+ u8 __unused[473];
+};
+
/* BTRFS_IOC_SNAP_CREATE is no longer used by the btrfs command */
#define BTRFS_QUOTA_CTL_ENABLE 1
#define BTRFS_QUOTA_CTL_DISABLE 2
--
2.8.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH v8 2/4] btrfs-progs: dedupe: Add enable command for dedupe command group
2016-06-17 1:02 [PATCH v8 0/4] Inband dedupe for btrfs-progs Qu Wenruo
2016-06-17 1:02 ` [PATCH v8 1/4] btrfs-progs: Basic framework for dedupe-inband command group Qu Wenruo
@ 2016-06-17 1:02 ` Qu Wenruo
2016-06-17 1:02 ` [PATCH v8 3/4] btrfs-progs: dedupe: Add disable support for inband dedupelication Qu Wenruo
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Qu Wenruo @ 2016-06-17 1:02 UTC (permalink / raw)
To: linux-btrfs
Add enable subcommand for dedupe commmand group.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
Documentation/btrfs-dedupe-inband.asciidoc | 106 ++++++++++++++++++++-
btrfs-completion | 6 +-
cmds-dedupe-ib.c | 148 +++++++++++++++++++++++++++++
ioctl.h | 2 +
4 files changed, 260 insertions(+), 2 deletions(-)
diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index 9ee2bc7..90c1a32 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -22,7 +22,111 @@ use with caution.
SUBCOMMAND
----------
-Nothing yet
+*enable* [options] <path>::
+Enable in-band de-duplication for a filesystem.
++
+`Options`
++
+-s|--storage-backend <BACKEND>::::
+Specify de-duplication hash storage backend.
+Only 'inmemory' backend is supported yet.
+If not specified, default value is 'inmemory'.
++
+Refer to *BACKENDS* sector for more information.
+
+-b|--blocksize <BLOCKSIZE>::::
+Specify dedupe block size.
+Supported values are power of 2 from '16K' to '8M'.
+Default value is '128K'.
++
+Refer to *BLOCKSIZE* sector for more information.
+
+-a|--hash-algorithm <HASH>::::
+Specify hash algorithm.
+Only 'sha256' is supported yet.
+
+-l|--limit-hash <LIMIT>::::
+Specify maximum number of hashes stored in memory.
+Only works for 'inmemory' backend.
+Conflicts with '-m' option.
++
+Only positive values are valid.
+Default value is '32K'.
+
+-m|--limit-memory <LIMIT>::::
+Specify maximum memory used for hashes.
+Only works for 'inmemory' backend.
+Conflicts with '-l' option.
++
+Only value larger than or equal to '1024' is valid.
+No default value.
++
+NOTE: Memory limit will be rounded down to kernel internal hash size,
+so the memory limit shown in 'btrfs dedupe status' may be different
+from the <LIMIT>.
+
+WARNING: Too large value for '-l' or '-m' will easily trigger OOM.
+Please use with caution according to system memory.
+
+NOTE: In-band de-duplication is not compactible with compression yet.
+And compression has higher priority than in-band de-duplication, means if
+compression and de-duplication is enabled at the same time, only compression
+will work.
+
+BACKENDS
+--------
+Btrfs in-band de-duplication will support different storage backends, with
+different use case and features.
+
+In-memory backend::
+This backend provides backward-compatibility, and more fine-tuning options.
+But hash pool is non-persistent and may exhaust kernel memory if not setup
+properly.
++
+This backend can be used on old btrfs(without '-O dedupe' mkfs option).
+When used on old btrfs, this backend needs to be enabled manually after mount.
++
+Designed for fast hash search speed, in-memory backend will keep all dedupe
+hashes in memory. (Although overall performance is still much the same with
+'ondisk' backend if all 'ondisk' hash can be cached in memory)
++
+And only keeps limited number of hash in memory to avoid exhausting memory.
+Hashes over the limit will be dropped following Last-Recent-Use behavior.
+So this backend has a consistent overhead for given limit but can\'t ensure
+all duplicated blocks will be de-duplicated.
++
+After umount and mount, in-memory backend need to refill its hash pool.
+
+On-disk backend::
+This backend provides persistent hash pool, with more smart memory management
+for hash pool.
+But it\'s not backward-compatible, meaning it must be used with '-O dedupe' mkfs
+option and older kernel can\'t mount it read-write.
++
+Designed for de-duplication rate, hash pool is stored as btrfs B+ tree on disk.
+This behavior may cause extra disk IO for hash search under high memory
+pressure.
++
+After umount and mount, on-disk backend still has its hash on disk, no need to
+refill its dedupe hash pool.
+
+Currently, only 'inmemory' backend is supported in btrfs-progs.
+
+DEDUPE BLOCK SIZE
+----------------
+In-band de-duplication is done at dedupe block size.
+Any data smaller than dedupe block size won\'t go through in-band
+de-duplication.
+
+And dedupe block size affects dedupe rate and fragmentation heavily.
+
+Smaller block size will cause more fragments, but higher dedupe rate.
+
+Larger block size will cause less fragments, but lower dedupe rate.
+
+In-band de-duplication rate is highly related to the workload pattern.
+So it\'s highly recommended to align dedupe block size to the workload
+block size to make full use of de-duplication.
EXIT STATUS
-----------
diff --git a/btrfs-completion b/btrfs-completion
index 3ede77b..50f7ea2 100644
--- a/btrfs-completion
+++ b/btrfs-completion
@@ -29,7 +29,7 @@ _btrfs()
local cmd=${words[1]}
- commands='subvolume filesystem balance device scrub check rescue restore inspect-internal property send receive quota qgroup replace help version'
+ commands='subvolume filesystem balance device scrub check rescue restore inspect-internal property send receive quota qgroup dedupe replace help version'
commands_subvolume='create delete list snapshot find-new get-default set-default show sync'
commands_filesystem='defragment sync resize show df label usage'
commands_balance='start pause cancel resume status'
@@ -40,6 +40,7 @@ _btrfs()
commands_property='get set list'
commands_quota='enable disable rescan'
commands_qgroup='assign remove create destroy show limit'
+ commands_dedupe='enable'
commands_replace='start status cancel'
if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
@@ -94,6 +95,9 @@ _btrfs()
qgroup)
opts="$commands_qgroup"
;;
+ dedupe)
+ opts="$commands_dedupe"
+ ;;
replace)
opts="$commands_replace"
;;
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index 160c8ec..94418c3 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -19,6 +19,7 @@
#include <getopt.h>
#include <unistd.h>
#include <sys/ioctl.h>
+#include <sys/ioctl.h>
#include "ctree.h"
#include "ioctl.h"
@@ -36,8 +37,155 @@ static const char * const dedupe_ib_cmd_group_usage[] = {
static const char dedupe_ib_cmd_group_info[] =
"manage inband(write time) de-duplication";
+static const char * const cmd_dedupe_ib_enable_usage[] = {
+ "btrfs dedupe-inband enable [options] <path>",
+ "Enable in-band(write time) de-duplication of a btrfs.",
+ "",
+ "-s|--storage-backend <BACKEND>",
+ " specify dedupe hash storage backend",
+ " supported backend: 'inmemory'",
+ "-b|--blocksize <BLOCKSIZE>",
+ " specify dedupe block size",
+ " default value is 128K",
+ "-a|--hash-algorithm <HASH>",
+ " specify hash algorithm",
+ " only 'sha256' is supported yet",
+ "-l|--limit-hash <LIMIT>",
+ " specify maximum number of hashes stored in memory",
+ " only for 'inmemory' backend",
+ " positive value is valid, default value is 32K",
+ "-m|--limit-mem <LIMIT>",
+ " specify maximum memory used for hashes",
+ " only for 'inmemory' backend",
+ " value larger than or equal to 1024 is valid, no default",
+ " only one of '-m' and '-l' is allowed",
+ NULL
+};
+
+static int cmd_dedupe_ib_enable(int argc, char **argv)
+{
+ int ret;
+ int fd;
+ char *path;
+ u64 blocksize = BTRFS_DEDUPE_BLOCKSIZE_DEFAULT;
+ u16 hash_type = BTRFS_DEDUPE_HASH_SHA256;
+ u16 backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
+ u64 limit_nr = 0;
+ u64 limit_mem = 0;
+ struct btrfs_ioctl_dedupe_args dargs;
+ DIR *dirstream;
+
+ while (1) {
+ int c;
+ static const struct option long_options[] = {
+ { "storage-backend", required_argument, NULL, 's'},
+ { "blocksize", required_argument, NULL, 'b'},
+ { "hash-algorithm", required_argument, NULL, 'a'},
+ { "limit-hash", required_argument, NULL, 'l'},
+ { "limit-memory", required_argument, NULL, 'm'},
+ { NULL, 0, NULL, 0}
+ };
+
+ c = getopt_long(argc, argv, "s:b:a:l:m:", long_options, NULL);
+ if (c < 0)
+ break;
+ switch (c) {
+ case 's':
+ if (!strcasecmp("inmemory", optarg))
+ backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
+ else {
+ error("unsupported dedupe backend: %s", optarg);
+ exit(1);
+ }
+ break;
+ case 'b':
+ blocksize = parse_size(optarg);
+ break;
+ case 'a':
+ if (strcmp("sha256", optarg)) {
+ error("unsupported dedupe hash algorithm: %s",
+ optarg);
+ return 1;
+ }
+ break;
+ case 'l':
+ limit_nr = parse_size(optarg);
+ if (limit_nr == 0) {
+ error("limit should be larger than 0");
+ return 1;
+ }
+ break;
+ case 'm':
+ limit_mem = parse_size(optarg);
+ /*
+ * Make sure at least one hash is allocated
+ * 1024 should be good enough though.
+ */
+ if (limit_mem < 1024) {
+ error("memory limit should be larger than or equal to 1024");
+ return 1;
+ }
+ break;
+ }
+ }
+
+ path = argv[optind];
+ if (check_argc_exact(argc - optind, 1))
+ usage(cmd_dedupe_ib_enable_usage);
+
+ /* Validation check */
+ if (!is_power_of_2(blocksize) ||
+ blocksize > BTRFS_DEDUPE_BLOCKSIZE_MAX ||
+ blocksize < BTRFS_DEDUPE_BLOCKSIZE_MIN) {
+ error("invalid dedupe blocksize: %llu, not in range [%u,%u] or power of 2",
+ blocksize, BTRFS_DEDUPE_BLOCKSIZE_MIN,
+ BTRFS_DEDUPE_BLOCKSIZE_MAX);
+ return 1;
+ }
+ if ((limit_nr || limit_mem) && backend != BTRFS_DEDUPE_BACKEND_INMEMORY) {
+ error("limit is only valid for 'inmemory' backend");
+ return 1;
+ }
+ if (limit_nr && limit_mem) {
+ error("limit-memory and limit-hash can't be given at the same time");
+ return 1;
+ }
+ /*
+ * TODO: Add check for limit_nr/limit_mem against current system
+ * memory to avoid wrongly set limit.
+ */
+
+ fd = open_file_or_dir(path, &dirstream);
+ if (fd < 0) {
+ error("failed to open file or directory: %s", path);
+ return 1;
+ }
+ memset(&dargs, 0, sizeof(dargs));
+ dargs.cmd = BTRFS_DEDUPE_CTL_ENABLE;
+ dargs.blocksize = blocksize;
+ dargs.hash_type = hash_type;
+ dargs.limit_nr = limit_nr;
+ dargs.limit_mem = limit_mem;
+ dargs.backend = backend;
+
+ ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
+ if (ret < 0) {
+ error("failed to enable inband deduplication: %s",
+ strerror(errno));
+ ret = 1;
+ goto out;
+ }
+ ret = 0;
+
+out:
+ close_file_or_dir(fd, dirstream);
+ return ret;
+}
+
const struct cmd_group dedupe_ib_cmd_group = {
dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
+ { "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
+ NULL, 0},
NULL_CMD_STRUCT
}
};
diff --git a/ioctl.h b/ioctl.h
index c88bca2..87120c2 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -735,6 +735,8 @@ static inline char *btrfs_err_str(enum btrfs_err_code err_code)
struct btrfs_ioctl_dev_replace_args)
#define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
struct btrfs_ioctl_same_args)
+#define BTRFS_IOC_DEDUPE_CTL _IOWR(BTRFS_IOCTL_MAGIC, 55, \
+ struct btrfs_ioctl_dedupe_args)
#define BTRFS_IOC_GET_FEATURES _IOR(BTRFS_IOCTL_MAGIC, 57, \
struct btrfs_ioctl_feature_flags)
#define BTRFS_IOC_SET_FEATURES _IOW(BTRFS_IOCTL_MAGIC, 57, \
--
2.8.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH v8 3/4] btrfs-progs: dedupe: Add disable support for inband dedupelication
2016-06-17 1:02 [PATCH v8 0/4] Inband dedupe for btrfs-progs Qu Wenruo
2016-06-17 1:02 ` [PATCH v8 1/4] btrfs-progs: Basic framework for dedupe-inband command group Qu Wenruo
2016-06-17 1:02 ` [PATCH v8 2/4] btrfs-progs: dedupe: Add enable command for dedupe " Qu Wenruo
@ 2016-06-17 1:02 ` Qu Wenruo
2016-06-17 1:02 ` [PATCH v8 4/4] btrfs-progs: dedupe: Add status subcommand Qu Wenruo
2016-06-17 1:18 ` [PATCH v8.1 1/4] btrfs-progs: Basic framework for dedupe-inband command group Qu Wenruo
4 siblings, 0 replies; 6+ messages in thread
From: Qu Wenruo @ 2016-06-17 1:02 UTC (permalink / raw)
To: linux-btrfs
Add disable subcommand for dedupe command group.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
Documentation/btrfs-dedupe-inband.asciidoc | 5 ++++
btrfs-completion | 2 +-
cmds-dedupe-ib.c | 42 ++++++++++++++++++++++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index 90c1a32..b2b0e82 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -22,6 +22,11 @@ use with caution.
SUBCOMMAND
----------
+*disable* <path>::
+Disable in-band de-duplication for a filesystem.
++
+This will trash all stored dedupe hash.
++
*enable* [options] <path>::
Enable in-band de-duplication for a filesystem.
+
diff --git a/btrfs-completion b/btrfs-completion
index 50f7ea2..9a6c73b 100644
--- a/btrfs-completion
+++ b/btrfs-completion
@@ -40,7 +40,7 @@ _btrfs()
commands_property='get set list'
commands_quota='enable disable rescan'
commands_qgroup='assign remove create destroy show limit'
- commands_dedupe='enable'
+ commands_dedupe='enable disable'
commands_replace='start status cancel'
if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index 94418c3..e3497b5 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -182,10 +182,52 @@ out:
return ret;
}
+static const char * const cmd_dedupe_ib_disable_usage[] = {
+ "btrfs dedupe disable <path>",
+ "Disable in-band(write time) de-duplication of a btrfs.",
+ NULL
+};
+
+static int cmd_dedupe_ib_disable(int argc, char **argv)
+{
+ struct btrfs_ioctl_dedupe_args dargs;
+ DIR *dirstream;
+ char *path;
+ int fd;
+ int ret;
+
+ if (check_argc_exact(argc, 2))
+ usage(cmd_dedupe_ib_disable_usage);
+
+ path = argv[1];
+ fd = open_file_or_dir(path, &dirstream);
+ if (fd < 0) {
+ error("failed to open file or directory: %s", path);
+ return 1;
+ }
+ memset(&dargs, 0, sizeof(dargs));
+ dargs.cmd = BTRFS_DEDUPE_CTL_DISABLE;
+
+ ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
+ if (ret < 0) {
+ error("failed to disable inband deduplication: %s",
+ strerror(errno));
+ ret = 1;
+ goto out;
+ }
+ ret = 0;
+
+out:
+ close_file_or_dir(fd, dirstream);
+ return 0;
+}
+
const struct cmd_group dedupe_ib_cmd_group = {
dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
NULL, 0},
+ { "disable", cmd_dedupe_ib_disable, cmd_dedupe_ib_disable_usage,
+ NULL, 0},
NULL_CMD_STRUCT
}
};
--
2.8.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH v8 4/4] btrfs-progs: dedupe: Add status subcommand
2016-06-17 1:02 [PATCH v8 0/4] Inband dedupe for btrfs-progs Qu Wenruo
` (2 preceding siblings ...)
2016-06-17 1:02 ` [PATCH v8 3/4] btrfs-progs: dedupe: Add disable support for inband dedupelication Qu Wenruo
@ 2016-06-17 1:02 ` Qu Wenruo
2016-06-17 1:18 ` [PATCH v8.1 1/4] btrfs-progs: Basic framework for dedupe-inband command group Qu Wenruo
4 siblings, 0 replies; 6+ messages in thread
From: Qu Wenruo @ 2016-06-17 1:02 UTC (permalink / raw)
To: linux-btrfs
Add status subcommand for dedupe command group.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
Documentation/btrfs-dedupe-inband.asciidoc | 3 ++
btrfs-completion | 2 +-
cmds-dedupe-ib.c | 81 ++++++++++++++++++++++++++++++
3 files changed, 85 insertions(+), 1 deletion(-)
diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index b2b0e82..767e46c 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -78,6 +78,9 @@ And compression has higher priority than in-band de-duplication, means if
compression and de-duplication is enabled at the same time, only compression
will work.
+*status* <path>::
+Show current in-band de-duplication status of a filesystem.
+
BACKENDS
--------
Btrfs in-band de-duplication will support different storage backends, with
diff --git a/btrfs-completion b/btrfs-completion
index 9a6c73b..fbaae0c 100644
--- a/btrfs-completion
+++ b/btrfs-completion
@@ -40,7 +40,7 @@ _btrfs()
commands_property='get set list'
commands_quota='enable disable rescan'
commands_qgroup='assign remove create destroy show limit'
- commands_dedupe='enable disable'
+ commands_dedupe='enable disable status'
commands_replace='start status cancel'
if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index e3497b5..9dd155e 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -222,12 +222,93 @@ out:
return 0;
}
+static const char * const cmd_dedupe_ib_status_usage[] = {
+ "btrfs dedupe status <path>",
+ "Show current in-band(write time) de-duplication status of a btrfs.",
+ NULL
+};
+
+static int cmd_dedupe_ib_status(int argc, char **argv)
+{
+ struct btrfs_ioctl_dedupe_args dargs;
+ DIR *dirstream;
+ char *path;
+ int fd;
+ int ret;
+ int print_limit = 1;
+
+ if (check_argc_exact(argc, 2))
+ usage(cmd_dedupe_ib_status_usage);
+
+ path = argv[1];
+ fd = open_file_or_dir(path, &dirstream);
+ if (fd < 0) {
+ error("failed to open file or directory: %s", path);
+ ret = 1;
+ goto out;
+ }
+ memset(&dargs, 0, sizeof(dargs));
+ dargs.cmd = BTRFS_DEDUPE_CTL_STATUS;
+
+ ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
+ if (ret < 0) {
+ error("failed to get inband deduplication status: %s",
+ strerror(errno));
+ ret = 1;
+ goto out;
+ }
+ ret = 0;
+ if (dargs.status == 0) {
+ printf("Status: \t\t\tDisabled\n");
+ goto out;
+ }
+ printf("Status:\t\t\tEnabled\n");
+
+ if (dargs.hash_type == BTRFS_DEDUPE_HASH_SHA256)
+ printf("Hash algorithm:\t\tSHA-256\n");
+ else
+ printf("Hash algorithm:\t\tUnrecognized(%x)\n",
+ dargs.hash_type);
+
+ if (dargs.backend == BTRFS_DEDUPE_BACKEND_INMEMORY) {
+ printf("Backend:\t\tIn-memory\n");
+ print_limit = 1;
+ } else {
+ printf("Backend:\t\tUnrecognized(%x)\n",
+ dargs.backend);
+ }
+
+ printf("Dedup Blocksize:\t%llu\n", dargs.blocksize);
+
+ if (print_limit) {
+ u64 cur_mem;
+
+ /* Limit nr may be 0 */
+ if (dargs.limit_nr)
+ cur_mem = dargs.current_nr * (dargs.limit_mem /
+ dargs.limit_nr);
+ else
+ cur_mem = 0;
+
+ printf("Number of hash: \t[%llu/%llu]\n", dargs.current_nr,
+ dargs.limit_nr);
+ printf("Memory usage: \t\t[%s/%s]\n",
+ pretty_size(cur_mem),
+ pretty_size(dargs.limit_mem));
+ }
+out:
+ close_file_or_dir(fd, dirstream);
+ return ret;
+}
+
const struct cmd_group dedupe_ib_cmd_group = {
dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
NULL, 0},
{ "disable", cmd_dedupe_ib_disable, cmd_dedupe_ib_disable_usage,
NULL, 0},
+ { "status", cmd_dedupe_ib_status, cmd_dedupe_ib_status_usage,
+ NULL, 0},
NULL_CMD_STRUCT
}
};
--
2.8.3
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH v8.1 1/4] btrfs-progs: Basic framework for dedupe-inband command group
2016-06-17 1:02 [PATCH v8 0/4] Inband dedupe for btrfs-progs Qu Wenruo
` (3 preceding siblings ...)
2016-06-17 1:02 ` [PATCH v8 4/4] btrfs-progs: dedupe: Add status subcommand Qu Wenruo
@ 2016-06-17 1:18 ` Qu Wenruo
4 siblings, 0 replies; 6+ messages in thread
From: Qu Wenruo @ 2016-06-17 1:18 UTC (permalink / raw)
To: linux-btrfs
Add basic ioctl header and command group framework for later use.
Alone with basic man page doc.
Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
Add missing dedupe-in.h
---
Documentation/Makefile.in | 1 +
Documentation/btrfs-dedupe-inband.asciidoc | 40 +++++++++++++++++++++++++
Documentation/btrfs.asciidoc | 4 +++
Makefile.in | 3 +-
btrfs.c | 2 ++
cmds-dedupe-ib.c | 48 ++++++++++++++++++++++++++++++
commands.h | 2 ++
dedupe-ib.h | 41 +++++++++++++++++++++++++
ioctl.h | 21 +++++++++++++
9 files changed, 161 insertions(+), 1 deletion(-)
create mode 100644 Documentation/btrfs-dedupe-inband.asciidoc
create mode 100644 cmds-dedupe-ib.c
create mode 100644 dedupe-ib.h
diff --git a/Documentation/Makefile.in b/Documentation/Makefile.in
index aea2cb4..7db4164 100644
--- a/Documentation/Makefile.in
+++ b/Documentation/Makefile.in
@@ -28,6 +28,7 @@ MAN8_TXT += btrfs-qgroup.asciidoc
MAN8_TXT += btrfs-replace.asciidoc
MAN8_TXT += btrfs-restore.asciidoc
MAN8_TXT += btrfs-property.asciidoc
+MAN8_TXT += btrfs-dedupe-inband.asciidoc
# Category 5 manual page
MAN5_TXT += btrfs-man5.asciidoc
diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
new file mode 100644
index 0000000..9ee2bc7
--- /dev/null
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -0,0 +1,40 @@
+btrfs-dedupe(8)
+==============
+
+NAME
+----
+btrfs-dedupe-inband - manage in-band (write time) de-duplication of a btrfs
+filesystem
+
+SYNOPSIS
+--------
+*btrfs dedupe-inband* <subcommand> <args>
+
+DESCRIPTION
+-----------
+*btrfs dedupe-inband* is used to enable/disable or show current in-band de-duplication
+status of a btrfs filesystem.
+
+Kernel support for in-band de-duplication starts from 4.8.
+
+WARNING: In-band de-duplication is still an experimental feautre of btrfs,
+use with caution.
+
+SUBCOMMAND
+----------
+Nothing yet
+
+EXIT STATUS
+-----------
+*btrfs dedupe-inband* returns a zero exit status if it succeeds. Non zero is
+returned in case of failure.
+
+AVAILABILITY
+------------
+*btrfs* is part of btrfs-progs.
+Please refer to the btrfs wiki http://btrfs.wiki.kernel.org for
+further details.
+
+SEE ALSO
+--------
+`mkfs.btrfs`(8),
diff --git a/Documentation/btrfs.asciidoc b/Documentation/btrfs.asciidoc
index 6a77a85..8ded842 100644
--- a/Documentation/btrfs.asciidoc
+++ b/Documentation/btrfs.asciidoc
@@ -43,6 +43,10 @@ COMMANDS
Do off-line check on a btrfs filesystem. +
See `btrfs-check`(8) for details.
+*dedupe*::
+ Control btrfs in-band(write time) de-duplication. +
+ See `btrfs-dedupe`(8) for details.
+
*device*::
Manage devices managed by btrfs, including add/delete/scan and so
on. +
diff --git a/Makefile.in b/Makefile.in
index ac6b353..5cf0fc2 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -86,7 +86,8 @@ cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
cmds-quota.o cmds-qgroup.o cmds-replace.o cmds-check.o \
cmds-restore.o cmds-rescue.o chunk-recover.o super-recover.o \
cmds-property.o cmds-fi-usage.o cmds-inspect-dump-tree.o \
- cmds-inspect-dump-super.o cmds-inspect-tree-stats.o cmds-fi-du.o
+ cmds-inspect-dump-super.o cmds-inspect-tree-stats.o cmds-fi-du.o \
+ cmds-dedupe-ib.o
libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \
uuid-tree.o utils-lib.o rbtree-utils.o
libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
diff --git a/btrfs.c b/btrfs.c
index cc70515..06c211b 100644
--- a/btrfs.c
+++ b/btrfs.c
@@ -199,6 +199,8 @@ static const struct cmd_group btrfs_cmd_group = {
{ "receive", cmd_receive, cmd_receive_usage, NULL, 0 },
{ "quota", cmd_quota, NULL, "a_cmd_group, 0 },
{ "qgroup", cmd_qgroup, NULL, &qgroup_cmd_group, 0 },
+ { "dedupe-inband", cmd_dedupe_ib, NULL, &dedupe_ib_cmd_group,
+ 0 },
{ "replace", cmd_replace, NULL, &replace_cmd_group, 0 },
{ "help", cmd_help, cmd_help_usage, NULL, 0 },
{ "version", cmd_version, cmd_version_usage, NULL, 0 },
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
new file mode 100644
index 0000000..160c8ec
--- /dev/null
+++ b/cmds-dedupe-ib.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Fujitsu. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "ctree.h"
+#include "ioctl.h"
+
+#include "commands.h"
+#include "utils.h"
+#include "kerncompat.h"
+#include "dedupe-ib.h"
+
+static const char * const dedupe_ib_cmd_group_usage[] = {
+ "btrfs dedupe-inband <command> [options] <path>",
+ NULL
+};
+
+static const char dedupe_ib_cmd_group_info[] =
+"manage inband(write time) de-duplication";
+
+const struct cmd_group dedupe_ib_cmd_group = {
+ dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
+ NULL_CMD_STRUCT
+ }
+};
+
+int cmd_dedupe_ib(int argc, char **argv)
+{
+ return handle_command_group(&dedupe_ib_cmd_group, argc, argv);
+}
diff --git a/commands.h b/commands.h
index 94229c1..2002c45 100644
--- a/commands.h
+++ b/commands.h
@@ -93,6 +93,7 @@ extern const struct cmd_group inspect_cmd_group;
extern const struct cmd_group property_cmd_group;
extern const struct cmd_group quota_cmd_group;
extern const struct cmd_group qgroup_cmd_group;
+extern const struct cmd_group dedupe_ib_cmd_group;
extern const struct cmd_group replace_cmd_group;
extern const struct cmd_group rescue_cmd_group;
@@ -118,6 +119,7 @@ int cmd_send(int argc, char **argv);
int cmd_receive(int argc, char **argv);
int cmd_quota(int argc, char **argv);
int cmd_qgroup(int argc, char **argv);
+int cmd_dedupe_ib(int argc, char **argv);
int cmd_replace(int argc, char **argv);
int cmd_restore(int argc, char **argv);
int cmd_select_super(int argc, char **argv);
diff --git a/dedupe-ib.h b/dedupe-ib.h
new file mode 100644
index 0000000..02c5f8f
--- /dev/null
+++ b/dedupe-ib.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2015 Fujitsu. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#ifndef __BTRFS_DEDUPE__
+#define __BTRFS_DEDUPE__
+
+/*
+ * Dedup storage backend
+ * On disk is persist storage but overhead is large
+ * In memory is fast but will lose all its hash on umount
+ */
+#define BTRFS_DEDUPE_BACKEND_INMEMORY 0
+#define BTRFS_DEDUPE_BACKEND_LAST 1
+
+/* Dedup block size limit and default value */
+#define BTRFS_DEDUPE_BLOCKSIZE_MAX (8 * 1024 * 1024)
+#define BTRFS_DEDUPE_BLOCKSIZE_MIN (16 * 1024)
+#define BTRFS_DEDUPE_BLOCKSIZE_DEFAULT (128 * 1024)
+
+/* Default dedupe limit on number of hash */
+#define BTRFS_DEDUPE_LIMIT_NR_DEFAULT (32 * 1024)
+
+/* Hash algorithm, only support SHA256 yet */
+#define BTRFS_DEDUPE_HASH_SHA256 0
+
+#endif
diff --git a/ioctl.h b/ioctl.h
index 5f18bcb..c88bca2 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -503,6 +503,27 @@ struct btrfs_ioctl_get_dev_stats {
__u64 unused[128 - 2 - BTRFS_DEV_STAT_VALUES_MAX]; /* pad to 1k */
};
+/*
+ * de-duplication control modes
+ * For re-config, enable will handle it
+ */
+#define BTRFS_DEDUPE_CTL_ENABLE 1
+#define BTRFS_DEDUPE_CTL_DISABLE 2
+#define BTRFS_DEDUPE_CTL_STATUS 3
+#define BTRFS_DEDUPE_CTL_LAST 4
+struct btrfs_ioctl_dedupe_args {
+ __u16 cmd; /* In: command(see above macro) */
+ __u64 blocksize; /* In/Out: For enable/status */
+ __u64 limit_nr; /* In/Out: For enable/status */
+ __u64 limit_mem; /* In/Out: For enable/status */
+ __u64 current_nr; /* Out: For status output */
+ __u16 backend; /* In/Out: For enable/status */
+ __u16 hash_type; /* In/Out: For enable/status */
+ u8 status; /* Out: For status output */
+ /* pad to 512 bytes */
+ u8 __unused[473];
+};
+
/* BTRFS_IOC_SNAP_CREATE is no longer used by the btrfs command */
#define BTRFS_QUOTA_CTL_ENABLE 1
#define BTRFS_QUOTA_CTL_DISABLE 2
--
2.8.3
^ permalink raw reply related [flat|nested] 6+ messages in thread