[PATCH v2 0/2] btrfs-progs: cmds/filesystem: add --usage-ratio and --wasted-bytes options

[PATCH v2 0/2] btrfs-progs: cmds/filesystem: add --usage-ratio and --wasted-bytes options

[Qu Wenruo]

[CHANGELOG]
v2:
- Sync the newer kernel uapi
- Remove the "lone" mentions
  Now the new options would be "--usage-ratio" and "--wasted-bytes".

This the progs support for the new kernel defrag parameters.

This adds 2 new fine tunning parameters, --usage-ratio and
--wasted-bytes.

The ratio is between [0, 100] (aka, percentage value), and wasted bytes
is between [0, U32_MAX], but in reality the value only makes sense below
max file extent size (for both compressed and regular extents).
Any value higher than max file extent size would mostly disable the
wasted bytes check (as it would always be false).

The default usage ratio is 5%, and 16MiB wasted bytes.
If the kernel doesn't support the new options, it would fall back to
the old ioctl flags without the 2 new flags.

Qu Wenruo (2):
  btrfs-progs: defrag: sync the usage ratio/wasted bytes fine-tunning
    from kernel
  btrfs-progs: cmds/filesystem: add --usage-ratio and --wasted-bytes for
    defrag

 Documentation/btrfs-filesystem.rst | 21 ++++++++++++
 cmds/filesystem.c                  | 53 ++++++++++++++++++++++++++++--
 kernel-shared/uapi/btrfs.h         | 39 ++++++++++++++++++++--
 3 files changed, 107 insertions(+), 6 deletions(-)

--
2.44.0

[PATCH v2 1/2] btrfs-progs: defrag: sync the usage ratio/wasted bytes fine-tunning from kernel

[Qu Wenruo]

This would add the defrag usage ratio and wasted bytes fine-tunning
support for defrag.

Signed-off-by: Qu Wenruo <wqu@suse.com>
---
 kernel-shared/uapi/btrfs.h | 39 +++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/kernel-shared/uapi/btrfs.h b/kernel-shared/uapi/btrfs.h
index 34e84b89edf2..b595a8deb0be 100644
--- a/kernel-shared/uapi/btrfs.h
+++ b/kernel-shared/uapi/btrfs.h
@@ -642,8 +642,15 @@ _static_assert(sizeof(struct btrfs_ioctl_clone_range_args) == 32);
  * Used by:
  * struct btrfs_ioctl_defrag_range_args.flags
  */
-#define BTRFS_DEFRAG_RANGE_COMPRESS 1
-#define BTRFS_DEFRAG_RANGE_START_IO 2
+#define BTRFS_DEFRAG_RANGE_COMPRESS		(1ULL << 0)
+#define BTRFS_DEFRAG_RANGE_START_IO		(1ULL << 1)
+#define BTRFS_DEFRAG_RANGE_USAGE_RATIO		(1ULL << 2)
+#define BTRFS_DEFRAG_RANGE_WASTED_BYTES		(1ULL << 3)
+#define BTRFS_DEFRAG_RANGE_FLAGS_SUPP	(BTRFS_DEFRAG_RANGE_COMPRESS |	\
+					 BTRFS_DEFRAG_RANGE_START_IO |	\
+					 BTRFS_DEFRAG_RANGE_USAGE_RATIO |\
+					 BTRFS_DEFRAG_RANGE_WASTED_BYTES)
+
 struct btrfs_ioctl_defrag_range_args {
 	/* start of the defrag operation */
 	__u64 start;
@@ -671,8 +678,34 @@ struct btrfs_ioctl_defrag_range_args {
 	 */
 	__u32 compress_type;
 
+	/*
+	 * File extents which has lower usage ratio than this would be defragged.
+	 *
+	 * Valid values are [0, 100].
+	 *
+	 * 0 means no check based on usage ratio.
+	 * 1 means one file extent would be defragged if its referred size
+	 * (file extent num bytes) is smaller than 1% of its on-disk extent size.
+	 * 100 means one file extent would be defragged if its referred size
+	 * (file extent num bytes) is smaller than 100% of its on-disk extent size.
+	 */
+	__u32 usage_ratio;
+
+	/*
+	 * File extents which has more "wasted" bytes than this would be
+	 * defragged.
+	 *
+	 * "Wasted" bytes just means the difference between the file extent size
+	 * (file extent num bytes) against the on-disk extent size
+	 * (file extent disk num bytes).
+	 *
+	 * Valid values are [0, U32_MAX], but values larger than
+	 * BTRFS_MAX_EXTENT_SIZE would not make much sense.
+	 */
+	__u32 wasted_bytes;
+
 	/* spare for later */
-	__u32 unused[4];
+	__u32 unused[2];
 };
 _static_assert(sizeof(struct btrfs_ioctl_defrag_range_args) == 48);
 
-- 
2.44.0

[PATCH v2 2/2] btrfs-progs: cmds/filesystem: add --usage-ratio and --wasted-bytes for defrag

[Qu Wenruo]

This allows defrag subcommand to accept new "--usage-ratio" and
"--wasted-bytes" options.

These two options allows defrag to handle extents which have no adjacent
mergable extents but has low usage ratio or high wasted bytes against
its on-disk extent.

The default behavior is, defrag would go with 5% as the usage_ratio
and 16M as the wasted bytes on the first try.

If the kernel doesn't support it, then retry without the fine tunning
flags.

Signed-off-by: Qu Wenruo <wqu@suse.com>
---
 Documentation/btrfs-filesystem.rst | 21 ++++++++++++
 cmds/filesystem.c                  | 53 ++++++++++++++++++++++++++++--
 2 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/Documentation/btrfs-filesystem.rst b/Documentation/btrfs-filesystem.rst
index 83207a5a263e..213be6ff4a14 100644
--- a/Documentation/btrfs-filesystem.rst
+++ b/Documentation/btrfs-filesystem.rst
@@ -144,6 +144,27 @@ defragment [options] <file>|<dir> [<file>|<dir>...]
                 The range is default (the whole file) or given by *-s* and *-l*, split into
                 the steps or done in one go if the step is larger. Minimum range size is 256KiB.
 
+	--usage-ratio <ratio>
+		If a file extent which is not a defrag target (e.g. no mergeable
+		adjacent extent), but has a usage ratio lower than (ratio / 100)
+		of the on-disk extent size, it would also be defraged
+		for its potential to free up the on-disk extent.
+
+		Valid values are in the range [0, 100].
+
+	--wasted-bytes <bytes>[kKmMgGgtTpPeE]
+		If a file extent which is not a defrag target (e.g. no mergeable
+		adjacent extent), but has more "wasted" bytes (the difference
+		between the file extent and the on-disk extent size) than this
+		value, it would also be defragged for its potential to free up
+		on-disk extent.
+
+		Valid values are in the range [0, U32_MAX], but any value larger than 128K
+		would make no sense for compressed file extents, and any value larger than
+		128M would make no sense for regular file extents.
+		As the largest file extent supported is 128K for compressed extent and 128M
+		otherwise, thus it's impossible to free up more bytes than those limits.
+
         -v
                 (deprecated) alias for global *-v* option
 
diff --git a/cmds/filesystem.c b/cmds/filesystem.c
index c9930a02d879..9cc224454e4c 100644
--- a/cmds/filesystem.c
+++ b/cmds/filesystem.c
@@ -914,6 +914,8 @@ static const char * const cmd_filesystem_defrag_usage[] = {
 	OPTLINE("-l len", "defragment only up to len bytes"),
 	OPTLINE("-t size", "target extent size hint (default: 32M)"),
 	OPTLINE("--step SIZE", "process the range in given steps, flush after each one"),
+	OPTLINE("--usage-ratio RATIO", "defrag extents with utilization ratio lower than RATIO%"),
+	OPTLINE("--wasted-bytes SIZE", "defrag extents which can free up more bytes than SIZE"),
 	OPTLINE("-v", "deprecated, alias for global -v option"),
 	HELPINFO_INSERT_GLOBALS,
 	HELPINFO_INSERT_VERBOSE,
@@ -929,13 +931,32 @@ static struct btrfs_ioctl_defrag_range_args defrag_global_range;
 static int defrag_global_errors;
 static u64 defrag_global_step;
 
+static int defrag_ioctl(int fd, struct btrfs_ioctl_defrag_range_args *range)
+{
+	int ret = 0;
+
+	ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, range);
+	if (ret < 0 && ((errno == EOPNOTSUPP) | (errno == ENOTTY)) &&
+	    range->flags & (BTRFS_DEFRAG_RANGE_USAGE_RATIO |
+			    BTRFS_DEFRAG_RANGE_WASTED_BYTES)) {
+		/*
+		 * Older kernel, no ratio/wasted bytes fine-tunning support.
+		 * Retry with ratio/wasted bytes fine-tunning disabled.
+		 */
+		range->flags &= ~(BTRFS_DEFRAG_RANGE_USAGE_RATIO |
+				  BTRFS_DEFRAG_RANGE_WASTED_BYTES);
+		ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, range);
+	}
+	return ret;
+}
+
 static int defrag_range_in_steps(int fd, const struct stat *st) {
 	int ret = 0;
 	u64 end;
 	struct btrfs_ioctl_defrag_range_args range;
 
 	if (defrag_global_step == 0)
-		return ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, &defrag_global_range);
+		return defrag_ioctl(fd, &defrag_global_range);
 
 	/*
 	 * If start is set but length is not within or beyond the u64 range,
@@ -946,13 +967,17 @@ static int defrag_range_in_steps(int fd, const struct stat *st) {
 
 	range = defrag_global_range;
 	range.flags |= BTRFS_DEFRAG_RANGE_START_IO;
+	range.flags |= (BTRFS_DEFRAG_RANGE_USAGE_RATIO |
+			BTRFS_DEFRAG_RANGE_WASTED_BYTES);
+	range.usage_ratio = defrag_global_range.usage_ratio;
+	range.wasted_bytes = defrag_global_range.wasted_bytes;
 	while (range.start < end) {
 		u64 start;
 
 		range.len = defrag_global_step;
 		pr_verbose(LOG_VERBOSE, "defrag range step: start=%llu len=%llu step=%llu\n",
 			   range.start, range.len, defrag_global_step);
-		ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, &range);
+		ret = defrag_ioctl(fd, &range);
 		if (ret < 0)
 			return ret;
 		if (check_add_overflow(range.start, defrag_global_step, &start))
@@ -1013,6 +1038,8 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 	bool recursive = false;
 	int ret = 0;
 	int compress_type = BTRFS_COMPRESS_NONE;
+	u32 usage_ratio = 5;
+	u32 wasted_bytes = SZ_16M;
 
 	/*
 	 * Kernel 4.19+ supports defragmention of files open read-only,
@@ -1046,9 +1073,14 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 	defrag_global_errors = 0;
 	optind = 0;
 	while(1) {
-		enum { GETOPT_VAL_STEP = GETOPT_VAL_FIRST };
+		enum { GETOPT_VAL_STEP = GETOPT_VAL_FIRST,
+		       GETOPT_VAL_USAGE_RATIO, GETOPT_VAL_WASTED_BYTES };
 		static const struct option long_options[] = {
 			{ "step", required_argument, NULL, GETOPT_VAL_STEP },
+			{ "usage-ratio", required_argument, NULL,
+				GETOPT_VAL_USAGE_RATIO },
+			{ "wasted-bytes", required_argument, NULL,
+				GETOPT_VAL_WASTED_BYTES },
 			{ NULL, 0, NULL, 0 }
 		};
 		int c;
@@ -1098,6 +1130,17 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 				defrag_global_step = SZ_256K;
 			}
 			break;
+		case GETOPT_VAL_USAGE_RATIO:
+			usage_ratio = arg_strtou64(optarg);
+			if (usage_ratio > 100) {
+				error("invalid ratio, has %u expect [0, 100]",
+					usage_ratio);
+				exit(1);
+			}
+			break;
+		case GETOPT_VAL_WASTED_BYTES:
+			wasted_bytes = arg_strtou64_with_suffix(optarg);
+			break;
 		default:
 			usage_unknown_option(cmd, argv);
 		}
@@ -1116,6 +1159,10 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 	}
 	if (flush)
 		defrag_global_range.flags |= BTRFS_DEFRAG_RANGE_START_IO;
+	defrag_global_range.usage_ratio = usage_ratio;
+	defrag_global_range.wasted_bytes = wasted_bytes;
+	defrag_global_range.flags |= (BTRFS_DEFRAG_RANGE_USAGE_RATIO |
+				      BTRFS_DEFRAG_RANGE_WASTED_BYTES);
 
 	/*
 	 * Look for directory arguments and warn if the recursive mode is not
-- 
2.44.0