[PATCH 1/4] Btrfs-progs: send, bump stream version

[PATCH 1/4] Btrfs-progs: send, bump stream version

[Filipe David Borba Manana]

This increases the send stream version from version 1 to version 2, adding
2 new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files.

This is preparation work for subsequent changes that implement the new features
(computing total data size and use fallocate for better performance).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---
 ioctl.h |  8 ++++++++
 send.h  | 13 ++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/ioctl.h b/ioctl.h
index 231660a..901a248 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -392,6 +392,14 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * The sum of all length fields the receiver will get in write, clone and
+ * fallocate commands.
+ * This can be used by the receiver to compute progress, at the expense of some
+ * initial metadata scan performed by the sender (kernel).
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */
 	__u64 clone_sources_count;	/* in */
diff --git a/send.h b/send.h
index e8da785..69e81fb 100644
--- a/send.h
+++ b/send.h
@@ -24,7 +24,7 @@ extern "C" {
 #endif
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION 2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)
@@ -91,6 +91,11 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/* added in stream version 2 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -129,10 +134,16 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/* added in stream version 2 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS,
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif
-- 
1.9.1

[PATCH 2/4] Btrfs-progs: send, implement total data size callback and progress report

[Filipe David Borba Manana]

This is a followup to the kernel patch titled:

    Btrfs: send, implement total data size command to allow for progress estimation

This makes the btrfs send and receive commands aware of the new send flag,
named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data
that is new between the parent and send snapshots/roots. As this command
immediately follows the commands to start a snapshot/subvolume, it can be
used to report and compute progress, by keeping a counter that is incremented
with the data length of each write, clone and fallocate command that is received
from the stream.

Example:

    $ btrfs send -o /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s

    $ btrfs send -o -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible and simple
to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send
output to btrfs-receive (directly to a file).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---
 Documentation/btrfs-send.txt |  3 ++
 cmds-receive.c               | 91 ++++++++++++++++++++++++++++++++++++++++++++
 cmds-send.c                  | 14 ++++++-
 send-stream.c                |  4 ++
 send-stream.h                |  1 +
 5 files changed, 111 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index 18a98fa..38470b0 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed).
 -f <outfile>::
 Output is normally written to stdout. To write to a file, use this option.
 An alternative would be to use pipes.
+-o::
+Obtain the total data size for each subvolume or snapshot to send. This demands additional
+processing (mostly IO bound) but is useful for the receive command to report progress.
 
 EXIT STATUS
 -----------
diff --git a/cmds-receive.c b/cmds-receive.c
index d6cd3da..19300fc 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -32,6 +32,7 @@
 #include <ftw.h>
 #include <wait.h>
 #include <assert.h>
+#include <time.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -71,6 +72,14 @@ struct btrfs_receive
 	struct subvol_uuid_search sus;
 
 	int honor_end_cmd;
+
+	/* For the subvolume/snapshot we're currently receiving. */
+	u64 total_data_size;
+	u64 bytes_received;
+	time_t last_progress_update;
+	u64 bytes_received_last_update;
+	float progress;
+	const char *target;
 };
 
 static int finish_subvol(struct btrfs_receive *r)
@@ -156,6 +165,12 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	r->total_data_size = 0;
+	r->bytes_received = 0;
+	r->progress = 0.0;
+	r->last_progress_update = 0;
+	r->bytes_received_last_update = 0;
+	r->target = "Subvolume";
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -205,6 +220,12 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	r->total_data_size = 0;
+	r->bytes_received = 0;
+	r->progress = 0.0;
+	r->last_progress_update = 0;
+	r->bytes_received_last_update = 0;
+	r->target = "Snapshot";
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -287,6 +308,73 @@ out:
 	return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+	struct btrfs_receive *r = user;
+
+	r->total_data_size = size;
+	fprintf(stdout, "About to receive %llu bytes\n", size);
+
+	return 0;
+}
+
+static void update_progress(struct btrfs_receive *r, u64 bytes)
+{
+	float new_progress;
+	time_t now;
+	time_t tdiff;
+
+	if (r->total_data_size == 0)
+		return;
+
+	r->bytes_received += bytes;
+
+	now = time(NULL);
+	tdiff = now - r->last_progress_update;
+	if (tdiff < 1) {
+		if (r->bytes_received == r->total_data_size)
+			fprintf(stdout, "\n");
+		return;
+	}
+
+	new_progress = ((float)r->bytes_received / r->total_data_size) * 100.0;
+
+	if ((int)(new_progress * 100) > (int)(r->progress * 100) ||
+	    r->bytes_received == r->total_data_size) {
+		char line[512];
+		float rate = r->bytes_received - r->bytes_received_last_update;
+		const char *rate_units;
+
+		rate /= tdiff;
+		if (rate > (1024 * 1024)) {
+			rate_units = "MB/s";
+			rate /= 1024 * 1024;
+		} else if (rate > 1024) {
+			rate_units = "KB/s";
+			rate /= 1024;
+		} else {
+			rate_units = "B/s";
+		}
+
+		snprintf(line, sizeof(line),
+			 "%s%s %s, %llu / %llu bytes received, %5.2f%%, %5.2f%s%s",
+			 (g_verbose ? "" : "\r"),
+			 r->target,
+			 r->full_subvol_path,
+			 r->bytes_received, r->total_data_size,
+			 new_progress, rate, rate_units,
+			 (g_verbose ? "\n" : ""));
+		fprintf(stdout, "%s%s", line, "      ");
+		fflush(stdout);
+	}
+
+	if (r->bytes_received == r->total_data_size)
+		fprintf(stdout, "\n");
+	r->progress = new_progress;
+	r->last_progress_update = now;
+	r->bytes_received_last_update = r->bytes_received;
+}
+
 static int process_mkfile(const char *path, void *user)
 {
 	int ret;
@@ -562,6 +650,7 @@ static int process_write(const char *path, const void *data, u64 offset,
 		}
 		pos += w;
 	}
+	update_progress(r, len);
 
 out:
 	free(full_path);
@@ -638,6 +727,7 @@ static int process_clone(const char *path, u64 offset, u64 len,
 				path, strerror(-ret));
 		goto out;
 	}
+	update_progress(r, len);
 
 out:
 	if (si) {
@@ -819,6 +909,7 @@ static struct btrfs_send_ops send_ops = {
 	.chmod = process_chmod,
 	.chown = process_chown,
 	.utimes = process_utimes,
+	.total_data_size = process_total_data_size,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/cmds-send.c b/cmds-send.c
index 1cd457d..3ebbfb9 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -45,6 +45,7 @@
 #include "send-utils.h"
 
 static int g_verbose = 0;
+static int get_total_data_size = 0;
 
 struct btrfs_send {
 	int send_fd;
@@ -281,6 +282,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
 	if (!is_last_subvol)
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+	if (get_total_data_size)
+		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -424,7 +427,7 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt(argc, argv, "vec:f:i:p:")) != -1) {
+	while ((c = getopt(argc, argv, "veoc:f:i:p:")) != -1) {
 		switch (c) {
 		case 'v':
 			g_verbose++;
@@ -511,6 +514,9 @@ int cmd_send(int argc, char **argv)
 				"ERROR: -i was removed, use -c instead\n");
 			ret = 1;
 			goto out;
+		case 'o':
+			get_total_data_size = 1;
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -673,7 +679,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {
 	"-f <outfile>     Output is normally written to stdout. To write to",
 	"                 a file, use this option. An alternative would be to",
 	"                 use pipes.",
+	"-o               Obtain the total data size for each subvolume or ",
+	"                 snapshot to send. This demands additional processing",
+	"                 (mostly IO bound) but is useful for the receive ",
+	"                 command to report progress.",
 	NULL
 };
diff --git a/send-stream.c b/send-stream.c
index 88e18e2..474d012 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->update_extent(path, offset, tmp, s->user);
 		break;
+	case BTRFS_SEND_C_TOTAL_DATA_SIZE:
+		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
+		ret = s->ops->total_data_size(tmp, s->user);
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 17bc669..3a653a9 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -54,6 +54,7 @@ struct btrfs_send_ops {
 		      struct timespec *mt, struct timespec *ct,
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
+	int (*total_data_size)(u64 size, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

[PATCH 3/4] Btrfs-progs: send, implement fallocate command callback

[Filipe David Borba Manana]

The fallocate send stream command, added in stream version 2, is used to
pre-allocate space for files and punch file holes. This change implements
the callback for that new command, using the fallocate function from the
standard C library to carry out the specified action (allocate file space
or punch a file hole).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---
 cmds-receive.c | 38 ++++++++++++++++++++++++++++++++++++++
 send-stream.c  | 13 +++++++++++++
 send-stream.h  |  2 ++
 3 files changed, 53 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index 19300fc..3f30066 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -41,6 +41,7 @@
 #include <sys/types.h>
 #include <sys/xattr.h>
 #include <uuid/uuid.h>
+#include <linux/falloc.h>
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -887,6 +888,42 @@ out:
 	return ret;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+			     u64 len, void *user)
+{
+	struct btrfs_receive *r = user;
+	char *full_path = path_cat(r->full_subvol_path, path);
+	int mode = 0;
+	int ret;
+
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+		mode |= FALLOC_FL_KEEP_SIZE;
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+		mode |= FALLOC_FL_PUNCH_HOLE;
+
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"fallocate %s - flags %u, offset %llu, len %llu\n",
+			path, flags, offset, len);
+
+	ret = open_inode_for_write(r, full_path);
+	if (ret < 0)
+		goto out;
+
+	ret = fallocate(r->write_fd, mode, offset, len);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr,
+			"ERROR: fallocate against %s failed. %s\n",
+			path, strerror(-ret));
+		goto out;
+	}
+	update_progress(r, len);
+
+out:
+	free(full_path);
+	return ret;
+}
 
 static struct btrfs_send_ops send_ops = {
 	.subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = {
 	.chown = process_chown,
 	.utimes = process_utimes,
 	.total_data_size = process_total_data_size,
+	.fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/send-stream.c b/send-stream.c
index 474d012..f117533 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->total_data_size(tmp, s->user);
 		break;
+	case BTRFS_SEND_C_FALLOCATE:
+		{
+			u32 flags;
+			u64 len;
+
+			TLV_GET_STRING(s, BTRFS_SEND_A_PATH, &path);
+			TLV_GET_U32(s, BTRFS_SEND_A_FALLOCATE_FLAGS, &flags);
+			TLV_GET_U64(s, BTRFS_SEND_A_FILE_OFFSET, &offset);
+			TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &len);
+			ret = s->ops->fallocate(path, flags, offset, len,
+						s->user);
+		}
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 3a653a9..479e40c 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -55,6 +55,8 @@ struct btrfs_send_ops {
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
 	int (*total_data_size)(u64 size, void *user);
+	int (*fallocate)(const char *path, u32 flags, u64 offset,
+			 u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

[PATCH 4/4] Btrfs-progs: add write and clone commands debug info to receive

[Filipe David Borba Manana]

When specifying -vv print information about received write and clone commands too,
as we do this for other commands already and it's very useful for debugging and
troubleshooting.

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---
 cmds-receive.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index 3f30066..7a23823 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -636,6 +636,10 @@ static int process_write(const char *path, const void *data, u64 offset,
 	u64 pos = 0;
 	int w;
 
+	if (g_verbose >= 2)
+		fprintf(stderr, "write %s, offset %llu, len %llu\n",
+			path, offset, len);
+
 	ret = open_inode_for_write(r, full_path);
 	if (ret < 0)
 		goto out;
@@ -672,6 +676,11 @@ static int process_clone(const char *path, u64 offset, u64 len,
 	char *full_clone_path = NULL;
 	int clone_fd = -1;
 
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"clone %s, offset %llu, len %llu, clone path %s, clone offset %llu\n",
+			path, offset, len, clone_path, clone_offset);
+
 	ret = open_inode_for_write(r, full_path);
 	if (ret < 0)
 		goto out;
-- 
1.9.1

[PATCH 1/4 v2] Btrfs-progs: send, bump stream version

[Filipe David Borba Manana]

This increases the send stream version from version 1 to version 2, adding
2 new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files.

This is preparation work for subsequent changes that implement the new features
(computing total data size and use fallocate for better performance).

This doesn't break compatibility with older kernels or clients. In order to get
a version 2 send stream, new flags must be passed to the send ioctl.

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
    stream is now only produced is the ioctl caller specifies at least one of
    the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
    BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).

 ioctl.h | 18 ++++++++++++++++++
 send.h  | 13 ++++++++++++-
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/ioctl.h b/ioctl.h
index 231660a..e2c506b 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -392,6 +392,24 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * The sum of all length fields the receiver will get in write, clone and
+ * fallocate commands.
+ * This can be used by the receiver to compute progress, at the expense of some
+ * initial metadata scan performed by the sender (kernel).
+ *
+ * Added in send stream version 2.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
+/*
+ * Use fallocate command to pre-allocate file extents and punch file holes,
+ * instead of write commands with data buffers filled with 0 value bytes.
+ *
+ * Added in send stream version 2.
+ */
+#define BTRFS_SEND_FLAG_SUPPORT_FALLOCATE       0x10
+
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */
 	__u64 clone_sources_count;	/* in */
diff --git a/send.h b/send.h
index e8da785..69e81fb 100644
--- a/send.h
+++ b/send.h
@@ -24,7 +24,7 @@ extern "C" {
 #endif
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION 2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)
@@ -91,6 +91,11 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/* added in stream version 2 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -129,10 +134,16 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/* added in stream version 2 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS,
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif
-- 
1.9.1

[PATCH 2/4 v2] Btrfs-progs: send, implement total data size callback and progress report

[Filipe David Borba Manana]

This is a followup to the kernel patch titled:

    Btrfs: send, implement total data size command to allow for progress estimation

This makes the btrfs send and receive commands aware of the new send flag,
named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data
that is new between the parent and send snapshots/roots. As this command
immediately follows the commands to start a snapshot/subvolume, it can be
used to report and compute progress, by keeping a counter that is incremented
with the data length of each write, clone and fallocate command that is received
from the stream.

Example:

    $ btrfs send -o /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s

    $ btrfs send -o -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible and simple
to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send
output to btrfs-receive (directly to a file).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
    stream is now only produced is the ioctl caller specifies at least one of
    the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
    BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).

 Documentation/btrfs-send.txt |  3 ++
 cmds-receive.c               | 91 ++++++++++++++++++++++++++++++++++++++++++++
 cmds-send.c                  | 14 ++++++-
 send-stream.c                |  4 ++
 send-stream.h                |  1 +
 5 files changed, 111 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index 18a98fa..38470b0 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed).
 -f <outfile>::
 Output is normally written to stdout. To write to a file, use this option.
 An alternative would be to use pipes.
+-o::
+Obtain the total data size for each subvolume or snapshot to send. This demands additional
+processing (mostly IO bound) but is useful for the receive command to report progress.
 
 EXIT STATUS
 -----------
diff --git a/cmds-receive.c b/cmds-receive.c
index d6cd3da..19300fc 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -32,6 +32,7 @@
 #include <ftw.h>
 #include <wait.h>
 #include <assert.h>
+#include <time.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -71,6 +72,14 @@ struct btrfs_receive
 	struct subvol_uuid_search sus;
 
 	int honor_end_cmd;
+
+	/* For the subvolume/snapshot we're currently receiving. */
+	u64 total_data_size;
+	u64 bytes_received;
+	time_t last_progress_update;
+	u64 bytes_received_last_update;
+	float progress;
+	const char *target;
 };
 
 static int finish_subvol(struct btrfs_receive *r)
@@ -156,6 +165,12 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	r->total_data_size = 0;
+	r->bytes_received = 0;
+	r->progress = 0.0;
+	r->last_progress_update = 0;
+	r->bytes_received_last_update = 0;
+	r->target = "Subvolume";
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -205,6 +220,12 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	r->total_data_size = 0;
+	r->bytes_received = 0;
+	r->progress = 0.0;
+	r->last_progress_update = 0;
+	r->bytes_received_last_update = 0;
+	r->target = "Snapshot";
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -287,6 +308,73 @@ out:
 	return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+	struct btrfs_receive *r = user;
+
+	r->total_data_size = size;
+	fprintf(stdout, "About to receive %llu bytes\n", size);
+
+	return 0;
+}
+
+static void update_progress(struct btrfs_receive *r, u64 bytes)
+{
+	float new_progress;
+	time_t now;
+	time_t tdiff;
+
+	if (r->total_data_size == 0)
+		return;
+
+	r->bytes_received += bytes;
+
+	now = time(NULL);
+	tdiff = now - r->last_progress_update;
+	if (tdiff < 1) {
+		if (r->bytes_received == r->total_data_size)
+			fprintf(stdout, "\n");
+		return;
+	}
+
+	new_progress = ((float)r->bytes_received / r->total_data_size) * 100.0;
+
+	if ((int)(new_progress * 100) > (int)(r->progress * 100) ||
+	    r->bytes_received == r->total_data_size) {
+		char line[512];
+		float rate = r->bytes_received - r->bytes_received_last_update;
+		const char *rate_units;
+
+		rate /= tdiff;
+		if (rate > (1024 * 1024)) {
+			rate_units = "MB/s";
+			rate /= 1024 * 1024;
+		} else if (rate > 1024) {
+			rate_units = "KB/s";
+			rate /= 1024;
+		} else {
+			rate_units = "B/s";
+		}
+
+		snprintf(line, sizeof(line),
+			 "%s%s %s, %llu / %llu bytes received, %5.2f%%, %5.2f%s%s",
+			 (g_verbose ? "" : "\r"),
+			 r->target,
+			 r->full_subvol_path,
+			 r->bytes_received, r->total_data_size,
+			 new_progress, rate, rate_units,
+			 (g_verbose ? "\n" : ""));
+		fprintf(stdout, "%s%s", line, "      ");
+		fflush(stdout);
+	}
+
+	if (r->bytes_received == r->total_data_size)
+		fprintf(stdout, "\n");
+	r->progress = new_progress;
+	r->last_progress_update = now;
+	r->bytes_received_last_update = r->bytes_received;
+}
+
 static int process_mkfile(const char *path, void *user)
 {
 	int ret;
@@ -562,6 +650,7 @@ static int process_write(const char *path, const void *data, u64 offset,
 		}
 		pos += w;
 	}
+	update_progress(r, len);
 
 out:
 	free(full_path);
@@ -638,6 +727,7 @@ static int process_clone(const char *path, u64 offset, u64 len,
 				path, strerror(-ret));
 		goto out;
 	}
+	update_progress(r, len);
 
 out:
 	if (si) {
@@ -819,6 +909,7 @@ static struct btrfs_send_ops send_ops = {
 	.chmod = process_chmod,
 	.chown = process_chown,
 	.utimes = process_utimes,
+	.total_data_size = process_total_data_size,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/cmds-send.c b/cmds-send.c
index 1cd457d..69f5ba1 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -45,6 +45,7 @@
 #include "send-utils.h"
 
 static int g_verbose = 0;
+static int g_total_data_size = 0;
 
 struct btrfs_send {
 	int send_fd;
@@ -281,6 +282,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
 	if (!is_last_subvol)
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+	if (g_total_data_size)
+		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -424,7 +427,7 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt(argc, argv, "vec:f:i:p:")) != -1) {
+	while ((c = getopt(argc, argv, "veoc:f:i:p:")) != -1) {
 		switch (c) {
 		case 'v':
 			g_verbose++;
@@ -511,6 +514,9 @@ int cmd_send(int argc, char **argv)
 				"ERROR: -i was removed, use -c instead\n");
 			ret = 1;
 			goto out;
+		case 'o':
+			g_total_data_size = 1;
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -673,7 +679,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {
 	"-f <outfile>     Output is normally written to stdout. To write to",
 	"                 a file, use this option. An alternative would be to",
 	"                 use pipes.",
+	"-o               Obtain the total data size for each subvolume or ",
+	"                 snapshot to send. This demands additional processing",
+	"                 (mostly IO bound) but is useful for the receive ",
+	"                 command to report progress.",
 	NULL
 };
diff --git a/send-stream.c b/send-stream.c
index 88e18e2..474d012 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->update_extent(path, offset, tmp, s->user);
 		break;
+	case BTRFS_SEND_C_TOTAL_DATA_SIZE:
+		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
+		ret = s->ops->total_data_size(tmp, s->user);
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 17bc669..3a653a9 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -54,6 +54,7 @@ struct btrfs_send_ops {
 		      struct timespec *mt, struct timespec *ct,
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
+	int (*total_data_size)(u64 size, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

[PATCH 3/4 v2] Btrfs-progs: send, implement fallocate command callback

[Filipe David Borba Manana]

The fallocate send stream command, added in stream version 2, is used to
pre-allocate space for files and punch file holes. This change implements
the callback for that new command, using the fallocate function from the
standard C library to carry out the specified action (allocate file space
or punch a file hole).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Use the new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE if the user
    asks for it (-a command line option), which will make the kernel generate
    a version 2 send stream, so that old clients aren't affected.

 Documentation/btrfs-send.txt |  3 +++
 cmds-receive.c               | 38 ++++++++++++++++++++++++++++++++++++++
 cmds-send.c                  | 12 ++++++++++--
 send-stream.c                | 13 +++++++++++++
 send-stream.h                |  2 ++
 5 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index 38470b0..e96be07 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -43,6 +43,9 @@ An alternative would be to use pipes.
 -o::
 Obtain the total data size for each subvolume or snapshot to send. This demands additional
 processing (mostly IO bound) but is useful for the receive command to report progress.
+-a::
+Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes
+to files.
 
 EXIT STATUS
 -----------
diff --git a/cmds-receive.c b/cmds-receive.c
index 19300fc..3f30066 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -41,6 +41,7 @@
 #include <sys/types.h>
 #include <sys/xattr.h>
 #include <uuid/uuid.h>
+#include <linux/falloc.h>
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -887,6 +888,42 @@ out:
 	return ret;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+			     u64 len, void *user)
+{
+	struct btrfs_receive *r = user;
+	char *full_path = path_cat(r->full_subvol_path, path);
+	int mode = 0;
+	int ret;
+
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+		mode |= FALLOC_FL_KEEP_SIZE;
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+		mode |= FALLOC_FL_PUNCH_HOLE;
+
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"fallocate %s - flags %u, offset %llu, len %llu\n",
+			path, flags, offset, len);
+
+	ret = open_inode_for_write(r, full_path);
+	if (ret < 0)
+		goto out;
+
+	ret = fallocate(r->write_fd, mode, offset, len);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr,
+			"ERROR: fallocate against %s failed. %s\n",
+			path, strerror(-ret));
+		goto out;
+	}
+	update_progress(r, len);
+
+out:
+	free(full_path);
+	return ret;
+}
 
 static struct btrfs_send_ops send_ops = {
 	.subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = {
 	.chown = process_chown,
 	.utimes = process_utimes,
 	.total_data_size = process_total_data_size,
+	.fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/cmds-send.c b/cmds-send.c
index 69f5ba1..2a62e68 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -46,6 +46,7 @@
 
 static int g_verbose = 0;
 static int g_total_data_size = 0;
+static int g_fallocate = 0;
 
 struct btrfs_send {
 	int send_fd;
@@ -284,6 +285,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
 	if (g_total_data_size)
 		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
+	if (g_fallocate)
+		io_send.flags |= BTRFS_SEND_FLAG_SUPPORT_FALLOCATE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -427,7 +430,7 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt(argc, argv, "veoc:f:i:p:")) != -1) {
+	while ((c = getopt(argc, argv, "veoac:f:i:p:")) != -1) {
 		switch (c) {
 		case 'v':
 			g_verbose++;
@@ -517,6 +520,9 @@ int cmd_send(int argc, char **argv)
 		case 'o':
 			g_total_data_size = 1;
 			break;
+		case 'a':
+			g_fallocate = 1;
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -679,7 +685,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-veoa] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -707,5 +713,7 @@ const char * const cmd_send_usage[] = {
 	"                 snapshot to send. This demands additional processing",
 	"                 (mostly IO bound) but is useful for the receive ",
 	"                 command to report progress.",
+	"-a               Use fallocate to pre-allocate file extents and to",
+	"                 punch file holes, instead of writing zeroes to files.",
 	NULL
 };
diff --git a/send-stream.c b/send-stream.c
index 474d012..f117533 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->total_data_size(tmp, s->user);
 		break;
+	case BTRFS_SEND_C_FALLOCATE:
+		{
+			u32 flags;
+			u64 len;
+
+			TLV_GET_STRING(s, BTRFS_SEND_A_PATH, &path);
+			TLV_GET_U32(s, BTRFS_SEND_A_FALLOCATE_FLAGS, &flags);
+			TLV_GET_U64(s, BTRFS_SEND_A_FILE_OFFSET, &offset);
+			TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &len);
+			ret = s->ops->fallocate(path, flags, offset, len,
+						s->user);
+		}
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 3a653a9..479e40c 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -55,6 +55,8 @@ struct btrfs_send_ops {
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
 	int (*total_data_size)(u64 size, void *user);
+	int (*fallocate)(const char *path, u32 flags, u64 offset,
+			 u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

[PATCH 4/4 v2] Btrfs-progs: add write and clone commands debug info to receive

[Filipe David Borba Manana]

When specifying -vv print information about received write and clone commands too,
as we do this for other commands already and it's very useful for debugging and
troubleshooting.

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
    stream is now only produced is the ioctl caller specifies at least one of
    the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
    BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).

 cmds-receive.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index 3f30066..7a23823 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -636,6 +636,10 @@ static int process_write(const char *path, const void *data, u64 offset,
 	u64 pos = 0;
 	int w;
 
+	if (g_verbose >= 2)
+		fprintf(stderr, "write %s, offset %llu, len %llu\n",
+			path, offset, len);
+
 	ret = open_inode_for_write(r, full_path);
 	if (ret < 0)
 		goto out;
@@ -672,6 +676,11 @@ static int process_clone(const char *path, u64 offset, u64 len,
 	char *full_clone_path = NULL;
 	int clone_fd = -1;
 
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"clone %s, offset %llu, len %llu, clone path %s, clone offset %llu\n",
+			path, offset, len, clone_path, clone_offset);
+
 	ret = open_inode_for_write(r, full_path);
 	if (ret < 0)
 		goto out;
-- 
1.9.1

Re: [PATCH 2/4 v2] Btrfs-progs: send, implement total data size callback and progress report

[David Sterba]

On Wed, Apr 16, 2014 at 03:56:15PM +0100, Filipe David Borba Manana wrote:
> V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
>     stream is now only produced is the ioctl caller specifies at least one of
>     the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
>     BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).

Good.

> @@ -156,6 +165,12 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
>  		goto out;
>  
>  	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
> +	r->total_data_size = 0;
> +	r->bytes_received = 0;
> +	r->progress = 0.0;
> +	r->last_progress_update = 0;
> +	r->bytes_received_last_update = 0;
> +	r->target = "Subvolume";
>  
>  	if (strlen(r->dest_dir_path) == 0)
>  		r->cur_subvol->path = strdup(path);
> @@ -205,6 +220,12 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
>  		goto out;
>  
>  	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
> +	r->total_data_size = 0;
> +	r->bytes_received = 0;
> +	r->progress = 0.0;
> +	r->last_progress_update = 0;
> +	r->bytes_received_last_update = 0;
> +	r->target = "Snapshot";

Nontrivial amount of duplicate code, a helper would be better.

> @@ -673,7 +679,7 @@ out:
>  }
>  
>  const char * const cmd_send_usage[] = {
> -	"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
> +	"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
>  	"Send the subvolume(s) to stdout.",
>  	"Sends the subvolume(s) specified by <subvol> to stdout.",
>  	"By default, this will send the whole subvolume. To do an incremental",
> @@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {
>  	"-f <outfile>     Output is normally written to stdout. To write to",
>  	"                 a file, use this option. An alternative would be to",
>  	"                 use pipes.",
> +	"-o               Obtain the total data size for each subvolume or ",

UI:

-o is sometimes used as an option for 'output', which makes sense in
context of send, but is already done via -f. I'm not sure if it's a good
choice. 'p' is already occupied.

As an alternative: how about -s ? mnemonic for 'size'.

> +	"                 snapshot to send. This demands additional processing",
> +	"                 (mostly IO bound) but is useful for the receive ",
> +	"                 command to report progress.",
>  	NULL
>  };

Re: [PATCH 2/4 v2] Btrfs-progs: send, implement total data size callback and progress report

[Filipe David Manana]

On Wed, Apr 16, 2014 at 6:43 PM, David Sterba <dsterba@suse.cz> wrote:
> On Wed, Apr 16, 2014 at 03:56:15PM +0100, Filipe David Borba Manana wrote:
>> V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
>>     stream is now only produced is the ioctl caller specifies at least one of
>>     the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
>>     BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
>
> Good.
>
>> @@ -156,6 +165,12 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
>>               goto out;
>>
>>       r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
>> +     r->total_data_size = 0;
>> +     r->bytes_received = 0;
>> +     r->progress = 0.0;
>> +     r->last_progress_update = 0;
>> +     r->bytes_received_last_update = 0;
>> +     r->target = "Subvolume";
>>
>>       if (strlen(r->dest_dir_path) == 0)
>>               r->cur_subvol->path = strdup(path);
>> @@ -205,6 +220,12 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
>>               goto out;
>>
>>       r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
>> +     r->total_data_size = 0;
>> +     r->bytes_received = 0;
>> +     r->progress = 0.0;
>> +     r->last_progress_update = 0;
>> +     r->bytes_received_last_update = 0;
>> +     r->target = "Snapshot";
>
> Nontrivial amount of duplicate code, a helper would be better.

Agree about the duplication, but can't agree with being non-trivial:
resetting to 0 a few counters/percentage/timestamp :)

>
>> @@ -673,7 +679,7 @@ out:
>>  }
>>
>>  const char * const cmd_send_usage[] = {
>> -     "btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
>> +     "btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
>>       "Send the subvolume(s) to stdout.",
>>       "Sends the subvolume(s) specified by <subvol> to stdout.",
>>       "By default, this will send the whole subvolume. To do an incremental",
>> @@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {
>>       "-f <outfile>     Output is normally written to stdout. To write to",
>>       "                 a file, use this option. An alternative would be to",
>>       "                 use pipes.",
>> +     "-o               Obtain the total data size for each subvolume or ",
>
> UI:
>
> -o is sometimes used as an option for 'output', which makes sense in
> context of send, but is already done via -f. I'm not sure if it's a good
> choice. 'p' is already occupied.
>
> As an alternative: how about -s ? mnemonic for 'size'.

Fine for me. Don't have a strong opinion about the letter.

thanks David

>
>> +     "                 snapshot to send. This demands additional processing",
>> +     "                 (mostly IO bound) but is useful for the receive ",
>> +     "                 command to report progress.",
>>       NULL
>>  };



-- 
Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

[PATCH 2/4 v3] Btrfs-progs: send, implement total data size callback and progress report

[Filipe David Borba Manana]

This is a followup to the kernel patch titled:

    Btrfs: send, implement total data size command to allow for progress estimation

This makes the btrfs send and receive commands aware of the new send flag,
named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data
that is new between the parent and send snapshots/roots. As this command
immediately follows the commands to start a snapshot/subvolume, it can be
used to report and compute progress, by keeping a counter that is incremented
with the data length of each write, clone and fallocate command that is received
from the stream.

Example:

    $ btrfs send -s /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s

    $ btrfs send -s -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible and simple
to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send
output to btrfs-receive (directly to a file).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
    stream is now only produced is the ioctl caller specifies at least one of
    the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
    BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Renamed option -o to -s, removed some duplicated code (progress reset).

 Documentation/btrfs-send.txt |  3 ++
 cmds-receive.c               | 91 ++++++++++++++++++++++++++++++++++++++++++++
 cmds-send.c                  | 14 ++++++-
 send-stream.c                |  4 ++
 send-stream.h                |  1 +
 5 files changed, 111 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index 18a98fa..a37d63c 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed).
 -f <outfile>::
 Output is normally written to stdout. To write to a file, use this option.
 An alternative would be to use pipes.
+-s::
+Obtain the total data size for each subvolume or snapshot to send. This demands additional
+processing (mostly IO bound) but is useful for the receive command to report progress.
 
 EXIT STATUS
 -----------
diff --git a/cmds-receive.c b/cmds-receive.c
index d6cd3da..4cbf276 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -32,6 +32,7 @@
 #include <ftw.h>
 #include <wait.h>
 #include <assert.h>
+#include <time.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -71,6 +72,14 @@ struct btrfs_receive
 	struct subvol_uuid_search sus;
 
 	int honor_end_cmd;
+
+	/* For the subvolume/snapshot we're currently receiving. */
+	u64 total_data_size;
+	u64 bytes_received;
+	time_t last_progress_update;
+	u64 bytes_received_last_update;
+	float progress;
+	const char *target;
 };
 
 static int finish_subvol(struct btrfs_receive *r)
@@ -143,6 +152,16 @@ out:
 	return ret;
 }
 
+static void reset_progress(struct btrfs_receive *r, const char *dest)
+{
+	r->total_data_size = 0;
+	r->bytes_received = 0;
+	r->progress = 0.0;
+	r->last_progress_update = 0;
+	r->bytes_received_last_update = 0;
+	r->target = dest;
+}
+
 static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 			  void *user)
 {
@@ -156,6 +175,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	reset_progress(r, "Subvolume");
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -205,6 +225,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	reset_progress(r, "Snapshot");
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -287,6 +308,73 @@ out:
 	return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+	struct btrfs_receive *r = user;
+
+	r->total_data_size = size;
+	fprintf(stdout, "About to receive %llu bytes\n", size);
+
+	return 0;
+}
+
+static void update_progress(struct btrfs_receive *r, u64 bytes)
+{
+	float new_progress;
+	time_t now;
+	time_t tdiff;
+
+	if (r->total_data_size == 0)
+		return;
+
+	r->bytes_received += bytes;
+
+	now = time(NULL);
+	tdiff = now - r->last_progress_update;
+	if (tdiff < 1) {
+		if (r->bytes_received == r->total_data_size)
+			fprintf(stdout, "\n");
+		return;
+	}
+
+	new_progress = ((float)r->bytes_received / r->total_data_size) * 100.0;
+
+	if ((int)(new_progress * 100) > (int)(r->progress * 100) ||
+	    r->bytes_received == r->total_data_size) {
+		char line[512];
+		float rate = r->bytes_received - r->bytes_received_last_update;
+		const char *rate_units;
+
+		rate /= tdiff;
+		if (rate > (1024 * 1024)) {
+			rate_units = "MB/s";
+			rate /= 1024 * 1024;
+		} else if (rate > 1024) {
+			rate_units = "KB/s";
+			rate /= 1024;
+		} else {
+			rate_units = "B/s";
+		}
+
+		snprintf(line, sizeof(line),
+			 "%s%s %s, %llu / %llu bytes received, %5.2f%%, %5.2f%s%s",
+			 (g_verbose ? "" : "\r"),
+			 r->target,
+			 r->full_subvol_path,
+			 r->bytes_received, r->total_data_size,
+			 new_progress, rate, rate_units,
+			 (g_verbose ? "\n" : ""));
+		fprintf(stdout, "%s%s", line, "      ");
+		fflush(stdout);
+	}
+
+	if (r->bytes_received == r->total_data_size)
+		fprintf(stdout, "\n");
+	r->progress = new_progress;
+	r->last_progress_update = now;
+	r->bytes_received_last_update = r->bytes_received;
+}
+
 static int process_mkfile(const char *path, void *user)
 {
 	int ret;
@@ -562,6 +650,7 @@ static int process_write(const char *path, const void *data, u64 offset,
 		}
 		pos += w;
 	}
+	update_progress(r, len);
 
 out:
 	free(full_path);
@@ -638,6 +727,7 @@ static int process_clone(const char *path, u64 offset, u64 len,
 				path, strerror(-ret));
 		goto out;
 	}
+	update_progress(r, len);
 
 out:
 	if (si) {
@@ -819,6 +909,7 @@ static struct btrfs_send_ops send_ops = {
 	.chmod = process_chmod,
 	.chown = process_chown,
 	.utimes = process_utimes,
+	.total_data_size = process_total_data_size,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/cmds-send.c b/cmds-send.c
index 1cd457d..6f2d7c1 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -45,6 +45,7 @@
 #include "send-utils.h"
 
 static int g_verbose = 0;
+static int g_total_data_size = 0;
 
 struct btrfs_send {
 	int send_fd;
@@ -281,6 +282,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
 	if (!is_last_subvol)
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+	if (g_total_data_size)
+		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -424,7 +427,7 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt(argc, argv, "vec:f:i:p:")) != -1) {
+	while ((c = getopt(argc, argv, "vesc:f:i:p:")) != -1) {
 		switch (c) {
 		case 'v':
 			g_verbose++;
@@ -511,6 +514,9 @@ int cmd_send(int argc, char **argv)
 				"ERROR: -i was removed, use -c instead\n");
 			ret = 1;
 			goto out;
+		case 's':
+			g_total_data_size = 1;
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -673,7 +679,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-ves] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {
 	"-f <outfile>     Output is normally written to stdout. To write to",
 	"                 a file, use this option. An alternative would be to",
 	"                 use pipes.",
+	"-s               Obtain the total data size for each subvolume or ",
+	"                 snapshot to send. This demands additional processing",
+	"                 (mostly IO bound) but is useful for the receive ",
+	"                 command to report progress.",
 	NULL
 };
diff --git a/send-stream.c b/send-stream.c
index 88e18e2..474d012 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->update_extent(path, offset, tmp, s->user);
 		break;
+	case BTRFS_SEND_C_TOTAL_DATA_SIZE:
+		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
+		ret = s->ops->total_data_size(tmp, s->user);
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 17bc669..3a653a9 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -54,6 +54,7 @@ struct btrfs_send_ops {
 		      struct timespec *mt, struct timespec *ct,
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
+	int (*total_data_size)(u64 size, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

[PATCH 3/4 v3] Btrfs-progs: send, implement fallocate command callback

[Filipe David Borba Manana]

The fallocate send stream command, added in stream version 2, is used to
pre-allocate space for files and punch file holes. This change implements
the callback for that new command, using the fallocate function from the
standard C library to carry out the specified action (allocate file space
or punch a file hole).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Use the new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE if the user
    asks for it (-a command line option), which will make the kernel generate
    a version 2 send stream, so that old clients aren't affected.
V3: Rebased on new patchset (new version of patch 2/4).

 Documentation/btrfs-send.txt |  3 +++
 cmds-receive.c               | 38 ++++++++++++++++++++++++++++++++++++++
 cmds-send.c                  | 12 ++++++++++--
 send-stream.c                | 13 +++++++++++++
 send-stream.h                |  2 ++
 5 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index a37d63c..6893b90 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -43,6 +43,9 @@ An alternative would be to use pipes.
 -s::
 Obtain the total data size for each subvolume or snapshot to send. This demands additional
 processing (mostly IO bound) but is useful for the receive command to report progress.
+-a::
+Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes
+to files.
 
 EXIT STATUS
 -----------
diff --git a/cmds-receive.c b/cmds-receive.c
index 4cbf276..908f968 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -41,6 +41,7 @@
 #include <sys/types.h>
 #include <sys/xattr.h>
 #include <uuid/uuid.h>
+#include <linux/falloc.h>
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -887,6 +888,42 @@ out:
 	return ret;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+			     u64 len, void *user)
+{
+	struct btrfs_receive *r = user;
+	char *full_path = path_cat(r->full_subvol_path, path);
+	int mode = 0;
+	int ret;
+
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+		mode |= FALLOC_FL_KEEP_SIZE;
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+		mode |= FALLOC_FL_PUNCH_HOLE;
+
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"fallocate %s - flags %u, offset %llu, len %llu\n",
+			path, flags, offset, len);
+
+	ret = open_inode_for_write(r, full_path);
+	if (ret < 0)
+		goto out;
+
+	ret = fallocate(r->write_fd, mode, offset, len);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr,
+			"ERROR: fallocate against %s failed. %s\n",
+			path, strerror(-ret));
+		goto out;
+	}
+	update_progress(r, len);
+
+out:
+	free(full_path);
+	return ret;
+}
 
 static struct btrfs_send_ops send_ops = {
 	.subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = {
 	.chown = process_chown,
 	.utimes = process_utimes,
 	.total_data_size = process_total_data_size,
+	.fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/cmds-send.c b/cmds-send.c
index 6f2d7c1..7203956 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -46,6 +46,7 @@
 
 static int g_verbose = 0;
 static int g_total_data_size = 0;
+static int g_fallocate = 0;
 
 struct btrfs_send {
 	int send_fd;
@@ -284,6 +285,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
 	if (g_total_data_size)
 		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
+	if (g_fallocate)
+		io_send.flags |= BTRFS_SEND_FLAG_SUPPORT_FALLOCATE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -427,7 +430,7 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt(argc, argv, "vesc:f:i:p:")) != -1) {
+	while ((c = getopt(argc, argv, "vesac:f:i:p:")) != -1) {
 		switch (c) {
 		case 'v':
 			g_verbose++;
@@ -517,6 +520,9 @@ int cmd_send(int argc, char **argv)
 		case 's':
 			g_total_data_size = 1;
 			break;
+		case 'a':
+			g_fallocate = 1;
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -679,7 +685,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ves] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-vesa] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -707,5 +713,7 @@ const char * const cmd_send_usage[] = {
 	"                 snapshot to send. This demands additional processing",
 	"                 (mostly IO bound) but is useful for the receive ",
 	"                 command to report progress.",
+	"-a               Use fallocate to pre-allocate file extents and to",
+	"                 punch file holes, instead of writing zeroes to files.",
 	NULL
 };
diff --git a/send-stream.c b/send-stream.c
index 474d012..f117533 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->total_data_size(tmp, s->user);
 		break;
+	case BTRFS_SEND_C_FALLOCATE:
+		{
+			u32 flags;
+			u64 len;
+
+			TLV_GET_STRING(s, BTRFS_SEND_A_PATH, &path);
+			TLV_GET_U32(s, BTRFS_SEND_A_FALLOCATE_FLAGS, &flags);
+			TLV_GET_U64(s, BTRFS_SEND_A_FILE_OFFSET, &offset);
+			TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &len);
+			ret = s->ops->fallocate(path, flags, offset, len,
+						s->user);
+		}
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 3a653a9..479e40c 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -55,6 +55,8 @@ struct btrfs_send_ops {
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
 	int (*total_data_size)(u64 size, void *user);
+	int (*fallocate)(const char *path, u32 flags, u64 offset,
+			 u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

Re: [PATCH 3/4 v3] Btrfs-progs: send, implement fallocate command callback

[David Sterba]

On Wed, Apr 16, 2014 at 08:58:14PM +0100, Filipe David Borba Manana wrote:
> The fallocate send stream command, added in stream version 2, is used to
> pre-allocate space for files and punch file holes. This change implements
> the callback for that new command, using the fallocate function from the
> standard C library to carry out the specified action (allocate file space
> or punch a file hole).

> +-a::
> +Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes
> +to files.

(I wrote that in another mail and repeat it again in the context of the patch)

for v2 the fallocate mode should be default and IMHO not configurable by
a flag at all - falloc is efficient and the (right) thing that the devs
choose for the user.

I think that letting the user decide if he should use falloc or not is
too low-level. From the UI perspective, specifying a protocol number
should be enough.

There's currently no way to find the highest protocol version supported,
but the sysfs flags can be enhanced with that.

Re: [PATCH 3/4 v3] Btrfs-progs: send, implement fallocate command callback

[Filipe David Manana]

On Fri, Apr 18, 2014 at 6:41 PM, David Sterba <dsterba@suse.cz> wrote:
> On Wed, Apr 16, 2014 at 08:58:14PM +0100, Filipe David Borba Manana wrote:
>> The fallocate send stream command, added in stream version 2, is used to
>> pre-allocate space for files and punch file holes. This change implements
>> the callback for that new command, using the fallocate function from the
>> standard C library to carry out the specified action (allocate file space
>> or punch a file hole).
>
>> +-a::
>> +Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes
>> +to files.
>
> (I wrote that in another mail and repeat it again in the context of the patch)
>
> for v2 the fallocate mode should be default and IMHO not configurable by
> a flag at all - falloc is efficient and the (right) thing that the devs
> choose for the user.
>
> I think that letting the user decide if he should use falloc or not is
> too low-level. From the UI perspective, specifying a protocol number
> should be enough.

Fine for me.

>
> There's currently no way to find the highest protocol version supported,
> but the sysfs flags can be enhanced with that.

You mean, I guess, adding an entry named like
/sys/fs/btrfs/features/send_stream_version

Thanks David



-- 
Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

[PATCH 1/4 v3] Btrfs-progs: send, bump stream version

[Filipe David Borba Manana]

This increases the send stream version from version 1 to version 2, adding
new commands:

1) total data size - used to tell the receiver how much file data the stream
   will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags;

4) set inode otime.

This is preparation work for subsequent changes that implement the new features.

This doesn't break compatibility with older kernels or clients. In order to get
a version 2 send stream, new flags must be passed to the send ioctl.

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
    stream is now only produced is the ioctl caller specifies at least one of
    the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
    BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE flag and -a command line option
    for btrfs-send. Both were replaced with BTRFS_SEND_FLAG_STREAM_V2 and
    --stream-version=<version_number> respectively. Added commands for inode
    sets flags and otime too.

 Documentation/btrfs-send.txt |  3 +++
 cmds-send.c                  | 57 ++++++++++++++++++++++++++++++++++----------
 ioctl.h                      | 15 ++++++++++++
 send-stream.c                |  2 +-
 send.h                       | 23 +++++++++++++++++-
 5 files changed, 85 insertions(+), 15 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index 18a98fa..067fc27 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed).
 -f <outfile>::
 Output is normally written to stdout. To write to a file, use this option.
 An alternative would be to use pipes.
+--stream-version <version>::
+Ask the kernel to produce a specific send stream version. More recent stream versions provide
+new features and better performance. Default value is 1.
 
 EXIT STATUS
 -----------
diff --git a/cmds-send.c b/cmds-send.c
index 1cd457d..bd575f8 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -32,6 +32,7 @@
 #include <libgen.h>
 #include <mntent.h>
 #include <assert.h>
+#include <getopt.h>
 
 #include <uuid/uuid.h>
 
@@ -45,6 +46,7 @@
 #include "send-utils.h"
 
 static int g_verbose = 0;
+static int g_stream_version = BTRFS_SEND_STREAM_VERSION_1;
 
 struct btrfs_send {
 	int send_fd;
@@ -281,6 +283,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
 	if (!is_last_subvol)
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+	if (g_stream_version == BTRFS_SEND_STREAM_VERSION_2)
+		io_send.flags |= BTRFS_SEND_FLAG_STREAM_V2;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -406,6 +410,11 @@ out:
 	return ret;
 }
 
+static const struct option long_options[] = {
+	{ "stream-version", 1, NULL, 'V' },
+	{ NULL, 0, NULL, 0 }
+};
+
 int cmd_send(int argc, char **argv)
 {
 	char *subvol = NULL;
@@ -424,7 +433,8 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt(argc, argv, "vec:f:i:p:")) != -1) {
+	while ((c = getopt_long(argc, argv, "vec:f:i:p:",
+				long_options, NULL)) != -1) {
 		switch (c) {
 		case 'v':
 			g_verbose++;
@@ -511,6 +521,24 @@ int cmd_send(int argc, char **argv)
 				"ERROR: -i was removed, use -c instead\n");
 			ret = 1;
 			goto out;
+		case 'V':
+			if (sscanf(optarg, "%d", &g_stream_version) != 1) {
+				fprintf(stderr,
+					"ERROR: invalid value for stream version: %s\n",
+					optarg);
+				ret = 1;
+				goto out;
+			}
+			if (g_stream_version <= 0 ||
+			    g_stream_version > BTRFS_SEND_STREAM_VERSION_MAX) {
+				fprintf(stderr,
+					"ERROR: unsupported stream version %d, minimum: 1, maximum: %d\n",
+					g_stream_version,
+					BTRFS_SEND_STREAM_VERSION_MAX);
+				ret = 1;
+				goto out;
+			}
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -673,7 +701,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-ve] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -686,16 +714,19 @@ const char * const cmd_send_usage[] = {
 	"which case 'btrfs send' will determine a suitable parent among the",
 	"clone sources itself.",
 	"\n",
-	"-v               Enable verbose debug output. Each occurrence of",
-	"                 this option increases the verbose level more.",
-	"-e               If sending multiple subvols at once, use the new",
-	"                 format and omit the end-cmd between the subvols.",
-	"-p <parent>      Send an incremental stream from <parent> to",
-	"                 <subvol>.",
-	"-c <clone-src>   Use this snapshot as a clone source for an ",
-	"                 incremental send (multiple allowed)",
-	"-f <outfile>     Output is normally written to stdout. To write to",
-	"                 a file, use this option. An alternative would be to",
-	"                 use pipes.",
+	"-v                          Enable verbose debug output. Each occurrence of",
+	"                            this option increases the verbose level more.",
+	"-e                          If sending multiple subvols at once, use the new",
+	"                            format and omit the end-cmd between the subvols.",
+	"-p <parent>                 Send an incremental stream from <parent> to",
+	"                            <subvol>.",
+	"-c <clone-src>              Use this snapshot as a clone source for an ",
+	"                            incremental send (multiple allowed)",
+	"-f <outfile>                Output is normally written to stdout. To write to",
+	"                            a file, use this option. An alternative would be to",
+	"                            use pipes.",
+	"--stream-version <version>  Ask the kernel to produce a specific send stream",
+	"                            version. More recent stream versions provide new",
+	"                            features and better performance. Default value is 1.",
 	NULL
 };
diff --git a/ioctl.h b/ioctl.h
index 231660a..933eb01 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -392,6 +392,21 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * The sum of all length fields the receiver will get in write, clone and
+ * fallocate commands.
+ * This can be used by the receiver to compute progress, at the expense of some
+ * initial metadata scan performed by the sender (kernel).
+ *
+ * Added in send stream version 2.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
+/*
+ * Used by a client to request a version 2 of the send stream.
+ */
+#define BTRFS_SEND_FLAG_STREAM_V2               0x10
+
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */
 	__u64 clone_sources_count;	/* in */
diff --git a/send-stream.c b/send-stream.c
index 88e18e2..60c2126 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -462,7 +462,7 @@ int btrfs_read_and_process_send_stream(int fd,
 	}
 
 	s.version = le32_to_cpu(hdr.version);
-	if (s.version > BTRFS_SEND_STREAM_VERSION) {
+	if (s.version > BTRFS_SEND_STREAM_VERSION_MAX) {
 		ret = -EINVAL;
 		fprintf(stderr, "ERROR: Stream version %d not supported. "
 				"Please upgrade btrfs-progs\n", s.version);
diff --git a/send.h b/send.h
index e8da785..ea56965 100644
--- a/send.h
+++ b/send.h
@@ -24,7 +24,10 @@ extern "C" {
 #endif
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
+/* Max supported stream version. */
+#define BTRFS_SEND_STREAM_VERSION_MAX BTRFS_SEND_STREAM_VERSION_2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)
@@ -91,6 +94,15 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/*
+	 * The following commands were added in stream version 2.
+	 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+	BTRFS_SEND_C_INODE_SET_FLAGS,
+	BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -129,10 +141,19 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/*
+	 * The following attributes were added in stream version 2.
+	 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS, /* 32 bits */
+	BTRFS_SEND_A_INODE_FLAGS,     /* 32 bits */
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif
-- 
1.9.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo

[PATCH 2/4 v4] Btrfs-progs: send, implement total data size callback and progress report

[Filipe David Borba Manana]

This is a followup to the kernel patch titled:

    Btrfs: send, implement total data size command to allow for progress estimation

This makes the btrfs send and receive commands aware of the new send flag,
named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data
that is new between the parent and send snapshots/roots. As this command
immediately follows the commands to start a snapshot/subvolume, it can be
used to report and compute progress, by keeping a counter that is incremented
with the data length of each write, clone and fallocate command that is received
from the stream.

Example:

    $ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s

    $ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible and simple
to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send
output to btrfs-receive (directly to a file).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
    stream is now only produced is the ioctl caller specifies at least one of
    the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or
    BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Renamed option -o to -s, removed some duplicated code (progress reset).
V4: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE flag and -a command line option
    for btrfs-send. Both were replaced with BTRFS_SEND_FLAG_STREAM_V2 and
    --stream-version=<version_number> respectively. Added commands for inode
    set flags and otime too.

 Documentation/btrfs-send.txt |  4 ++
 cmds-receive.c               | 91 ++++++++++++++++++++++++++++++++++++++++++++
 cmds-send.c                  | 23 ++++++++++-
 send-stream.c                |  4 ++
 send-stream.h                |  1 +
 5 files changed, 121 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-send.txt b/Documentation/btrfs-send.txt
index 067fc27..1b18d32 100644
--- a/Documentation/btrfs-send.txt
+++ b/Documentation/btrfs-send.txt
@@ -43,6 +43,10 @@ An alternative would be to use pipes.
 --stream-version <version>::
 Ask the kernel to produce a specific send stream version. More recent stream versions provide
 new features and better performance. Default value is 1.
+-s::
+Obtain the total data size for each subvolume or snapshot to send. This demands additional
+processing (mostly IO bound) but is useful for the receive command to report progress. This
+option requires send stream version 2 or higher.
 
 EXIT STATUS
 -----------
diff --git a/cmds-receive.c b/cmds-receive.c
index d6cd3da..bd5255c 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -32,6 +32,7 @@
 #include <ftw.h>
 #include <wait.h>
 #include <assert.h>
+#include <time.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -71,6 +72,14 @@ struct btrfs_receive
 	struct subvol_uuid_search sus;
 
 	int honor_end_cmd;
+
+	/* For the subvolume/snapshot we're currently receiving. */
+	u64 total_data_size;
+	u64 bytes_received;
+	time_t last_progress_update;
+	u64 bytes_received_last_update;
+	float progress;
+	const char *target;
 };
 
 static int finish_subvol(struct btrfs_receive *r)
@@ -143,6 +152,16 @@ out:
 	return ret;
 }
 
+static void reset_progress(struct btrfs_receive *r, const char *dest)
+{
+	r->total_data_size = 0;
+	r->bytes_received = 0;
+	r->progress = 0.0;
+	r->last_progress_update = 0;
+	r->bytes_received_last_update = 0;
+	r->target = dest;
+}
+
 static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 			  void *user)
 {
@@ -156,6 +175,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	reset_progress(r, "Subvolume");
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -205,6 +225,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
 		goto out;
 
 	r->cur_subvol = calloc(1, sizeof(*r->cur_subvol));
+	reset_progress(r, "Snapshot");
 
 	if (strlen(r->dest_dir_path) == 0)
 		r->cur_subvol->path = strdup(path);
@@ -287,6 +308,73 @@ out:
 	return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+	struct btrfs_receive *r = user;
+
+	r->total_data_size = size;
+	fprintf(stdout, "About to receive %llu bytes\n", size);
+
+	return 0;
+}
+
+static void update_progress(struct btrfs_receive *r, u64 bytes)
+{
+	float new_progress;
+	time_t now;
+	time_t tdiff;
+
+	if (r->total_data_size == 0)
+		return;
+
+	r->bytes_received += bytes;
+
+	now = time(NULL);
+	tdiff = now - r->last_progress_update;
+	if (tdiff < 1) {
+		if (r->bytes_received == r->total_data_size)
+			fprintf(stdout, "\n");
+		return;
+	}
+
+	new_progress = ((float)r->bytes_received / r->total_data_size) * 100.0;
+
+	if ((int)(new_progress * 100) > (int)(r->progress * 100) ||
+	    r->bytes_received == r->total_data_size) {
+		char line[512];
+		float rate = r->bytes_received - r->bytes_received_last_update;
+		const char *rate_units;
+
+		rate /= tdiff;
+		if (rate > (1024 * 1024)) {
+			rate_units = "MB/s";
+			rate /= 1024 * 1024;
+		} else if (rate > 1024) {
+			rate_units = "KB/s";
+			rate /= 1024;
+		} else {
+			rate_units = "B/s";
+		}
+
+		snprintf(line, sizeof(line),
+			 "%s%s %s, %llu / %llu bytes received, %5.2f%%, %5.2f%s%s",
+			 (g_verbose ? "" : "\r"),
+			 r->target,
+			 r->full_subvol_path,
+			 r->bytes_received, r->total_data_size,
+			 new_progress, rate, rate_units,
+			 (g_verbose ? "\n" : ""));
+		fprintf(stdout, "%s%s", line, (g_verbose ? "" : "        "));
+		fflush(stdout);
+	}
+
+	if (r->bytes_received == r->total_data_size)
+		fprintf(stdout, "\n");
+	r->progress = new_progress;
+	r->last_progress_update = now;
+	r->bytes_received_last_update = r->bytes_received;
+}
+
 static int process_mkfile(const char *path, void *user)
 {
 	int ret;
@@ -562,6 +650,7 @@ static int process_write(const char *path, const void *data, u64 offset,
 		}
 		pos += w;
 	}
+	update_progress(r, len);
 
 out:
 	free(full_path);
@@ -638,6 +727,7 @@ static int process_clone(const char *path, u64 offset, u64 len,
 				path, strerror(-ret));
 		goto out;
 	}
+	update_progress(r, len);
 
 out:
 	if (si) {
@@ -819,6 +909,7 @@ static struct btrfs_send_ops send_ops = {
 	.chmod = process_chmod,
 	.chown = process_chown,
 	.utimes = process_utimes,
+	.total_data_size = process_total_data_size,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/cmds-send.c b/cmds-send.c
index bd575f8..ff17092 100644
--- a/cmds-send.c
+++ b/cmds-send.c
@@ -47,6 +47,7 @@
 
 static int g_verbose = 0;
 static int g_stream_version = BTRFS_SEND_STREAM_VERSION_1;
+static int g_total_data_size = 0;
 
 struct btrfs_send {
 	int send_fd;
@@ -285,6 +286,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
 	if (g_stream_version == BTRFS_SEND_STREAM_VERSION_2)
 		io_send.flags |= BTRFS_SEND_FLAG_STREAM_V2;
+	if (g_total_data_size)
+		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret) {
 		ret = -errno;
@@ -433,7 +436,7 @@ int cmd_send(int argc, char **argv)
 	memset(&send, 0, sizeof(send));
 	send.dump_fd = fileno(stdout);
 
-	while ((c = getopt_long(argc, argv, "vec:f:i:p:",
+	while ((c = getopt_long(argc, argv, "vesc:f:i:p:",
 				long_options, NULL)) != -1) {
 		switch (c) {
 		case 'v':
@@ -539,6 +542,9 @@ int cmd_send(int argc, char **argv)
 				goto out;
 			}
 			break;
+		case 's':
+			g_total_data_size = 1;
+			break;
 		case '?':
 		default:
 			fprintf(stderr, "ERROR: send args invalid.\n");
@@ -550,6 +556,14 @@ int cmd_send(int argc, char **argv)
 	if (optind == argc)
 		usage(cmd_send_usage);
 
+	if (g_total_data_size &&
+	    g_stream_version < BTRFS_SEND_STREAM_VERSION_2) {
+		fprintf(stderr,
+			"ERROR: option total data size (-s) requires use of the send stream version 2 or higher\n");
+		ret = 1;
+		goto out;
+	}
+
 	if (outname != NULL) {
 		send.dump_fd = creat(outname, 0600);
 		if (send.dump_fd == -1) {
@@ -701,7 +715,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-ves] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"By default, this will send the whole subvolume. To do an incremental",
@@ -728,5 +742,10 @@ const char * const cmd_send_usage[] = {
 	"--stream-version <version>  Ask the kernel to produce a specific send stream",
 	"                            version. More recent stream versions provide new",
 	"                            features and better performance. Default value is 1.",
+	"-s                          Obtain the total data size for each subvolume or ",
+	"                            snapshot to send. This demands additional processing",
+	"                            (mostly IO bound) but is useful for the receive ",
+	"                            command to report progress. This option requires use",
+	"                            of the send stream version 2 or higher.",
 	NULL
 };
diff --git a/send-stream.c b/send-stream.c
index 60c2126..e1bd4ce 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->update_extent(path, offset, tmp, s->user);
 		break;
+	case BTRFS_SEND_C_TOTAL_DATA_SIZE:
+		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
+		ret = s->ops->total_data_size(tmp, s->user);
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 17bc669..3a653a9 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -54,6 +54,7 @@ struct btrfs_send_ops {
 		      struct timespec *mt, struct timespec *ct,
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
+	int (*total_data_size)(u64 size, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo

[PATCH 3/4 v4] Btrfs-progs: send, implement fallocate command callback

[Filipe David Borba Manana]

The fallocate send stream command, added in stream version 2, is used to
pre-allocate space for files and punch file holes. This change implements
the callback for that new command, using the fallocate function from the
standard C library to carry out the specified action (allocate file space
or punch a file hole).

Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>
---

V2: Use the new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE if the user
    asks for it (-a command line option), which will make the kernel generate
    a version 2 send stream, so that old clients aren't affected.
V3: Rebased on new patchset (new version of patch 2/4).
V4: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE flag and -a command line option
    for btrfs-send. Both were replaced with BTRFS_SEND_FLAG_STREAM_V2 and
    --stream-version=<version_number> respectively. Added commands for inode
    set flags and otime too.

 cmds-receive.c | 38 ++++++++++++++++++++++++++++++++++++++
 send-stream.c  | 13 +++++++++++++
 send-stream.h  |  2 ++
 3 files changed, 53 insertions(+)

diff --git a/cmds-receive.c b/cmds-receive.c
index bd5255c..5e96423 100644
--- a/cmds-receive.c
+++ b/cmds-receive.c
@@ -41,6 +41,7 @@
 #include <sys/types.h>
 #include <sys/xattr.h>
 #include <uuid/uuid.h>
+#include <linux/falloc.h>
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -887,6 +888,42 @@ out:
 	return ret;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+			     u64 len, void *user)
+{
+	struct btrfs_receive *r = user;
+	char *full_path = path_cat(r->full_subvol_path, path);
+	int mode = 0;
+	int ret;
+
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+		mode |= FALLOC_FL_KEEP_SIZE;
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+		mode |= FALLOC_FL_PUNCH_HOLE;
+
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"fallocate %s - flags %u, offset %llu, len %llu\n",
+			path, flags, offset, len);
+
+	ret = open_inode_for_write(r, full_path);
+	if (ret < 0)
+		goto out;
+
+	ret = fallocate(r->write_fd, mode, offset, len);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr,
+			"ERROR: fallocate against %s failed. %s\n",
+			path, strerror(-ret));
+		goto out;
+	}
+	update_progress(r, len);
+
+out:
+	free(full_path);
+	return ret;
+}
 
 static struct btrfs_send_ops send_ops = {
 	.subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = {
 	.chown = process_chown,
 	.utimes = process_utimes,
 	.total_data_size = process_total_data_size,
+	.fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *r, const char *tomnt, int r_fd)
diff --git a/send-stream.c b/send-stream.c
index e1bd4ce..812639f 100644
--- a/send-stream.c
+++ b/send-stream.c
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)
 		TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &tmp);
 		ret = s->ops->total_data_size(tmp, s->user);
 		break;
+	case BTRFS_SEND_C_FALLOCATE:
+		{
+			u32 flags;
+			u64 len;
+
+			TLV_GET_STRING(s, BTRFS_SEND_A_PATH, &path);
+			TLV_GET_U32(s, BTRFS_SEND_A_FALLOCATE_FLAGS, &flags);
+			TLV_GET_U64(s, BTRFS_SEND_A_FILE_OFFSET, &offset);
+			TLV_GET_U64(s, BTRFS_SEND_A_SIZE, &len);
+			ret = s->ops->fallocate(path, flags, offset, len,
+						s->user);
+		}
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
diff --git a/send-stream.h b/send-stream.h
index 3a653a9..479e40c 100644
--- a/send-stream.h
+++ b/send-stream.h
@@ -55,6 +55,8 @@ struct btrfs_send_ops {
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
 	int (*total_data_size)(u64 size, void *user);
+	int (*fallocate)(const char *path, u32 flags, u64 offset,
+			 u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
-- 
1.9.1

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo

Re: [PATCH 1/4] Btrfs-progs: send, bump stream version

[David Sterba]

On Tue, Apr 15, 2014 at 05:40:48PM +0100, Filipe David Borba Manana wrote:
> This increases the send stream version from version 1 to version 2, adding
> 2 new commands:
> 
> 1) total data size - used to tell the receiver how much file data the stream
>    will add or update;
> 
> 2) fallocate - used to pre-allocate space for files and to punch holes in files.
> 
> This is preparation work for subsequent changes that implement the new features
> (computing total data size and use fallocate for better performance).
> 
> Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com>

The changes in the v2/3/4 look good, thanks.  Patches added to next
integratin.