[PATCH 1/2] fsx: Add fallocate collapse range operation

[PATCH 1/2] fsx: Add fallocate collapse range operation

[Lukas Czerner]

This commit adds fallocate FALLOC_FL_COLLAPSE_RANGE support for fsx.

Signed-off-by: Lukas Czerner <lczerner@redhat.com>
---
 ltp/fsx.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 100 insertions(+), 7 deletions(-)

diff --git a/ltp/fsx.c b/ltp/fsx.c
index bd1ed76..47d3ee8 100644
--- a/ltp/fsx.c
+++ b/ltp/fsx.c
@@ -90,11 +90,12 @@ int			logcount = 0;	/* total ops */
 #define OP_MAX_LITE	4
 
 /* !lite operations */
-#define OP_TRUNCATE	4
-#define OP_FALLOCATE	5
-#define OP_PUNCH_HOLE	6
-#define OP_ZERO_RANGE	7
-#define OP_MAX_FULL	8
+#define OP_TRUNCATE		4
+#define OP_FALLOCATE		5
+#define OP_PUNCH_HOLE		6
+#define OP_ZERO_RANGE		7
+#define OP_COLLAPSE_RANGE	8
+#define OP_MAX_FULL		9
 
 /* operation modifiers */
 #define OP_CLOSEOPEN	100
@@ -111,6 +112,7 @@ char	*temp_buf;			/* a pointer to the current data */
 char	*fname;				/* name of our test file */
 int	fd;				/* fd for our test file */
 
+blksize_t	block_size = 0;
 off_t		file_size = 0;
 off_t		biggest = 0;
 char		state[256];
@@ -142,6 +144,7 @@ int     mapped_writes = 1;              /* -W flag disables */
 int     fallocate_calls = 1;            /* -F flag disables */
 int     punch_hole_calls = 1;           /* -H flag disables */
 int     zero_range_calls = 1;           /* -z flag disables */
+int	collapse_range_calls = 1;	/* -C flag disables */
 int 	mapped_reads = 1;		/* -R flag disables it */
 int	fsxgoodfd = 0;
 int	o_direct;			/* -Z */
@@ -321,13 +324,21 @@ logdump(void)
 				prt("\t******PPPP");
 			break;
 		case OP_ZERO_RANGE:
-			prt("ZERO    0x%x thru 0x%x\t(0x%x bytes)",
+			prt("ZERO     0x%x thru 0x%x\t(0x%x bytes)",
 			    lp->args[0], lp->args[0] + lp->args[1] - 1,
 			    lp->args[1]);
 			if (badoff >= lp->args[0] && badoff <
 						     lp->args[0] + lp->args[1])
 				prt("\t******ZZZZ");
 			break;
+		case OP_COLLAPSE_RANGE:
+			prt("COLLAPSE 0x%x thru 0x%x\t(0x%x bytes)",
+			    lp->args[0], lp->args[0] + lp->args[1] - 1,
+			    lp->args[1]);
+			if (badoff >= lp->args[0] && badoff <
+						     lp->args[0] + lp->args[1])
+				prt("\t******CCCC");
+			break;
 		case OP_SKIPPED:
 			prt("SKIPPED (no operation)");
 			break;
@@ -949,6 +960,58 @@ do_zero_range(unsigned offset, unsigned length)
 }
 #endif
 
+#ifdef FALLOC_FL_COLLAPSE_RANGE
+void
+do_collapse_range(unsigned offset, unsigned length)
+{
+	unsigned end_offset;
+	int mode = FALLOC_FL_COLLAPSE_RANGE;
+
+	if (length == 0) {
+		if (!quiet && testcalls > simulatedopcount)
+			prt("skipping zero length collapse range\n");
+		log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, length);
+		return;
+	}
+
+	end_offset = offset + length;
+	if ((loff_t)end_offset >= file_size) {
+		if (!quiet && testcalls > simulatedopcount)
+			prt("skipping collapse range behind EOF\n");
+		log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, length);
+		return;
+	}
+
+	log4(OP_COLLAPSE_RANGE, offset, length, 0);
+
+	if (testcalls <= simulatedopcount)
+		return;
+
+	if ((progressinterval && testcalls % progressinterval == 0) ||
+	    (debug && (monitorstart == -1 || monitorend == -1 ||
+		      end_offset <= monitorend))) {
+		prt("%lu collapse\tfrom 0x%x to 0x%x, (0x%x bytes)\n", testcalls,
+			offset, offset+length, length);
+	}
+	if (fallocate(fd, mode, (loff_t)offset, (loff_t)length) == -1) {
+		prt("collapse range: %x to %x\n", offset, length);
+		prterr("do_collapse_range: fallocate");
+		report_failure(161);
+	}
+
+	memmove(good_buf + offset, good_buf + end_offset,
+		file_size - end_offset);
+	file_size -= length;
+}
+
+#else
+void
+do_collapse_range(unsigned offset, unsigned length)
+{
+	return;
+}
+#endif
+
 #ifdef HAVE_LINUX_FALLOC_H
 /* fallocate is basically a no-op unless extending, then a lot like a truncate */
 void
@@ -1123,6 +1186,12 @@ test(void)
 			goto out;
 		}
 		break;
+	case OP_COLLAPSE_RANGE:
+		if (!collapse_range_calls) {
+			log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, size);
+			goto out;
+		}
+		break;
 	}
 
 	switch (op) {
@@ -1165,6 +1234,16 @@ test(void)
 		TRIM_OFF_LEN(offset, size, file_size);
 		do_zero_range(offset, size);
 		break;
+	case OP_COLLAPSE_RANGE:
+		TRIM_OFF_LEN(offset, size, file_size - 1);
+		offset = offset & ~(block_size - 1);
+		size = size & ~(block_size - 1);
+		if (size == 0) {
+			log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, size);
+			goto out;
+		}
+		do_collapse_range(offset, size);
+		break;
 	default:
 		prterr("test: unknown operation");
 		report_failure(42);
@@ -1225,6 +1304,9 @@ usage(void)
 #ifdef FALLOC_FL_ZERO_RANGE
 "	-z: Do not use zero range calls\n"
 #endif
+#ifdef FALLOC_FL_COLLAPSE_RANGE
+"	-C: Do not use collapse range calls\n"
+#endif
 "	-L: fsxLite - no file creations & no file size changes\n\
 	-N numops: total # operations to do (default infinity)\n\
 	-O: use oplen (see -o flag) for every op (default random)\n\
@@ -1399,6 +1481,7 @@ main(int argc, char **argv)
 	char	*endp;
 	char goodfile[1024];
 	char logfile[1024];
+	struct stat statbuf;
 
 	goodfile[0] = 0;
 	logfile[0] = 0;
@@ -1410,7 +1493,7 @@ main(int argc, char **argv)
 
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
-	while ((ch = getopt(argc, argv, "b:c:dfl:m:no:p:qr:s:t:w:xyAD:FHzLN:OP:RS:WZ"))
+	while ((ch = getopt(argc, argv, "b:c:dfl:m:no:p:qr:s:t:w:xyAD:FHzCLN:OP:RS:WZ"))
 	       != EOF)
 		switch (ch) {
 		case 'b':
@@ -1513,6 +1596,9 @@ main(int argc, char **argv)
 		case 'z':
 			zero_range_calls = 0;
 			break;
+		case 'C':
+			collapse_range_calls = 0;
+			break;
 		case 'L':
 		        lite = 1;
 			break;
@@ -1579,6 +1665,11 @@ main(int argc, char **argv)
 		prterr(fname);
 		exit(91);
 	}
+	if (fstat(fd, &statbuf)) {
+		prterr("check_size: fstat");
+		exit(91);
+	}
+	block_size = statbuf.st_blksize;
 #ifdef XFS
 	if (prealloc) {
 		xfs_flock64_t	resv = { 0 };
@@ -1665,6 +1756,8 @@ main(int argc, char **argv)
 						  FALLOC_FL_KEEP_SIZE);
 	if (zero_range_calls)
 		zero_range_calls = test_fallocate(FALLOC_FL_ZERO_RANGE);
+	if (collapse_range_calls)
+		collapse_range_calls = test_fallocate(FALLOC_FL_COLLAPSE_RANGE);
 
 	while (numops == -1 || numops--)
 		test();
-- 
1.8.3.1

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs

[PATCH 2/2] fsstress: Add fallocate collapse range operation

[Lukas Czerner]

This commit adds collapse operation support for fsstress, which is
meant to exercise fallocate FALLOC_FL_COLLAPSE_RANGE support.

Signed-off-by: Lukas Czerner <lczerner@redhat.com>
---
 ltp/fsstress.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ltp/fsstress.c b/ltp/fsstress.c
index fd258bf..1eec11a 100644
--- a/ltp/fsstress.c
+++ b/ltp/fsstress.c
@@ -71,6 +71,7 @@ typedef enum {
 	OP_MKNOD,
 	OP_PUNCH,
 	OP_ZERO,
+	OP_COLLAPSE,
 	OP_READ,
 	OP_READLINK,
 	OP_RENAME,
@@ -168,6 +169,7 @@ void	mkdir_f(int, long);
 void	mknod_f(int, long);
 void	punch_f(int, long);
 void	zero_f(int, long);
+void	collapse_f(int, long);
 void	read_f(int, long);
 void	readlink_f(int, long);
 void	rename_f(int, long);
@@ -206,6 +208,7 @@ opdesc_t	ops[] = {
 	{ OP_MKNOD, "mknod", mknod_f, 2, 1 },
 	{ OP_PUNCH, "punch", punch_f, 1, 1 },
 	{ OP_ZERO, "zero", zero_f, 1, 1 },
+	{ OP_COLLAPSE, "collapse", collapse_f, 1, 1 },
 	{ OP_READ, "read", read_f, 1, 0 },
 	{ OP_READLINK, "readlink", readlink_f, 1, 0 },
 	{ OP_RENAME, "rename", rename_f, 2, 1 },
@@ -2173,6 +2176,7 @@ struct print_flags falloc_flags [] = {
 	{ FALLOC_FL_NO_HIDE_STALE, "NO_HIDE_STALE"},
 	{ FALLOC_FL_COLLAPSE_RANGE, "COLLAPSE_RANGE"},
 	{ FALLOC_FL_ZERO_RANGE, "ZERO_RANGE"},
+	{ FALLOC_FL_COLLAPSE_RANGE, "COLLAPSE_RANGE"},
 	{ -1, NULL}
 };
 
@@ -2223,6 +2227,14 @@ do_fallocate(int opno, long r, int mode)
 	off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
 	off %= maxfsize;
 	len = (off64_t)(random() % (1024 * 1024));
+	/*
+	 * Collapse range requires off and len to be block aligned, make it
+	 * more likely to be the case.
+	 */
+	if (FALLOC_FL_COLLAPSE_RANGE && (opno % 2)) {
+		off = ((off + stb.st_blksize - 1) & ~(stb.st_blksize - 1));
+		len = ((len + stb.st_blksize - 1) & ~(stb.st_blksize - 1));
+	}
 	mode |= FALLOC_FL_KEEP_SIZE & random();
 	e = fallocate(fd, mode, (loff_t)off, (loff_t)len) < 0 ? errno : 0;
 	if (v)
@@ -2637,6 +2649,14 @@ zero_f(int opno, long r)
 }
 
 void
+collapse_f(int opno, long r)
+{
+#ifdef HAVE_LINUX_FALLOC_H
+	do_fallocate(opno, r, FALLOC_FL_COLLAPSE_RANGE);
+#endif
+}
+
+void
 read_f(int opno, long r)
 {
 	char		*buf;
-- 
1.8.3.1

Re: [PATCH 1/2] fsx: Add fallocate collapse range operation

[Lukáš Czerner]

On Wed, 2 Apr 2014, Lukas Czerner wrote:

> Date: Wed,  2 Apr 2014 17:33:19 +0200
> From: Lukas Czerner <lczerner@redhat.com>
> To: xfs@oss.sgi.com
> Cc: linux-ext4@vger.kernel.org, linux-fsdevel@vger.kernel.org,
>     Lukas Czerner <lczerner@redhat.com>
> Subject: [PATCH 1/2] fsx: Add fallocate collapse range operation
> 
> This commit adds fallocate FALLOC_FL_COLLAPSE_RANGE support for fsx.

Btw, this actually makes the fsx fail on ext4 and xfs after a while.
I was trying to find a problem in fsx itself but I think it is
actually ok. That means that we could possibly have bugs in ext4 and
xfs collapse range implementation. More eyes are needed on this!

Thanks!
-Lukas

> 
> Signed-off-by: Lukas Czerner <lczerner@redhat.com>
> ---
>  ltp/fsx.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
>  1 file changed, 100 insertions(+), 7 deletions(-)
> 
> diff --git a/ltp/fsx.c b/ltp/fsx.c
> index bd1ed76..47d3ee8 100644
> --- a/ltp/fsx.c
> +++ b/ltp/fsx.c
> @@ -90,11 +90,12 @@ int			logcount = 0;	/* total ops */
>  #define OP_MAX_LITE	4
>  
>  /* !lite operations */
> -#define OP_TRUNCATE	4
> -#define OP_FALLOCATE	5
> -#define OP_PUNCH_HOLE	6
> -#define OP_ZERO_RANGE	7
> -#define OP_MAX_FULL	8
> +#define OP_TRUNCATE		4
> +#define OP_FALLOCATE		5
> +#define OP_PUNCH_HOLE		6
> +#define OP_ZERO_RANGE		7
> +#define OP_COLLAPSE_RANGE	8
> +#define OP_MAX_FULL		9
>  
>  /* operation modifiers */
>  #define OP_CLOSEOPEN	100
> @@ -111,6 +112,7 @@ char	*temp_buf;			/* a pointer to the current data */
>  char	*fname;				/* name of our test file */
>  int	fd;				/* fd for our test file */
>  
> +blksize_t	block_size = 0;
>  off_t		file_size = 0;
>  off_t		biggest = 0;
>  char		state[256];
> @@ -142,6 +144,7 @@ int     mapped_writes = 1;              /* -W flag disables */
>  int     fallocate_calls = 1;            /* -F flag disables */
>  int     punch_hole_calls = 1;           /* -H flag disables */
>  int     zero_range_calls = 1;           /* -z flag disables */
> +int	collapse_range_calls = 1;	/* -C flag disables */
>  int 	mapped_reads = 1;		/* -R flag disables it */
>  int	fsxgoodfd = 0;
>  int	o_direct;			/* -Z */
> @@ -321,13 +324,21 @@ logdump(void)
>  				prt("\t******PPPP");
>  			break;
>  		case OP_ZERO_RANGE:
> -			prt("ZERO    0x%x thru 0x%x\t(0x%x bytes)",
> +			prt("ZERO     0x%x thru 0x%x\t(0x%x bytes)",
>  			    lp->args[0], lp->args[0] + lp->args[1] - 1,
>  			    lp->args[1]);
>  			if (badoff >= lp->args[0] && badoff <
>  						     lp->args[0] + lp->args[1])
>  				prt("\t******ZZZZ");
>  			break;
> +		case OP_COLLAPSE_RANGE:
> +			prt("COLLAPSE 0x%x thru 0x%x\t(0x%x bytes)",
> +			    lp->args[0], lp->args[0] + lp->args[1] - 1,
> +			    lp->args[1]);
> +			if (badoff >= lp->args[0] && badoff <
> +						     lp->args[0] + lp->args[1])
> +				prt("\t******CCCC");
> +			break;
>  		case OP_SKIPPED:
>  			prt("SKIPPED (no operation)");
>  			break;
> @@ -949,6 +960,58 @@ do_zero_range(unsigned offset, unsigned length)
>  }
>  #endif
>  
> +#ifdef FALLOC_FL_COLLAPSE_RANGE
> +void
> +do_collapse_range(unsigned offset, unsigned length)
> +{
> +	unsigned end_offset;
> +	int mode = FALLOC_FL_COLLAPSE_RANGE;
> +
> +	if (length == 0) {
> +		if (!quiet && testcalls > simulatedopcount)
> +			prt("skipping zero length collapse range\n");
> +		log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, length);
> +		return;
> +	}
> +
> +	end_offset = offset + length;
> +	if ((loff_t)end_offset >= file_size) {
> +		if (!quiet && testcalls > simulatedopcount)
> +			prt("skipping collapse range behind EOF\n");
> +		log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, length);
> +		return;
> +	}
> +
> +	log4(OP_COLLAPSE_RANGE, offset, length, 0);
> +
> +	if (testcalls <= simulatedopcount)
> +		return;
> +
> +	if ((progressinterval && testcalls % progressinterval == 0) ||
> +	    (debug && (monitorstart == -1 || monitorend == -1 ||
> +		      end_offset <= monitorend))) {
> +		prt("%lu collapse\tfrom 0x%x to 0x%x, (0x%x bytes)\n", testcalls,
> +			offset, offset+length, length);
> +	}
> +	if (fallocate(fd, mode, (loff_t)offset, (loff_t)length) == -1) {
> +		prt("collapse range: %x to %x\n", offset, length);
> +		prterr("do_collapse_range: fallocate");
> +		report_failure(161);
> +	}
> +
> +	memmove(good_buf + offset, good_buf + end_offset,
> +		file_size - end_offset);
> +	file_size -= length;
> +}
> +
> +#else
> +void
> +do_collapse_range(unsigned offset, unsigned length)
> +{
> +	return;
> +}
> +#endif
> +
>  #ifdef HAVE_LINUX_FALLOC_H
>  /* fallocate is basically a no-op unless extending, then a lot like a truncate */
>  void
> @@ -1123,6 +1186,12 @@ test(void)
>  			goto out;
>  		}
>  		break;
> +	case OP_COLLAPSE_RANGE:
> +		if (!collapse_range_calls) {
> +			log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, size);
> +			goto out;
> +		}
> +		break;
>  	}
>  
>  	switch (op) {
> @@ -1165,6 +1234,16 @@ test(void)
>  		TRIM_OFF_LEN(offset, size, file_size);
>  		do_zero_range(offset, size);
>  		break;
> +	case OP_COLLAPSE_RANGE:
> +		TRIM_OFF_LEN(offset, size, file_size - 1);
> +		offset = offset & ~(block_size - 1);
> +		size = size & ~(block_size - 1);
> +		if (size == 0) {
> +			log4(OP_SKIPPED, OP_COLLAPSE_RANGE, offset, size);
> +			goto out;
> +		}
> +		do_collapse_range(offset, size);
> +		break;
>  	default:
>  		prterr("test: unknown operation");
>  		report_failure(42);
> @@ -1225,6 +1304,9 @@ usage(void)
>  #ifdef FALLOC_FL_ZERO_RANGE
>  "	-z: Do not use zero range calls\n"
>  #endif
> +#ifdef FALLOC_FL_COLLAPSE_RANGE
> +"	-C: Do not use collapse range calls\n"
> +#endif
>  "	-L: fsxLite - no file creations & no file size changes\n\
>  	-N numops: total # operations to do (default infinity)\n\
>  	-O: use oplen (see -o flag) for every op (default random)\n\
> @@ -1399,6 +1481,7 @@ main(int argc, char **argv)
>  	char	*endp;
>  	char goodfile[1024];
>  	char logfile[1024];
> +	struct stat statbuf;
>  
>  	goodfile[0] = 0;
>  	logfile[0] = 0;
> @@ -1410,7 +1493,7 @@ main(int argc, char **argv)
>  
>  	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
>  
> -	while ((ch = getopt(argc, argv, "b:c:dfl:m:no:p:qr:s:t:w:xyAD:FHzLN:OP:RS:WZ"))
> +	while ((ch = getopt(argc, argv, "b:c:dfl:m:no:p:qr:s:t:w:xyAD:FHzCLN:OP:RS:WZ"))
>  	       != EOF)
>  		switch (ch) {
>  		case 'b':
> @@ -1513,6 +1596,9 @@ main(int argc, char **argv)
>  		case 'z':
>  			zero_range_calls = 0;
>  			break;
> +		case 'C':
> +			collapse_range_calls = 0;
> +			break;
>  		case 'L':
>  		        lite = 1;
>  			break;
> @@ -1579,6 +1665,11 @@ main(int argc, char **argv)
>  		prterr(fname);
>  		exit(91);
>  	}
> +	if (fstat(fd, &statbuf)) {
> +		prterr("check_size: fstat");
> +		exit(91);
> +	}
> +	block_size = statbuf.st_blksize;
>  #ifdef XFS
>  	if (prealloc) {
>  		xfs_flock64_t	resv = { 0 };
> @@ -1665,6 +1756,8 @@ main(int argc, char **argv)
>  						  FALLOC_FL_KEEP_SIZE);
>  	if (zero_range_calls)
>  		zero_range_calls = test_fallocate(FALLOC_FL_ZERO_RANGE);
> +	if (collapse_range_calls)
> +		collapse_range_calls = test_fallocate(FALLOC_FL_COLLAPSE_RANGE);
>  
>  	while (numops == -1 || numops--)
>  		test();
> 

Re: [PATCH 1/2] fsx: Add fallocate collapse range operation

[Dave Chinner]

On Wed, Apr 02, 2014 at 05:55:07PM +0200, Lukáš Czerner wrote:
> On Wed, 2 Apr 2014, Lukas Czerner wrote:
> 
> > Date: Wed,  2 Apr 2014 17:33:19 +0200
> > From: Lukas Czerner <lczerner@redhat.com>
> > To: xfs@oss.sgi.com
> > Cc: linux-ext4@vger.kernel.org, linux-fsdevel@vger.kernel.org,
> >     Lukas Czerner <lczerner@redhat.com>
> > Subject: [PATCH 1/2] fsx: Add fallocate collapse range operation
> > 
> > This commit adds fallocate FALLOC_FL_COLLAPSE_RANGE support for fsx.
> 
> Btw, this actually makes the fsx fail on ext4 and xfs after a while.
> I was trying to find a problem in fsx itself but I think it is
> actually ok. That means that we could possibly have bugs in ext4 and
> xfs collapse range implementation. More eyes are needed on this!

No surprise, really. As it is, your previous ZERO_RANGE additions
caused problems for XFS, and those are mostly fixed in the 6 patch
series I sent before LSFMM.  Hence I wouldn't be surprised if that's
what you are seeing (on XFs, at least) and it's not actually
COLLAPSE_RANGE that is causing problems...

Cheers,

Dave.
-- 
Dave Chinner
david@fromorbit.com
--
To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html