[PATCH v4 0/5] fsx additions for crash consistency tests

[PATCH v4 0/5] fsx additions for crash consistency tests

[Amir Goldstein]

Eryu,

This is a re-post of v3 fsx patches with a correction to
the 2nd patch which introduces -j <logid>.

Amir.

Amir Goldstein (5):
  fsx: add support for integrity check with dm-log-writes target
  fsx: add optional logid prefix to log messages
  fsx: add support for recording operations to a file
  fsx: add support for writing constant instead of random data
  fsx: add support for keeping existing file

 ltp/fsx.c | 274 ++++++++++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 214 insertions(+), 60 deletions(-)

-- 
2.7.4

[PATCH v4 1/5] fsx: add support for integrity check with dm-log-writes target

[Amir Goldstein]

Cherry-picked the relevant fsx bits from commit 70d41e17164b
in Josef Bacik's fstests tree (https://github.com/josefbacik/fstests).
Quoting from Josef's commit message:

  I've rigged up fsx to have an integrity check mode.  Basically it works
  like it normally works, but when it fsync()'s it marks the log with a
  unique mark and dumps it's buffer to a file with the mark in the filename.
  I did this with a system() call simply because it was the fastest.  I can
  link the device-mapper libraries and do it programatically if that would
  be preferred, but this works pretty well.

  Signed-off-by: Josef Bacik <jbacik@fb.com>

[Amir:]
- Fix some exit codes
- Require -P dirpath for -i logdev

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
---
 ltp/fsx.c | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 123 insertions(+), 24 deletions(-)

diff --git a/ltp/fsx.c b/ltp/fsx.c
index ea00ff7..ac466ee 100644
--- a/ltp/fsx.c
+++ b/ltp/fsx.c
@@ -67,15 +67,17 @@ int			logcount = 0;	/* total ops */
  * be careful in how we select the different operations. The active operations
  * are mapped to numbers as follows:
  *
- *		lite	!lite
- * READ:	0	0
- * WRITE:	1	1
- * MAPREAD:	2	2
- * MAPWRITE:	3	3
- * TRUNCATE:	-	4
- * FALLOCATE:	-	5
- * PUNCH HOLE:	-	6
- * ZERO RANGE:	-	7
+ *			lite	!lite	integrity
+ * READ:		0	0	0
+ * WRITE:		1	1	1
+ * MAPREAD:		2	2	2
+ * MAPWRITE:		3	3	3
+ * TRUNCATE:		-	4	4
+ * FALLOCATE:		-	5	5
+ * PUNCH HOLE:		-	6	6
+ * ZERO RANGE:		-	7	7
+ * COLLAPSE RANGE:	-	8	8
+ * FSYNC:		-	-	9
  *
  * When mapped read/writes are disabled, they are simply converted to normal
  * reads and writes. When fallocate/fpunch calls are disabled, they are
@@ -102,6 +104,10 @@ int			logcount = 0;	/* total ops */
 #define OP_INSERT_RANGE	9
 #define OP_MAX_FULL		10
 
+/* integrity operations */
+#define OP_FSYNC		10
+#define OP_MAX_INTEGRITY	11
+
 #undef PAGE_SIZE
 #define PAGE_SIZE       getpagesize()
 #undef PAGE_MASK
@@ -111,6 +117,10 @@ char	*original_buf;			/* a pointer to the original data */
 char	*good_buf;			/* a pointer to the correct data */
 char	*temp_buf;			/* a pointer to the current data */
 char	*fname;				/* name of our test file */
+char	*bname;				/* basename of our test file */
+char	*logdev;			/* -i flag */
+char	dname[1024];			/* -P flag */
+int	dirpath = 0;			/* -P flag */
 int	fd;				/* fd for our test file */
 
 blksize_t	block_size = 0;
@@ -148,9 +158,11 @@ int     zero_range_calls = 1;           /* -z flag disables */
 int	collapse_range_calls = 1;	/* -C flag disables */
 int	insert_range_calls = 1;		/* -I flag disables */
 int 	mapped_reads = 1;		/* -R flag disables it */
+int	integrity = 0;			/* -i flag */
 int	fsxgoodfd = 0;
 int	o_direct;			/* -Z */
 int	aio = 0;
+int	mark_nr = 0;
 
 int page_size;
 int page_mask;
@@ -234,6 +246,7 @@ static const char *op_names[] = {
 	[OP_ZERO_RANGE] = "zero_range",
 	[OP_COLLAPSE_RANGE] = "collapse_range",
 	[OP_INSERT_RANGE] = "insert_range",
+	[OP_FSYNC] = "fsync",
 };
 
 static const char *op_name(int operation)
@@ -397,6 +410,9 @@ logdump(void)
 			if (overlap)
 				prt("\t******IIII");
 			break;
+		case OP_FSYNC:
+			prt("FSYNC");
+			break;
 		default:
 			prt("BOGUS LOG ENTRY (operation code = %d)!",
 			    lp->operation);
@@ -500,6 +516,43 @@ report_failure(int status)
 				        *(((unsigned char *)(cp)) + 1)))
 
 void
+mark_log(void)
+{
+	char command[256];
+	int ret;
+
+	snprintf(command, 256, "dmsetup message %s 0 mark %s.mark%d", logdev,
+		 bname, mark_nr);
+	ret = system(command);
+	if (ret) {
+		prterr("dmsetup mark failed");
+		exit(211);
+	}
+}
+
+void
+dump_fsync_buffer(void)
+{
+	char fname_buffer[1024];
+	int good_fd;
+
+	if (!good_buf)
+		return;
+
+	snprintf(fname_buffer, 1024, "%s%s.mark%d", dname,
+		 bname, mark_nr);
+	good_fd = open(fname_buffer, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+	if (good_fd < 0) {
+		prterr(fname_buffer);
+		exit(212);
+	}
+
+	save_buffer(good_buf, file_size, good_fd);
+	close(good_fd);
+	prt("Dumped fsync buffer to %s\n", fname_buffer + dirpath);
+}
+
+void
 check_buffers(unsigned offset, unsigned size)
 {
 	unsigned char c, t;
@@ -1256,6 +1309,25 @@ docloseopen(void)
 	}
 }
 
+void
+dofsync(void)
+{
+	int ret;
+
+	if (testcalls <= simulatedopcount)
+		return;
+	if (debug)
+		prt("%lu fsync\n", testcalls);
+	log4(OP_FSYNC, 0, 0, 0);
+	ret = fsync(fd);
+	if (ret < 0) {
+		prterr("dofsync");
+		report_failure(210);
+	}
+	mark_log();
+	dump_fsync_buffer();
+	mark_nr++;
+}
 
 #define TRIM_OFF(off, size)			\
 do {						\
@@ -1403,8 +1475,10 @@ test(void)
 	/* calculate appropriate op to run */
 	if (lite)
 		op = rv % OP_MAX_LITE;
-	else
+	else if (!integrity)
 		op = rv % OP_MAX_FULL;
+	else
+		op = rv % OP_MAX_INTEGRITY;
 
 	switch(op) {
 	case OP_TRUNCATE:
@@ -1528,6 +1602,9 @@ have_op:
 
 		do_insert_range(offset, size);
 		break;
+	case OP_FSYNC:
+		dofsync();
+		break;
 	default:
 		prterr("test: unknown operation");
 		report_failure(42);
@@ -1547,11 +1624,12 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
+		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-i logdev] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\
 	-f flush and invalidate cache after I/O\n\
+	-i logdev: do integrity testing, logdev is the dm log writes device\n\
 	-l flen: the upper bound on file size (default 262144)\n\
 	-m startop:endop: monitor (print debug output) specified byte range (default 0:infinity)\n\
 	-n: no verifications of file size\n\
@@ -1767,14 +1845,14 @@ int
 main(int argc, char **argv)
 {
 	int	i, style, ch;
-	char	*endp;
+	char	*endp, *tmp;
 	char goodfile[1024];
 	char logfile[1024];
-	int dirpath = 0;
 	struct stat statbuf;
 
 	goodfile[0] = 0;
 	logfile[0] = 0;
+	dname[0] = 0;
 
 	page_size = getpagesize();
 	page_mask = page_size - 1;
@@ -1784,7 +1862,7 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "b:c:dfl:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
+				 "b:c:dfi:l:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
@@ -1811,6 +1889,14 @@ main(int argc, char **argv)
 		case 'f':
 			flush = 1;
 			break;
+		case 'i':
+			integrity = 1;
+			logdev = strdup(optarg);
+			if (!logdev) {
+				prterr("strdup");
+				exit(101);
+			}
+			break;
 		case 'l':
 			maxfilelen = getnum(optarg, &endp);
 			if (maxfilelen <= 0)
@@ -1908,13 +1994,13 @@ main(int argc, char **argv)
 			randomoplen = 0;
 			break;
 		case 'P':
-			strncpy(goodfile, optarg, sizeof(goodfile));
-			strcat(goodfile, "/");
-			strncpy(logfile, optarg, sizeof(logfile));
-			strcat(logfile, "/");
-			strncpy(opsfile, optarg, sizeof(logfile));
-			strcat(opsfile, "/");
-			dirpath = 1;
+			strncpy(dname, optarg, sizeof(dname));
+			strcat(dname, "/");
+			dirpath = strlen(dname);
+
+			strncpy(goodfile, dname, sizeof(goodfile));
+			strncpy(logfile, dname, sizeof(logfile));
+			strncpy(opsfile, dname, sizeof(logfile));
 			break;
                 case 'R':
                         mapped_reads = 0;
@@ -1949,7 +2035,19 @@ main(int argc, char **argv)
 	argv += optind;
 	if (argc != 1)
 		usage();
+
+	if (integrity && !dirpath) {
+		fprintf(stderr, "option -i <logdev> requires -P <dirpath>\n");
+		usage();
+	}
+
 	fname = argv[0];
+	tmp = strdup(fname);
+	if (!tmp) {
+		prterr("strdup");
+		exit(101);
+	}
+	bname = basename(tmp);
 
 	signal(SIGHUP,	cleanup);
 	signal(SIGINT,	cleanup);
@@ -1991,21 +2089,21 @@ main(int argc, char **argv)
 		}
 	}
 #endif
-	strncat(goodfile, dirpath ? basename(fname) : fname, 256);
+	strncat(goodfile, dirpath ? bname : fname, 256);
 	strcat (goodfile, ".fsxgood");
 	fsxgoodfd = open(goodfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
 	if (fsxgoodfd < 0) {
 		prterr(goodfile);
 		exit(92);
 	}
-	strncat(logfile, dirpath ? basename(fname) : fname, 256);
+	strncat(logfile, dirpath ? bname : fname, 256);
 	strcat (logfile, ".fsxlog");
 	fsxlogf = fopen(logfile, "w");
 	if (fsxlogf == NULL) {
 		prterr(logfile);
 		exit(93);
 	}
-	strncat(opsfile, dirpath ? basename(fname) : fname, 256);
+	strncat(opsfile, dirpath ? bname : fname, 256);
 	strcat(opsfile, ".fsxops");
 	unlink(opsfile);
 
@@ -2081,6 +2179,7 @@ main(int argc, char **argv)
 		if (!test())
 			break;
 
+	free(tmp);
 	if (close(fd)) {
 		prterr("close");
 		report_failure(99);
-- 
2.7.4

[PATCH v4 2/5] fsx: add optional logid prefix to log messages

[Amir Goldstein]

When redirecting the intermixed output of several fsx processes
to a single output file, it is usefull to prefix debug log messages
with a log id. Use fsx -j <logid> to define the log messages prefix.

Fix implementation of prt() function to avoid using a temp buffer
and convert some more printf() calls to use ptr() instead.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
---
 ltp/fsx.c | 56 +++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/ltp/fsx.c b/ltp/fsx.c
index ac466ee..4e733f9 100644
--- a/ltp/fsx.c
+++ b/ltp/fsx.c
@@ -119,6 +119,7 @@ char	*temp_buf;			/* a pointer to the current data */
 char	*fname;				/* name of our test file */
 char	*bname;				/* basename of our test file */
 char	*logdev;			/* -i flag */
+char	*logid;				/* -j flag */
 char	dname[1024];			/* -P flag */
 int	dirpath = 0;			/* -P flag */
 int	fd;				/* fd for our test file */
@@ -195,13 +196,16 @@ static void *round_ptr_up(void *ptr, unsigned long align, unsigned long offset)
 }
 
 void
-vwarnc(int code, const char *fmt, va_list ap) {
-  fprintf(stderr, "fsx: ");
-  if (fmt != NULL) {
-	vfprintf(stderr, fmt, ap);
-	fprintf(stderr, ": ");
-  }
-  fprintf(stderr, "%s\n", strerror(code));
+vwarnc(int code, const char *fmt, va_list ap)
+{
+	if (logid)
+		fprintf(stderr, "%s: ", logid);
+	fprintf(stderr, "fsx: ");
+	if (fmt != NULL) {
+		vfprintf(stderr, fmt, ap);
+		fprintf(stderr, ": ");
+	}
+	fprintf(stderr, "%s\n", strerror(code));
 }
 
 void
@@ -212,20 +216,21 @@ warn(const char * fmt, ...)  {
 	va_end(ap);
 }
 
-#define BUF_SIZE 1024
-
 void
 prt(const char *fmt, ...)
 {
 	va_list args;
-	char buffer[BUF_SIZE];
 
+	if (logid)
+		fprintf(stdout, "%s: ", logid);
 	va_start(args, fmt);
-	vsnprintf(buffer, BUF_SIZE, fmt, args);
+	vfprintf(stdout, fmt, args);
 	va_end(args);
-	fprintf(stdout, "%s", buffer);
-	if (fsxlogf)
-		fprintf(fsxlogf, "%s", buffer);
+	if (fsxlogf) {
+		va_start(args, fmt);
+		vfprintf(fsxlogf, fmt, args);
+		va_end(args);
+	}
 }
 
 void
@@ -1624,12 +1629,13 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-i logdev] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
+		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-i logdev] [-j logid] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\
 	-f flush and invalidate cache after I/O\n\
 	-i logdev: do integrity testing, logdev is the dm log writes device\n\
+	-j logid: prefix debug log messsages with this id\n\
 	-l flen: the upper bound on file size (default 262144)\n\
 	-m startop:endop: monitor (print debug output) specified byte range (default 0:infinity)\n\
 	-n: no verifications of file size\n\
@@ -1862,14 +1868,13 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "b:c:dfi:l:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
+				 "b:c:dfi:j:l:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
 			simulatedopcount = getnum(optarg, &endp);
 			if (!quiet)
-				fprintf(stdout, "Will begin at operation %ld\n",
-					simulatedopcount);
+				prt("Will begin at operation %ld\n", simulatedopcount);
 			if (simulatedopcount == 0)
 				usage();
 			simulatedopcount -= 1;
@@ -1877,9 +1882,7 @@ main(int argc, char **argv)
 		case 'c':
 			closeprob = getnum(optarg, &endp);
 			if (!quiet)
-				fprintf(stdout,
-					"Chance of close/open is 1 in %d\n",
-					closeprob);
+				prt("Chance of close/open is 1 in %d\n", closeprob);
 			if (closeprob <= 0)
 				usage();
 			break;
@@ -1897,6 +1900,13 @@ main(int argc, char **argv)
 				exit(101);
 			}
 			break;
+		case 'j':
+			logid = strdup(optarg);
+			if (!logid) {
+				prterr("strdup");
+				exit(101);
+			}
+			break;
 		case 'l':
 			maxfilelen = getnum(optarg, &endp);
 			if (maxfilelen <= 0)
@@ -2012,14 +2022,14 @@ main(int argc, char **argv)
 				seed += (int)getpid();
 			}
 			if (!quiet)
-				fprintf(stdout, "Seed set to %d\n", seed);
+				prt("Seed set to %d\n", seed);
 			if (seed < 0)
 				usage();
 			break;
 		case 'W':
 		        mapped_writes = 0;
 			if (!quiet)
-				fprintf(stdout, "mapped writes DISABLED\n");
+				prt("mapped writes DISABLED\n");
 			break;
 		case 'Z':
 			o_direct = O_DIRECT;
-- 
2.7.4

[PATCH v4 3/5] fsx: add support for recording operations to a file

[Amir Goldstein]

Usually, fsx dumps an .fsxops file on failure with same basename
as work file and possibly under dirctory specified by -P dirpath.

The --record-ops[=opsfile] flag can be use to dump ops file also
on success and to optionally specify the ops file name.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
---
 ltp/fsx.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/ltp/fsx.c b/ltp/fsx.c
index 4e733f9..67d76dc 100644
--- a/ltp/fsx.c
+++ b/ltp/fsx.c
@@ -180,6 +180,7 @@ int aio_rw(int rw, int fd, char *buf, unsigned len, unsigned offset);
 #endif
 
 const char *replayops = NULL;
+const char *recordops = NULL;
 FILE *	fsxlogf = NULL;
 FILE *	replayopsf = NULL;
 char opsfile[1024];
@@ -1676,6 +1677,8 @@ usage(void)
 	-W: mapped write operations DISabled\n\
         -R: read() system calls only (mapped reads disabled)\n\
         -Z: O_DIRECT (use -R, -W, -r and -w too)\n\
+	--replay-ops opsfile: replay ops from recorded .fsxops file\n\
+	--record-ops[=opsfile]: dump ops file also on success. optionally specify ops file name\n\
 	fname: this filename is REQUIRED (no default)\n");
 	exit(90);
 }
@@ -1844,6 +1847,7 @@ __test_fallocate(int mode, const char *mode_str)
 
 static struct option longopts[] = {
 	{"replay-ops", required_argument, 0, 256},
+	{"record-ops", optional_argument, 0, 255},
 	{ }
 };
 
@@ -2034,6 +2038,11 @@ main(int argc, char **argv)
 		case 'Z':
 			o_direct = O_DIRECT;
 			break;
+		case 255:  /* --record-ops */
+			if (optarg)
+				strncpy(opsfile, optarg, sizeof(opsfile));
+			recordops = opsfile;
+			break;
 		case 256:  /* --replay-ops */
 			replayops = optarg;
 			break;
@@ -2113,8 +2122,10 @@ main(int argc, char **argv)
 		prterr(logfile);
 		exit(93);
 	}
-	strncat(opsfile, dirpath ? bname : fname, 256);
-	strcat(opsfile, ".fsxops");
+	if (!*opsfile) {
+		strncat(opsfile, dirpath ? bname : fname, 256);
+		strcat(opsfile, ".fsxops");
+	}
 	unlink(opsfile);
 
 	if (replayops) {
@@ -2195,6 +2206,8 @@ main(int argc, char **argv)
 		report_failure(99);
 	}
 	prt("All %lu operations completed A-OK!\n", testcalls);
+	if (recordops)
+		logdump();
 
 	exit(0);
 	return 0;
-- 
2.7.4

[PATCH v4 4/5] fsx: add support for writing constant instead of random data

[Amir Goldstein]

-g X: write character X instead of random generated data

This is useful to compare holes between good and bad files
because hexdump of good and bad files compacts the contigious
ranges of X and zeroes.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
---
 ltp/fsx.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/ltp/fsx.c b/ltp/fsx.c
index 67d76dc..fd00962 100644
--- a/ltp/fsx.c
+++ b/ltp/fsx.c
@@ -133,6 +133,7 @@ unsigned long	simulatedopcount = 0;	/* -b flag */
 int	closeprob = 0;			/* -c flag */
 int	debug = 0;			/* -d flag */
 unsigned long	debugstart = 0;		/* -D flag */
+char	filldata = 0;			/* -g flag */
 int	flush = 0;			/* -f flag */
 int	do_fsync = 0;			/* -y flag */
 unsigned long	maxfilelen = 256 * 1024;	/* -l flag */
@@ -813,9 +814,13 @@ void
 gendata(char *original_buf, char *good_buf, unsigned offset, unsigned size)
 {
 	while (size--) {
-		good_buf[offset] = testcalls % 256; 
-		if (offset % 2)
-			good_buf[offset] += original_buf[offset];
+		if (filldata) {
+			good_buf[offset] = filldata;
+		} else {
+			good_buf[offset] = testcalls % 256;
+			if (offset % 2)
+				good_buf[offset] += original_buf[offset];
+		}
 		offset++;
 	}
 }
@@ -1630,11 +1635,12 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-i logdev] [-j logid] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
+		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\
 	-f flush and invalidate cache after I/O\n\
+	-g X: write character X instead of random generated data\n\
 	-i logdev: do integrity testing, logdev is the dm log writes device\n\
 	-j logid: prefix debug log messsages with this id\n\
 	-l flen: the upper bound on file size (default 262144)\n\
@@ -1872,7 +1878,7 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "b:c:dfi:j:l:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
+				 "b:c:dfg:i:j:l:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
@@ -1896,6 +1902,9 @@ main(int argc, char **argv)
 		case 'f':
 			flush = 1;
 			break;
+		case 'g':
+			filldata = *optarg;
+			break;
 		case 'i':
 			integrity = 1;
 			logdev = strdup(optarg);
-- 
2.7.4

[PATCH v4 5/5] fsx: add support for keeping existing file

[Amir Goldstein]

With fsx -k, do not truncate existing file and use its size as upper
bound on file size.

This is needed to prevent fsx from truncating the file on start of
test when testing fsx on cloned files.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
---
 ltp/fsx.c | 49 ++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/ltp/fsx.c b/ltp/fsx.c
index fd00962..5cd599d 100644
--- a/ltp/fsx.c
+++ b/ltp/fsx.c
@@ -628,17 +628,18 @@ void
 check_trunc_hack(void)
 {
 	struct stat statbuf;
+	off_t offset = file_size + (off_t)100000;
 
-	if (ftruncate(fd, (off_t)0))
+	if (ftruncate(fd, file_size))
 		goto ftruncate_err;
-	if (ftruncate(fd, (off_t)100000))
+	if (ftruncate(fd, offset))
 		goto ftruncate_err;
 	fstat(fd, &statbuf);
-	if (statbuf.st_size != (off_t)100000) {
+	if (statbuf.st_size != offset) {
 		prt("no extend on truncate! not posix!\n");
 		exit(130);
 	}
-	if (ftruncate(fd, 0)) {
+	if (ftruncate(fd, file_size)) {
 ftruncate_err:
 		prterr("check_trunc_hack: ftruncate");
 		exit(131);
@@ -1635,7 +1636,7 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dnqxAFLOWZ] [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
+		"fsx [-dknqxAFLOWZ] [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
 	-b opnum: beginning operation number (default 1)\n\
 	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
 	-d: debug output for all operations\n\
@@ -1643,6 +1644,7 @@ usage(void)
 	-g X: write character X instead of random generated data\n\
 	-i logdev: do integrity testing, logdev is the dm log writes device\n\
 	-j logid: prefix debug log messsages with this id\n\
+	-k: do not truncate existing file and use its size as upper bound on file size\n\
 	-l flen: the upper bound on file size (default 262144)\n\
 	-m startop:endop: monitor (print debug output) specified byte range (default 0:infinity)\n\
 	-n: no verifications of file size\n\
@@ -1833,7 +1835,7 @@ __test_fallocate(int mode, const char *mode_str)
 #ifdef HAVE_LINUX_FALLOC_H
 	int ret = 0;
 	if (!lite) {
-		if (fallocate(fd, mode, 0, 1) && errno == EOPNOTSUPP) {
+		if (fallocate(fd, mode, file_size, 1) && errno == EOPNOTSUPP) {
 			if(!quiet)
 				fprintf(stderr,
 					"main: filesystem does not support "
@@ -1841,7 +1843,7 @@ __test_fallocate(int mode, const char *mode_str)
 					mode_str);
 		} else {
 			ret = 1;
-			if (ftruncate(fd, 0)) {
+			if (ftruncate(fd, file_size)) {
 				warn("main: ftruncate");
 				exit(132);
 			}
@@ -1865,6 +1867,7 @@ main(int argc, char **argv)
 	char goodfile[1024];
 	char logfile[1024];
 	struct stat statbuf;
+	int o_flags = O_RDWR|O_CREAT|O_TRUNC;
 
 	goodfile[0] = 0;
 	logfile[0] = 0;
@@ -1878,7 +1881,7 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "b:c:dfg:i:j:l:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
+				 "b:c:dfg:i:j:kl:m:no:p:qr:s:t:w:xyAD:FKHzCILN:OP:RS:WZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
@@ -1920,6 +1923,9 @@ main(int argc, char **argv)
 				exit(101);
 			}
 			break;
+		case 'k':
+			o_flags &= ~O_TRUNC;
+			break;
 		case 'l':
 			maxfilelen = getnum(optarg, &endp);
 			if (maxfilelen <= 0)
@@ -2007,6 +2013,7 @@ main(int argc, char **argv)
 			break;
 		case 'L':
 		        lite = 1;
+			o_flags &= ~(O_CREAT|O_TRUNC);
 			break;
 		case 'N':
 			numops = getnum(optarg, &endp);
@@ -2046,6 +2053,7 @@ main(int argc, char **argv)
 			break;
 		case 'Z':
 			o_direct = O_DIRECT;
+			o_flags |= O_DIRECT;
 			break;
 		case 255:  /* --record-ops */
 			if (optarg)
@@ -2089,8 +2097,7 @@ main(int argc, char **argv)
 	signal(SIGUSR2,	cleanup);
 
 	srandom(seed);
-	fd = open(fname,
-		O_RDWR|(lite ? 0 : O_CREAT|O_TRUNC)|o_direct, 0666);
+	fd = open(fname, o_flags, 0666);
 	if (fd < 0) {
 		prterr(fname);
 		exit(91);
@@ -2150,9 +2157,10 @@ main(int argc, char **argv)
 		aio_setup();
 #endif
 
-	if (lite) {
+	if (!(o_flags & O_TRUNC)) {
 		off_t ret;
-		file_size = maxfilelen = lseek(fd, (off_t)0, SEEK_END);
+		file_size = maxfilelen = biggest = lseek(fd, (off_t)0, SEEK_END);
+		prt("file_size is %lu\n", file_size);
 		if (file_size == (off_t)-1) {
 			prterr(fname);
 			warn("main: lseek eof");
@@ -2189,8 +2197,23 @@ main(int argc, char **argv)
 					maxfilelen);
 			exit(98);
 		}
-	} else 
+	} else {
+		ssize_t ret, len = file_size;
+		off_t off = 0;
+
+		while (len > 0) {
+			ret = read(fd, good_buf + off, len);
+			if (ret == -1) {
+				prterr(fname);
+				warn("main: error on read");
+				exit(98);
+			}
+			len -= ret;
+			off += ret;
+		}
+
 		check_trunc_hack();
+	}
 
 	if (fallocate_calls)
 		fallocate_calls = test_fallocate(0);
-- 
2.7.4

Re: [PATCH v4 0/5] fsx additions for crash consistency tests

[Eryu Guan]

On Thu, Sep 07, 2017 at 10:26:33AM +0300, Amir Goldstein wrote:
> Eryu,
> 
> This is a re-post of v3 fsx patches with a correction to
> the 2nd patch which introduces -j <logid>.

Thanks a lot for the repost! I plan to pull this log-writes work in this
week after my minor nits to the previous patchset are addressed, if
there're no more review comments/complains from other reviewers.

Thanks,
Eryu

> 
> Amir.
> 
> Amir Goldstein (5):
>   fsx: add support for integrity check with dm-log-writes target
>   fsx: add optional logid prefix to log messages
>   fsx: add support for recording operations to a file
>   fsx: add support for writing constant instead of random data
>   fsx: add support for keeping existing file
> 
>  ltp/fsx.c | 274 ++++++++++++++++++++++++++++++++++++++++++++++++--------------
>  1 file changed, 214 insertions(+), 60 deletions(-)
> 
> -- 
> 2.7.4
> 

Re: [PATCH v4 5/5] fsx: add support for keeping existing file

[Amir Goldstein]

On Thu, Sep 7, 2017 at 10:26 AM, Amir Goldstein <amir73il@gmail.com> wrote:
> With fsx -k, do not truncate existing file and use its size as upper
> bound on file size.
>
> This is needed to prevent fsx from truncating the file on start of
> test when testing fsx on cloned files.
>
> Signed-off-by: Amir Goldstein <amir73il@gmail.com>
> ---

> -       if (lite) {
> +       if (!(o_flags & O_TRUNC)) {
>                 off_t ret;
> -               file_size = maxfilelen = lseek(fd, (off_t)0, SEEK_END);
> +               file_size = maxfilelen = biggest = lseek(fd, (off_t)0, SEEK_END);
> +               prt("file_size is %lu\n", file_size);

Eryu,

Please remove this added debug log before merging.
It breaks expected output of test generic/127

Amir.

Re: [PATCH v4 5/5] fsx: add support for keeping existing file

[Eryu Guan]

On Thu, Sep 07, 2017 at 03:38:59PM +0300, Amir Goldstein wrote:
> On Thu, Sep 7, 2017 at 10:26 AM, Amir Goldstein <amir73il@gmail.com> wrote:
> > With fsx -k, do not truncate existing file and use its size as upper
> > bound on file size.
> >
> > This is needed to prevent fsx from truncating the file on start of
> > test when testing fsx on cloned files.
> >
> > Signed-off-by: Amir Goldstein <amir73il@gmail.com>
> > ---
> 
> > -       if (lite) {
> > +       if (!(o_flags & O_TRUNC)) {
> >                 off_t ret;
> > -               file_size = maxfilelen = lseek(fd, (off_t)0, SEEK_END);
> > +               file_size = maxfilelen = biggest = lseek(fd, (off_t)0, SEEK_END);
> > +               prt("file_size is %lu\n", file_size);
> 
> Eryu,
> 
> Please remove this added debug log before merging.
> It breaks expected output of test generic/127

Sure, thanks for the heads-up.

Eryu