[Cluster-devel] [PATCH 00/13] gfs2_edit restoremeta: Add bzip2 support

[Cluster-devel] [PATCH 00/13] gfs2_edit restoremeta: Add bzip2 support

[Andrew Price]

This patch set allows gfs2_edit restoremeta to read gfs2 metadata dumps that were compressed with bzip2.

In doing so, it speeds up reading of metadata files by around 45% based on this test:

  $ time while read f; do gfs2_edit printsavedmeta $f &>/dev/null; done < metadata_files_for_comparison

Andrew Price (13):
  restoremeta: Use zlib by file descriptor
  restoremeta: Abstract out metadata file opening
  restoremeta: Use metafd instead of gzFile for parameters
  restoremeta: Abstract out decompression operations
  restoremeta: Combine restore_init() and open_metadata()
  restoremeta: Don't seek in restore_header()
  savemeta: Remove anthropomorphize()
  restoremeta: Remove find_highest_block()
  restoremeta: Metadata file reading overhaul
  restoremeta: Convert iseof function to a flag
  restoremeta: Combine parse_header() and check_header()
  restoremeta: Add bzip2 support
  restoremeta: Skip the right number of bytes for the superblock

 configure.ac          |   1 +
 gfs2/edit/Makefile.am |   2 +
 gfs2/edit/savemeta.c  | 413 ++++++++++++++++++++++++------------------
 tests/edit.at         |   2 +-
 4 files changed, 238 insertions(+), 180 deletions(-)

-- 
2.24.1

[Cluster-devel] [PATCH 01/13] restoremeta: Use zlib by file descriptor

[Andrew Price]

Open the metadata file with open and pass the fd to zlib instead of the
path. This is groundwork for supporting bzip2. (gzclose() still closes
the fd later).

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index 18432b7c..9de53a97 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -1197,6 +1197,7 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	gzFile gzfd;
 	off_t pos = 0;
 	struct savemeta_header smh = {0};
+	int fd;
 
 	termlines = 0;
 	if (!in_fn)
@@ -1204,7 +1205,12 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	if (!printonly && !out_device)
 		complain("No destination file system specified.");
 
-	gzfd = gzopen(in_fn, "rb");
+	fd = open(in_fn, O_RDONLY|O_CLOEXEC);
+	if (fd < 0) {
+		perror("Could not open file");
+		exit(1);
+	}
+	gzfd = gzdopen(fd, "rb");
 	if (!gzfd)
 		die("Can't open source file %s: %s\n",
 		    in_fn, strerror(errno));
-- 
2.24.1

[Cluster-devel] [PATCH 02/13] restoremeta: Abstract out metadata file opening

[Andrew Price]

Provide wrapper functions for the operations which make use of zlib
functions, to make it simpler to provide ones for other compression
methods later.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 46 +++++++++++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 16 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index 9de53a97..a20d7aa0 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -1191,13 +1191,33 @@ static void complain(const char *complaint)
 	    "<dest file system>\n");
 }
 
+static int open_metadata(const char *path, struct metafd *mfd)
+{
+	mfd->filename = path;
+	mfd->fd = open(path, O_RDONLY|O_CLOEXEC);
+	if (mfd->fd < 0) {
+		perror("Could not open metadata file");
+		return 1;
+	}
+	mfd->gzfd = gzdopen(mfd->fd, "rb");
+	if (!mfd->gzfd) {
+		perror("gzdopen");
+		return 1;
+	}
+	return 0;
+}
+
+static void close_metadata(struct metafd *mfd)
+{
+	gzclose(mfd->gzfd);
+}
+
 void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 {
 	int error;
-	gzFile gzfd;
 	off_t pos = 0;
 	struct savemeta_header smh = {0};
-	int fd;
+	struct metafd mfd = {0};
 
 	termlines = 0;
 	if (!in_fn)
@@ -1205,15 +1225,9 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	if (!printonly && !out_device)
 		complain("No destination file system specified.");
 
-	fd = open(in_fn, O_RDONLY|O_CLOEXEC);
-	if (fd < 0) {
-		perror("Could not open file");
-		exit(1);
-	}
-	gzfd = gzdopen(fd, "rb");
-	if (!gzfd)
-		die("Can't open source file %s: %s\n",
-		    in_fn, strerror(errno));
+	error = open_metadata(in_fn, &mfd);
+	if (error != 0)
+		exit(error);
 
 	if (!printonly) {
 		sbd.device_fd = open(out_device, O_RDWR);
@@ -1224,8 +1238,8 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 				  optional block no */
 		printonly = check_keywords(out_device);
 
-	pos = restore_init(gzfd, &smh);
-	error = restore_super(gzfd, pos);
+	pos = restore_init(mfd.gzfd, &smh);
+	error = restore_super(mfd.gzfd, pos);
 	if (error)
 		exit(1);
 
@@ -1236,16 +1250,16 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 		printf("There are %"PRIu64" free blocks on the destination device.\n", space);
 	}
 
-	error = find_highest_block(gzfd, pos, sbd.fssize);
+	error = find_highest_block(mfd.gzfd, pos, sbd.fssize);
 	if (error)
 		exit(1);
 
-	error = restore_data(sbd.device_fd, gzfd, pos, printonly);
+	error = restore_data(sbd.device_fd, mfd.gzfd, pos, printonly);
 	printf("File %s %s %s.\n", in_fn,
 	       (printonly ? "print" : "restore"),
 	       (error ? "error" : "successful"));
 
-	gzclose(gzfd);
+	close_metadata(&mfd);
 	if (!printonly)
 		close(sbd.device_fd);
 	free(indirect);
-- 
2.24.1

[Cluster-devel] [PATCH 03/13] restoremeta: Use metafd instead of gzFile for parameters

[Andrew Price]

More decoupling of the restoremeta code from zlib functions and types.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 50 ++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index a20d7aa0..82d43d8f 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -848,13 +848,13 @@ static int save_header(struct metafd *mfd, uint64_t fsbytes)
 	return 0;
 }
 
-static int read_header(gzFile gzin_fd, struct savemeta_header *smh)
+static int read_header(struct metafd *mfd, struct savemeta_header *smh)
 {
 	size_t rs;
 	struct savemeta_header smh_be = {0};
 
-	gzseek(gzin_fd, 0, SEEK_SET);
-	rs = gzread(gzin_fd, &smh_be, sizeof(smh_be));
+	gzseek(mfd->gzfd, 0, SEEK_SET);
+	rs = gzread(mfd->gzfd, &smh_be, sizeof(smh_be));
 	if (rs == -1) {
 		perror("Failed to read savemeta file header");
 		return -1;
@@ -964,7 +964,7 @@ void savemeta(char *out_fn, int saveoption, int gziplevel)
 	exit(0);
 }
 
-static off_t restore_init(gzFile gzfd, struct savemeta_header *smh)
+static off_t restore_init(struct metafd *mfd, struct savemeta_header *smh)
 {
 	int err;
 	unsigned i;
@@ -973,7 +973,7 @@ static off_t restore_init(gzFile gzfd, struct savemeta_header *smh)
 	off_t startpos = 0;
 	struct gfs2_meta_header sbmh;
 
-	err = read_header(gzfd, smh);
+	err = read_header(mfd, smh);
 	if (err < 0) {
 		exit(1);
 	} else if (check_header(smh) != 0) {
@@ -982,8 +982,8 @@ static off_t restore_init(gzFile gzfd, struct savemeta_header *smh)
 		startpos = sizeof(*smh);
 	}
 
-	gzseek(gzfd, startpos, SEEK_SET);
-	rs = gzread(gzfd, buf, sizeof(buf));
+	gzseek(mfd->gzfd, startpos, SEEK_SET);
+	rs = gzread(mfd->gzfd, buf, sizeof(buf));
 	if (rs != sizeof(buf)) {
 		fprintf(stderr, "Error: File is too small.\n");
 		exit(1);
@@ -1003,14 +1003,14 @@ static off_t restore_init(gzFile gzfd, struct savemeta_header *smh)
 }
 
 
-static int restore_block(gzFile gzfd, struct saved_metablock *svb, char *buf, uint16_t maxlen)
+static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *buf, uint16_t maxlen)
 {
 	int gzerr;
 	int ret;
 	uint16_t checklen;
 	const char *errstr;
 
-	ret = gzread(gzfd, svb, sizeof(*svb));
+	ret = gzread(mfd->gzfd, svb, sizeof(*svb));
 	if (ret < sizeof(*svb)) {
 		goto gzread_err;
 	}
@@ -1036,7 +1036,7 @@ static int restore_block(gzFile gzfd, struct saved_metablock *svb, char *buf, ui
 	}
 
 	if (buf != NULL && maxlen != 0) {
-		ret = gzread(gzfd, buf, svb->siglen);
+		ret = gzread(mfd->gzfd, buf, svb->siglen);
 		if (ret < svb->siglen) {
 			goto gzread_err;
 		}
@@ -1045,17 +1045,17 @@ static int restore_block(gzFile gzfd, struct saved_metablock *svb, char *buf, ui
 	return 0;
 
 gzread_err:
-	if (gzeof(gzfd))
+	if (gzeof(mfd->gzfd))
 		return 1;
 
-	errstr = gzerror(gzfd, &gzerr);
+	errstr = gzerror(mfd->gzfd, &gzerr);
 	if (gzerr == Z_ERRNO)
 		errstr = strerror(errno);
 	fprintf(stderr, "Failed to restore block: %s\n", errstr);
 	return -1;
 }
 
-static int restore_super(gzFile gzfd, off_t pos)
+static int restore_super(struct metafd *mfd, off_t pos)
 {
 	int ret;
 	struct saved_metablock svb = {0};
@@ -1066,8 +1066,8 @@ static int restore_super(gzFile gzfd, off_t pos)
 		perror("Failed to restore super block");
 		exit(1);
 	}
-	gzseek(gzfd, pos, SEEK_SET);
-	ret = restore_block(gzfd, &svb, buf, sizeof(struct gfs2_sb));
+	gzseek(mfd->gzfd, pos, SEEK_SET);
+	ret = restore_block(mfd, &svb, buf, sizeof(struct gfs2_sb));
 	if (ret == 1) {
 		fprintf(stderr, "Reached end of file while restoring superblock\n");
 		goto err;
@@ -1093,15 +1093,15 @@ err:
 	return -1;
 }
 
-static int find_highest_block(gzFile gzfd, off_t pos, uint64_t fssize)
+static int find_highest_block(struct metafd *mfd, off_t pos, uint64_t fssize)
 {
 	int err = 0;
 	uint64_t highest = 0;
 	struct saved_metablock svb = {0};
 
 	while (1) {
-		gzseek(gzfd, pos, SEEK_SET);
-		err = restore_block(gzfd, &svb, NULL, 0);
+		gzseek(mfd->gzfd, pos, SEEK_SET);
+		err = restore_block(mfd, &svb, NULL, 0);
 		if (err == 1)
 			break;
 		if (err != 0)
@@ -1124,7 +1124,7 @@ static int find_highest_block(gzFile gzfd, off_t pos, uint64_t fssize)
 	return 0;
 }
 
-static int restore_data(int fd, gzFile gzin_fd, off_t pos, int printonly)
+static int restore_data(int fd, struct metafd *mfd, off_t pos, int printonly)
 {
 	struct saved_metablock savedata = {0};
 	uint64_t writes = 0;
@@ -1136,11 +1136,11 @@ static int restore_data(int fd, gzFile gzin_fd, off_t pos, int printonly)
 		exit(1);
 	}
 
-	gzseek(gzin_fd, pos, SEEK_SET);
+	gzseek(mfd->gzfd, pos, SEEK_SET);
 	blks_saved = 0;
 	while (TRUE) {
 		int err;
-		err = restore_block(gzin_fd, &savedata, buf, sbd.bsize);
+		err = restore_block(mfd, &savedata, buf, sbd.bsize);
 		if (err == 1)
 			break;
 		if (err != 0) {
@@ -1238,8 +1238,8 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 				  optional block no */
 		printonly = check_keywords(out_device);
 
-	pos = restore_init(mfd.gzfd, &smh);
-	error = restore_super(mfd.gzfd, pos);
+	pos = restore_init(&mfd, &smh);
+	error = restore_super(&mfd, pos);
 	if (error)
 		exit(1);
 
@@ -1250,11 +1250,11 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 		printf("There are %"PRIu64" free blocks on the destination device.\n", space);
 	}
 
-	error = find_highest_block(mfd.gzfd, pos, sbd.fssize);
+	error = find_highest_block(&mfd, pos, sbd.fssize);
 	if (error)
 		exit(1);
 
-	error = restore_data(sbd.device_fd, mfd.gzfd, pos, printonly);
+	error = restore_data(sbd.device_fd, &mfd, pos, printonly);
 	printf("File %s %s %s.\n", in_fn,
 	       (printonly ? "print" : "restore"),
 	       (error ? "error" : "successful"));
-- 
2.24.1

[Cluster-devel] [PATCH 04/13] restoremeta: Abstract out decompression operations

[Andrew Price]

Wrap the gzip functions that restoremeta uses and call them via
function pointers, to provide the framework for supporting other
compression methods.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 76 +++++++++++++++++++++++++++++++-------------
 1 file changed, 54 insertions(+), 22 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index 82d43d8f..ad565c1f 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -54,8 +54,43 @@ struct metafd {
 	gzFile gzfd;
 	const char *filename;
 	int gziplevel;
+	int (*seek)(struct metafd *mfd, off_t off, int whence);
+	int (*read)(struct metafd *mfd, void *buf, unsigned len);
+	int (*iseof)(struct metafd *mfd);
+	void (*close)(struct metafd *mfd);
+	const char* (*strerr)(struct metafd *mfd);
 };
 
+static const char *gz_strerr(struct metafd *mfd)
+{
+	int err;
+	const char *errstr = gzerror(mfd->gzfd, &err);
+
+	if (err == Z_ERRNO)
+		return strerror(errno);
+	return errstr;
+}
+
+static int gz_seek(struct metafd *mfd, off_t off, int whence)
+{
+	return gzseek(mfd->gzfd, off, whence);
+}
+
+static int gz_read(struct metafd *mfd, void *buf, unsigned len)
+{
+	return gzread(mfd->gzfd, buf, len);
+}
+
+static int gz_iseof(struct metafd *mfd)
+{
+	return gzeof(mfd->gzfd);
+}
+
+static void gz_close(struct metafd *mfd)
+{
+	gzclose(mfd->gzfd);
+}
+
 static uint64_t blks_saved;
 static uint64_t journal_blocks[MAX_JOURNALS_SAVED];
 static uint64_t gfs1_journal_size = 0; /* in blocks */
@@ -853,8 +888,8 @@ static int read_header(struct metafd *mfd, struct savemeta_header *smh)
 	size_t rs;
 	struct savemeta_header smh_be = {0};
 
-	gzseek(mfd->gzfd, 0, SEEK_SET);
-	rs = gzread(mfd->gzfd, &smh_be, sizeof(smh_be));
+	mfd->seek(mfd, 0, SEEK_SET);
+	rs = mfd->read(mfd, &smh_be, sizeof(smh_be));
 	if (rs == -1) {
 		perror("Failed to read savemeta file header");
 		return -1;
@@ -982,8 +1017,8 @@ static off_t restore_init(struct metafd *mfd, struct savemeta_header *smh)
 		startpos = sizeof(*smh);
 	}
 
-	gzseek(mfd->gzfd, startpos, SEEK_SET);
-	rs = gzread(mfd->gzfd, buf, sizeof(buf));
+	mfd->seek(mfd, startpos, SEEK_SET);
+	rs = mfd->read(mfd, buf, sizeof(buf));
 	if (rs != sizeof(buf)) {
 		fprintf(stderr, "Error: File is too small.\n");
 		exit(1);
@@ -1005,14 +1040,13 @@ static off_t restore_init(struct metafd *mfd, struct savemeta_header *smh)
 
 static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *buf, uint16_t maxlen)
 {
-	int gzerr;
 	int ret;
 	uint16_t checklen;
 	const char *errstr;
 
-	ret = gzread(mfd->gzfd, svb, sizeof(*svb));
+	ret = mfd->read(mfd, svb, sizeof(*svb));
 	if (ret < sizeof(*svb)) {
-		goto gzread_err;
+		goto read_err;
 	}
 	svb->blk = be64_to_cpu(svb->blk);
 	svb->siglen = be16_to_cpu(svb->siglen);
@@ -1036,21 +1070,19 @@ static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *
 	}
 
 	if (buf != NULL && maxlen != 0) {
-		ret = gzread(mfd->gzfd, buf, svb->siglen);
+		ret = mfd->read(mfd, buf, svb->siglen);
 		if (ret < svb->siglen) {
-			goto gzread_err;
+			goto read_err;
 		}
 	}
 
 	return 0;
 
-gzread_err:
-	if (gzeof(mfd->gzfd))
+read_err:
+	if (mfd->iseof(mfd))
 		return 1;
 
-	errstr = gzerror(mfd->gzfd, &gzerr);
-	if (gzerr == Z_ERRNO)
-		errstr = strerror(errno);
+	errstr = mfd->strerr(mfd);
 	fprintf(stderr, "Failed to restore block: %s\n", errstr);
 	return -1;
 }
@@ -1066,7 +1098,7 @@ static int restore_super(struct metafd *mfd, off_t pos)
 		perror("Failed to restore super block");
 		exit(1);
 	}
-	gzseek(mfd->gzfd, pos, SEEK_SET);
+	mfd->seek(mfd, pos, SEEK_SET);
 	ret = restore_block(mfd, &svb, buf, sizeof(struct gfs2_sb));
 	if (ret == 1) {
 		fprintf(stderr, "Reached end of file while restoring superblock\n");
@@ -1100,7 +1132,7 @@ static int find_highest_block(struct metafd *mfd, off_t pos, uint64_t fssize)
 	struct saved_metablock svb = {0};
 
 	while (1) {
-		gzseek(mfd->gzfd, pos, SEEK_SET);
+		mfd->seek(mfd, pos, SEEK_SET);
 		err = restore_block(mfd, &svb, NULL, 0);
 		if (err == 1)
 			break;
@@ -1199,6 +1231,11 @@ static int open_metadata(const char *path, struct metafd *mfd)
 		perror("Could not open metadata file");
 		return 1;
 	}
+	mfd->seek = gz_seek;
+	mfd->read = gz_read;
+	mfd->iseof = gz_iseof;
+	mfd->close = gz_close;
+	mfd->strerr = gz_strerr;
 	mfd->gzfd = gzdopen(mfd->fd, "rb");
 	if (!mfd->gzfd) {
 		perror("gzdopen");
@@ -1207,11 +1244,6 @@ static int open_metadata(const char *path, struct metafd *mfd)
 	return 0;
 }
 
-static void close_metadata(struct metafd *mfd)
-{
-	gzclose(mfd->gzfd);
-}
-
 void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 {
 	int error;
@@ -1259,7 +1291,7 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	       (printonly ? "print" : "restore"),
 	       (error ? "error" : "successful"));
 
-	close_metadata(&mfd);
+	mfd.close(&mfd);
 	if (!printonly)
 		close(sbd.device_fd);
 	free(indirect);
-- 
2.24.1

[Cluster-devel] [PATCH 05/13] restoremeta: Combine restore_init() and open_metadata()

[Andrew Price]

Later we'll need to know which compression method to use for the
metadata file in order to initialise a valid metadata file descriptor,
and that requires reading an initial chunk. So it makes sense to combine
the open_metadata() function with restore_init() which reads the
preamble from the metadata file.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 77 ++++++++++++++++++++------------------------
 1 file changed, 35 insertions(+), 42 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index ad565c1f..f77c6bf7 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -999,45 +999,6 @@ void savemeta(char *out_fn, int saveoption, int gziplevel)
 	exit(0);
 }
 
-static off_t restore_init(struct metafd *mfd, struct savemeta_header *smh)
-{
-	int err;
-	unsigned i;
-	size_t rs;
-	char buf[256];
-	off_t startpos = 0;
-	struct gfs2_meta_header sbmh;
-
-	err = read_header(mfd, smh);
-	if (err < 0) {
-		exit(1);
-	} else if (check_header(smh) != 0) {
-		printf("No valid file header found. Falling back to old format...\n");
-	} else if (err == 0) {
-		startpos = sizeof(*smh);
-	}
-
-	mfd->seek(mfd, startpos, SEEK_SET);
-	rs = mfd->read(mfd, buf, sizeof(buf));
-	if (rs != sizeof(buf)) {
-		fprintf(stderr, "Error: File is too small.\n");
-		exit(1);
-	}
-	/* Scan for the beginning of the file body. Required to support old formats(?). */
-	for (i = 0; i < (256 - sizeof(struct saved_metablock) - sizeof(sbmh)); i++) {
-		off_t off = i + sizeof(struct saved_metablock);
-
-		memcpy(&sbmh, &buf[off], sizeof(sbmh));
-		if (sbmh.mh_magic == cpu_to_be32(GFS2_MAGIC) &&
-		     sbmh.mh_type == cpu_to_be32(GFS2_METATYPE_SB))
-			break;
-	}
-	if (i == (sizeof(buf) - sizeof(struct saved_metablock) - sizeof(sbmh)))
-		i = 0;
-	return startpos + i; /* File offset of saved sb */
-}
-
-
 static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *buf, uint16_t maxlen)
 {
 	int ret;
@@ -1223,8 +1184,15 @@ static void complain(const char *complaint)
 	    "<dest file system>\n");
 }
 
-static int open_metadata(const char *path, struct metafd *mfd)
+static int restore_init(const char *path, struct metafd *mfd, struct savemeta_header *smh, off_t *pos)
 {
+	struct gfs2_meta_header sbmh;
+	off_t startpos = 0;
+	char buf[256];
+	unsigned i;
+	size_t rs;
+	int err;
+
 	mfd->filename = path;
 	mfd->fd = open(path, O_RDONLY|O_CLOEXEC);
 	if (mfd->fd < 0) {
@@ -1241,6 +1209,32 @@ static int open_metadata(const char *path, struct metafd *mfd)
 		perror("gzdopen");
 		return 1;
 	}
+	err = read_header(mfd, smh);
+	if (err < 0) {
+		return 1;
+	} else if (check_header(smh) != 0) {
+		printf("No valid file header found. Falling back to old format...\n");
+	} else if (err == 0) {
+		startpos = sizeof(*smh);
+	}
+	mfd->seek(mfd, startpos, SEEK_SET);
+	rs = mfd->read(mfd, buf, sizeof(buf));
+	if (rs != sizeof(buf)) {
+		fprintf(stderr, "Error: File is too small.\n");
+		return 1;
+	}
+	/* Scan for the beginning of the file body. Required to support old formats(?). */
+	for (i = 0; i < (256 - sizeof(struct saved_metablock) - sizeof(sbmh)); i++) {
+		off_t off = i + sizeof(struct saved_metablock);
+
+		memcpy(&sbmh, &buf[off], sizeof(sbmh));
+		if (sbmh.mh_magic == cpu_to_be32(GFS2_MAGIC) &&
+		     sbmh.mh_type == cpu_to_be32(GFS2_METATYPE_SB))
+			break;
+	}
+	if (i == (sizeof(buf) - sizeof(struct saved_metablock) - sizeof(sbmh)))
+		i = 0;
+	*pos = startpos + i; /* File offset of saved sb */
 	return 0;
 }
 
@@ -1257,7 +1251,7 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	if (!printonly && !out_device)
 		complain("No destination file system specified.");
 
-	error = open_metadata(in_fn, &mfd);
+	error = restore_init(in_fn, &mfd, &smh, &pos);
 	if (error != 0)
 		exit(error);
 
@@ -1270,7 +1264,6 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 				  optional block no */
 		printonly = check_keywords(out_device);
 
-	pos = restore_init(&mfd, &smh);
 	error = restore_super(&mfd, pos);
 	if (error)
 		exit(1);
-- 
2.24.1

[Cluster-devel] [PATCH 06/13] restoremeta: Don't seek in restore_header()

[Andrew Price]

This is the first time we read the file so there's no need to seek to
offset 0. The eventual goal here is to have restoremeta not seek at all
as (de)compression libraries do not support it, or emulate it slowly.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index f77c6bf7..ad0c537d 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -888,7 +888,6 @@ static int read_header(struct metafd *mfd, struct savemeta_header *smh)
 	size_t rs;
 	struct savemeta_header smh_be = {0};
 
-	mfd->seek(mfd, 0, SEEK_SET);
 	rs = mfd->read(mfd, &smh_be, sizeof(smh_be));
 	if (rs == -1) {
 		perror("Failed to read savemeta file header");
-- 
2.24.1

[Cluster-devel] [PATCH 07/13] savemeta: Remove anthropomorphize()

[Andrew Price]

This function doesn't work properly (e.g. converts 5GB in bytes to
4.1023MB) and we don't really need it as we can just report sizes in GB
with a simpler conversion instead.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 27 ++++-----------------------
 tests/edit.at        |  2 +-
 2 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index ad0c537d..f73cb0e0 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -206,25 +206,6 @@ static int block_is_systemfile(uint64_t blk)
 		block_is_per_node(blk) || block_is_in_per_node(blk);
 }
 
-/**
- * anthropomorphize - make a uint64_t number more human
- */
-static const char *anthropomorphize(unsigned long long inhuman_value)
-{
-	const char *symbols = " KMGTPE";
-	int i;
-	unsigned long long val = inhuman_value, remainder = 0;
-	static char out_val[32];
-
-	memset(out_val, 0, sizeof(out_val));
-	for (i = 0; i < 6 && val > 1024; i++) {
-		remainder = val % 1024;
-		val /= 1024;
-	}
-	sprintf(out_val, "%llu.%llu%cB", val, remainder, symbols[i]);
-	return out_val;
-}
-
 static size_t di_save_len(struct gfs2_buffer_head *bh, uint64_t owner)
 {
 	struct gfs2_inode *inode;
@@ -909,7 +890,7 @@ static int check_header(struct savemeta_header *smh)
 	if (smh->sh_magic != SAVEMETA_MAGIC || smh->sh_format > SAVEMETA_FORMAT)
 		return -1;
 	printf("Metadata saved at %s", ctime((time_t *)&smh->sh_time)); /* ctime() adds \n */
-	printf("File system size %s\n", anthropomorphize(smh->sh_fs_bytes));
+	printf("File system size %.2fGB\n", smh->sh_fs_bytes / ((float)(1 << 30)));
 	return 0;
 }
 
@@ -940,7 +921,7 @@ void savemeta(char *out_fn, int saveoption, int gziplevel)
 		do_dinode_extended(&di, lbh);
 	brelse(lbh);
 
-	printf("Filesystem size: %s\n", anthropomorphize(sbd.fssize * sbd.bsize));
+	printf("Filesystem size: %.2fGB\n", (sbd.fssize * sbd.bsize) / ((float)(1 << 30)));
 	get_journal_inode_blocks();
 
 	err = init_per_node_lookup();
@@ -1105,8 +1086,8 @@ static int find_highest_block(struct metafd *mfd, off_t pos, uint64_t fssize)
 	}
 
 	if (fssize > 0) {
-		printf("Saved file system size is %"PRIu64" (0x%"PRIx64") blocks, %s\n",
-		       fssize, fssize, anthropomorphize(fssize * sbd.bsize));
+		printf("Saved file system size is %"PRIu64" blocks, %.2fGB\n",
+		       fssize, (fssize * sbd.bsize) / ((float)(1 << 30)));
 		sbd.fssize = fssize;
 	} else {
 		sbd.fssize = highest + 1;
diff --git a/tests/edit.at b/tests/edit.at
index e1a0fca7..c9c341b9 100644
--- a/tests/edit.at
+++ b/tests/edit.at
@@ -7,7 +7,7 @@ GFS_TGT_REGEN
 AT_CHECK([$GFS_MKFS -p lock_nolock $GFS_TGT $(($(gfs_max_blocks 4096)/2))], 0, [ignore], [ignore])
 AT_CHECK([gfs2_edit savemeta $GFS_TGT test.meta > savemeta.log], 0, [ignore], [ignore])
 AT_CHECK([head -2 savemeta.log], 0, [There are 1310716 blocks of 4096 bytes in the filesystem.
-Filesystem size: 4.1023GB
+Filesystem size: 5.00GB
 ], [ignore])
 GFS_TGT_REGEN
 AT_CHECK([gfs2_edit restoremeta test.meta $GFS_TGT], 0, [ignore], [ignore])
-- 
2.24.1

[Cluster-devel] [PATCH 08/13] restoremeta: Remove find_highest_block()

[Andrew Price]

Doing a first pass through the metadata file just to find the highest
saved block and report it is a bit excessive, so remove
find_highest_block() and simply use the value from the metadata file
header if present. For ancient files that don't have the header, we just
don't report the size, which is a reasonable tradeoff as old files are
really only encountered during testing.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 41 ++++++-----------------------------------
 1 file changed, 6 insertions(+), 35 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index f73cb0e0..dd203974 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -1066,37 +1066,6 @@ err:
 	return -1;
 }
 
-static int find_highest_block(struct metafd *mfd, off_t pos, uint64_t fssize)
-{
-	int err = 0;
-	uint64_t highest = 0;
-	struct saved_metablock svb = {0};
-
-	while (1) {
-		mfd->seek(mfd, pos, SEEK_SET);
-		err = restore_block(mfd, &svb, NULL, 0);
-		if (err == 1)
-			break;
-		if (err != 0)
-			return -1;
-
-		if (svb.blk > highest)
-			highest = svb.blk;
-		pos += sizeof(svb) + svb.siglen;
-	}
-
-	if (fssize > 0) {
-		printf("Saved file system size is %"PRIu64" blocks, %.2fGB\n",
-		       fssize, (fssize * sbd.bsize) / ((float)(1 << 30)));
-		sbd.fssize = fssize;
-	} else {
-		sbd.fssize = highest + 1;
-	}
-
-	printf("Highest saved block is %"PRIu64" (0x%"PRIx64")\n", highest, highest);
-	return 0;
-}
-
 static int restore_data(int fd, struct metafd *mfd, off_t pos, int printonly)
 {
 	struct saved_metablock savedata = {0};
@@ -1248,6 +1217,12 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	if (error)
 		exit(1);
 
+	if (smh.sh_fs_bytes > 0) {
+		sbd.fssize = smh.sh_fs_bytes / sbd.bsize;
+		printf("Saved file system size is %"PRIu64" blocks, %.2fGB\n",
+		       sbd.fssize, smh.sh_fs_bytes / ((float)(1 << 30)));
+	}
+
 	printf("This is gfs%c metadata.\n", sbd.gfs1 ? '1': '2');
 
 	if (!printonly) {
@@ -1255,10 +1230,6 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 		printf("There are %"PRIu64" free blocks on the destination device.\n", space);
 	}
 
-	error = find_highest_block(&mfd, pos, sbd.fssize);
-	if (error)
-		exit(1);
-
 	error = restore_data(sbd.device_fd, &mfd, pos, printonly);
 	printf("File %s %s %s.\n", in_fn,
 	       (printonly ? "print" : "restore"),
-- 
2.24.1

[Cluster-devel] [PATCH 09/13] restoremeta: Metadata file reading overhaul

[Andrew Price]

Read the metadata file in large chunks for improved performance and
remove seeks so that we're just chomping through the file sequentially.
This is required to support other compression methods as compressed
seeking is a slow (emulated with reads) operation in zlib and not
supported at all by libbzip2.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 199 +++++++++++++++++++++----------------------
 1 file changed, 98 insertions(+), 101 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index dd203974..99b88e42 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -54,7 +54,6 @@ struct metafd {
 	gzFile gzfd;
 	const char *filename;
 	int gziplevel;
-	int (*seek)(struct metafd *mfd, off_t off, int whence);
 	int (*read)(struct metafd *mfd, void *buf, unsigned len);
 	int (*iseof)(struct metafd *mfd);
 	void (*close)(struct metafd *mfd);
@@ -71,11 +70,6 @@ static const char *gz_strerr(struct metafd *mfd)
 	return errstr;
 }
 
-static int gz_seek(struct metafd *mfd, off_t off, int whence)
-{
-	return gzseek(mfd->gzfd, off, whence);
-}
-
 static int gz_read(struct metafd *mfd, void *buf, unsigned len)
 {
 	return gzread(mfd->gzfd, buf, len);
@@ -864,25 +858,14 @@ static int save_header(struct metafd *mfd, uint64_t fsbytes)
 	return 0;
 }
 
-static int read_header(struct metafd *mfd, struct savemeta_header *smh)
+static void parse_header(char *buf, struct savemeta_header *smh)
 {
-	size_t rs;
-	struct savemeta_header smh_be = {0};
-
-	rs = mfd->read(mfd, &smh_be, sizeof(smh_be));
-	if (rs == -1) {
-		perror("Failed to read savemeta file header");
-		return -1;
-	}
-	if (rs != sizeof(smh_be))
-		return 1;
+	struct savemeta_header *smh_be = (void *)buf;
 
-	smh->sh_magic = be32_to_cpu(smh_be.sh_magic);
-	smh->sh_format = be32_to_cpu(smh_be.sh_format);
-	smh->sh_time = be64_to_cpu(smh_be.sh_time);
-	smh->sh_fs_bytes = be64_to_cpu(smh_be.sh_fs_bytes);
-
-	return 0;
+	smh->sh_magic = be32_to_cpu(smh_be->sh_magic);
+	smh->sh_format = be32_to_cpu(smh_be->sh_format);
+	smh->sh_time = be64_to_cpu(smh_be->sh_time);
+	smh->sh_fs_bytes = be64_to_cpu(smh_be->sh_fs_bytes);
 }
 
 static int check_header(struct savemeta_header *smh)
@@ -979,18 +962,40 @@ void savemeta(char *out_fn, int saveoption, int gziplevel)
 	exit(0);
 }
 
-static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *buf, uint16_t maxlen)
+char *restore_buf;
+ssize_t restore_left;
+off_t restore_off;
+#define RESTORE_BUF_SIZE (2 * 1024 * 1024)
+
+static char *restore_buf_next(struct metafd *mfd, size_t required_len)
 {
-	int ret;
-	uint16_t checklen;
+	if (restore_left < required_len) {
+		char *tail = restore_buf + restore_off;
+		int ret;
+
+		memmove(restore_buf, tail, restore_left);
+		ret = mfd->read(mfd, restore_buf + restore_left, RESTORE_BUF_SIZE - restore_left);
+		if (ret < required_len - restore_left)
+			return NULL;
+		restore_left += ret;
+		restore_off = 0;
+	}
+	restore_left -= required_len;
+	restore_off += required_len;
+	return &restore_buf[restore_off - required_len];
+}
+
+static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char **buf, uint16_t maxlen)
+{
+	struct saved_metablock *svb_be;
 	const char *errstr;
+	uint16_t checklen;
 
-	ret = mfd->read(mfd, svb, sizeof(*svb));
-	if (ret < sizeof(*svb)) {
+	svb_be = (struct saved_metablock *)(restore_buf_next(mfd, sizeof(*svb)));
+	if (svb_be == NULL)
 		goto read_err;
-	}
-	svb->blk = be64_to_cpu(svb->blk);
-	svb->siglen = be16_to_cpu(svb->siglen);
+	svb->blk = be64_to_cpu(svb_be->blk);
+	svb->siglen = be16_to_cpu(svb_be->siglen);
 
 	if (sbd.fssize && svb->blk >= sbd.fssize) {
 		fprintf(stderr, "Error: File system is too small to restore this metadata.\n");
@@ -1011,12 +1016,10 @@ static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *
 	}
 
 	if (buf != NULL && maxlen != 0) {
-		ret = mfd->read(mfd, buf, svb->siglen);
-		if (ret < svb->siglen) {
+		*buf = restore_buf_next(mfd, svb->siglen);
+		if (*buf == NULL)
 			goto read_err;
-		}
 	}
-
 	return 0;
 
 read_err:
@@ -1028,49 +1031,34 @@ read_err:
 	return -1;
 }
 
-static int restore_super(struct metafd *mfd, off_t pos)
+static int restore_super(struct metafd *mfd, char *buf, int printonly)
 {
 	int ret;
-	struct saved_metablock svb = {0};
-	char *buf;
-
-	buf = calloc(1, sizeof(struct gfs2_sb));
-	if (buf == NULL) {
-		perror("Failed to restore super block");
-		exit(1);
-	}
-	mfd->seek(mfd, pos, SEEK_SET);
-	ret = restore_block(mfd, &svb, buf, sizeof(struct gfs2_sb));
-	if (ret == 1) {
-		fprintf(stderr, "Reached end of file while restoring superblock\n");
-		goto err;
-	} else if (ret != 0) {
-		goto err;
-	}
 
 	gfs2_sb_in(&sbd.sd_sb, buf);
 	sbd1 = (struct gfs_sb *)&sbd.sd_sb;
 	ret = check_sb(&sbd.sd_sb);
 	if (ret < 0) {
-		fprintf(stderr,"Error: Invalid superblock data.\n");
-		goto err;
+		fprintf(stderr, "Error: Invalid superblock in metadata file.\n");
+		return -1;
 	}
 	if (ret == 1)
 		sbd.gfs1 = 1;
 	sbd.bsize = sbd.sd_sb.sb_bsize;
-	free(buf);
+	if ((!printonly) && lgfs2_sb_write(&sbd.sd_sb, sbd.device_fd, sbd.bsize)) {
+		fprintf(stderr, "Failed to write superblock\n");
+		return -1;
+	}
 	printf("Block size is %uB\n", sbd.bsize);
 	return 0;
-err:
-	free(buf);
-	return -1;
 }
 
-static int restore_data(int fd, struct metafd *mfd, off_t pos, int printonly)
+static int restore_data(int fd, struct metafd *mfd, int printonly)
 {
 	struct saved_metablock savedata = {0};
 	uint64_t writes = 0;
 	char *buf;
+	char *bp;
 
 	buf = calloc(1, sbd.bsize);
 	if (buf == NULL) {
@@ -1078,11 +1066,11 @@ static int restore_data(int fd, struct metafd *mfd, off_t pos, int printonly)
 		exit(1);
 	}
 
-	gzseek(mfd->gzfd, pos, SEEK_SET);
 	blks_saved = 0;
 	while (TRUE) {
 		int err;
-		err = restore_block(mfd, &savedata, buf, sbd.bsize);
+
+		err = restore_block(mfd, &savedata, &bp, sbd.bsize);
 		if (err == 1)
 			break;
 		if (err != 0) {
@@ -1092,7 +1080,7 @@ static int restore_data(int fd, struct metafd *mfd, off_t pos, int printonly)
 
 		if (printonly) {
 			struct gfs2_buffer_head dummy_bh = {
-				.b_data = buf,
+				.b_data = bp,
 				.b_blocknr = savedata.blk,
 			};
 			if (printonly > 1 && printonly == savedata.blk) {
@@ -1105,6 +1093,7 @@ static int restore_data(int fd, struct metafd *mfd, off_t pos, int printonly)
 			}
 		} else {
 			warm_fuzzy_stuff(savedata.blk, FALSE);
+			memcpy(buf, bp, savedata.siglen);
 			memset(buf + savedata.siglen, 0, sbd.bsize - savedata.siglen);
 			if (pwrite(fd, buf, sbd.bsize, savedata.blk * sbd.bsize) != sbd.bsize) {
 				fprintf(stderr, "write error: %s from %s:%d: block %lld (0x%llx)\n",
@@ -1133,14 +1122,20 @@ static void complain(const char *complaint)
 	    "<dest file system>\n");
 }
 
-static int restore_init(const char *path, struct metafd *mfd, struct savemeta_header *smh, off_t *pos)
+static int restore_init(const char *path, struct metafd *mfd, struct savemeta_header *smh, int printonly)
 {
-	struct gfs2_meta_header sbmh;
-	off_t startpos = 0;
-	char buf[256];
-	unsigned i;
-	size_t rs;
-	int err;
+	struct gfs2_meta_header *sbmh;
+	char *end;
+	char *bp;
+	int ret;
+
+	restore_buf = malloc(RESTORE_BUF_SIZE);
+	if (restore_buf == NULL) {
+		perror("Restore failed");
+		return -1;
+	}
+	restore_off = 0;
+	restore_left = 0;
 
 	mfd->filename = path;
 	mfd->fd = open(path, O_RDONLY|O_CLOEXEC);
@@ -1148,7 +1143,6 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 		perror("Could not open metadata file");
 		return 1;
 	}
-	mfd->seek = gz_seek;
 	mfd->read = gz_read;
 	mfd->iseof = gz_iseof;
 	mfd->close = gz_close;
@@ -1158,41 +1152,48 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 		perror("gzdopen");
 		return 1;
 	}
-	err = read_header(mfd, smh);
-	if (err < 0) {
-		return 1;
-	} else if (check_header(smh) != 0) {
-		printf("No valid file header found. Falling back to old format...\n");
-	} else if (err == 0) {
-		startpos = sizeof(*smh);
+	restore_left = mfd->read(mfd, restore_buf, RESTORE_BUF_SIZE);
+	if (restore_left < 512) {
+		fprintf(stderr, "Failed to read metadata file header and superblock\n");
+		return -1;
 	}
-	mfd->seek(mfd, startpos, SEEK_SET);
-	rs = mfd->read(mfd, buf, sizeof(buf));
-	if (rs != sizeof(buf)) {
-		fprintf(stderr, "Error: File is too small.\n");
-		return 1;
+	bp = restore_buf;
+	parse_header(bp, smh);
+	if (check_header(smh) != 0)
+		printf("No valid file header found. Falling back to old format...\n");
+	else {
+		bp = restore_buf + sizeof(*smh);
+		restore_off = sizeof(*smh);
 	}
-	/* Scan for the beginning of the file body. Required to support old formats(?). */
-	for (i = 0; i < (256 - sizeof(struct saved_metablock) - sizeof(sbmh)); i++) {
-		off_t off = i + sizeof(struct saved_metablock);
 
-		memcpy(&sbmh, &buf[off], sizeof(sbmh));
-		if (sbmh.mh_magic == cpu_to_be32(GFS2_MAGIC) &&
-		     sbmh.mh_type == cpu_to_be32(GFS2_METATYPE_SB))
+	/* Scan for the position of the superblock. Required to support old formats(?). */
+	end = &restore_buf[256 + sizeof(struct saved_metablock) + sizeof(*sbmh)];
+	while (bp <= end) {
+		sbmh = (struct gfs2_meta_header *)(bp + sizeof(struct saved_metablock));
+		if (sbmh->mh_magic == cpu_to_be32(GFS2_MAGIC) &&
+		    sbmh->mh_type == cpu_to_be32(GFS2_METATYPE_SB))
 			break;
+		bp++;
+	}
+	if (bp > end) {
+		fprintf(stderr, "No superblock found in metadata file\n");
+		return -1;
 	}
-	if (i == (sizeof(buf) - sizeof(struct saved_metablock) - sizeof(sbmh)))
-		i = 0;
-	*pos = startpos + i; /* File offset of saved sb */
+	ret = restore_super(mfd, bp + sizeof(struct saved_metablock), printonly);
+	if (ret != 0)
+		return ret;
+
+	bp += sizeof(struct saved_metablock) + sizeof(sbd.sd_sb);
+	restore_off = bp - restore_buf;
+	restore_left -= restore_off;
 	return 0;
 }
 
 void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 {
-	int error;
-	off_t pos = 0;
 	struct savemeta_header smh = {0};
 	struct metafd mfd = {0};
+	int error;
 
 	termlines = 0;
 	if (!in_fn)
@@ -1200,10 +1201,6 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 	if (!printonly && !out_device)
 		complain("No destination file system specified.");
 
-	error = restore_init(in_fn, &mfd, &smh, &pos);
-	if (error != 0)
-		exit(error);
-
 	if (!printonly) {
 		sbd.device_fd = open(out_device, O_RDWR);
 		if (sbd.device_fd < 0)
@@ -1213,9 +1210,9 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 				  optional block no */
 		printonly = check_keywords(out_device);
 
-	error = restore_super(&mfd, pos);
-	if (error)
-		exit(1);
+	error = restore_init(in_fn, &mfd, &smh, printonly);
+	if (error != 0)
+		exit(error);
 
 	if (smh.sh_fs_bytes > 0) {
 		sbd.fssize = smh.sh_fs_bytes / sbd.bsize;
@@ -1230,7 +1227,7 @@ void restoremeta(const char *in_fn, const char *out_device, uint64_t printonly)
 		printf("There are %"PRIu64" free blocks on the destination device.\n", space);
 	}
 
-	error = restore_data(sbd.device_fd, &mfd, pos, printonly);
+	error = restore_data(sbd.device_fd, &mfd, printonly);
 	printf("File %s %s %s.\n", in_fn,
 	       (printonly ? "print" : "restore"),
 	       (error ? "error" : "successful"));
-- 
2.24.1

[Cluster-devel] [PATCH 10/13] restoremeta: Convert iseof function to a flag

[Andrew Price]

EOF is flagged up on short reads so it can be detected in the ->read()
operation.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index 99b88e42..2cfa2f44 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -54,8 +54,8 @@ struct metafd {
 	gzFile gzfd;
 	const char *filename;
 	int gziplevel;
+	int eof;
 	int (*read)(struct metafd *mfd, void *buf, unsigned len);
-	int (*iseof)(struct metafd *mfd);
 	void (*close)(struct metafd *mfd);
 	const char* (*strerr)(struct metafd *mfd);
 };
@@ -72,12 +72,10 @@ static const char *gz_strerr(struct metafd *mfd)
 
 static int gz_read(struct metafd *mfd, void *buf, unsigned len)
 {
-	return gzread(mfd->gzfd, buf, len);
-}
-
-static int gz_iseof(struct metafd *mfd)
-{
-	return gzeof(mfd->gzfd);
+	int ret = gzread(mfd->gzfd, buf, len);
+	if (ret < len && gzeof(mfd->gzfd))
+		mfd->eof = 1;
+	return ret;
 }
 
 static void gz_close(struct metafd *mfd)
@@ -1023,7 +1021,7 @@ static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char *
 	return 0;
 
 read_err:
-	if (mfd->iseof(mfd))
+	if (mfd->eof)
 		return 1;
 
 	errstr = mfd->strerr(mfd);
@@ -1144,7 +1142,6 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 		return 1;
 	}
 	mfd->read = gz_read;
-	mfd->iseof = gz_iseof;
 	mfd->close = gz_close;
 	mfd->strerr = gz_strerr;
 	mfd->gzfd = gzdopen(mfd->fd, "rb");
-- 
2.24.1

[Cluster-devel] [PATCH 11/13] restoremeta: Combine parse_header() and check_header()

[Andrew Price]

Fixes an issue where smh was being left with garbage values with
old metadata files with no header.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index 2cfa2f44..7e47b810 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -856,20 +856,19 @@ static int save_header(struct metafd *mfd, uint64_t fsbytes)
 	return 0;
 }
 
-static void parse_header(char *buf, struct savemeta_header *smh)
+static int parse_header(char *buf, struct savemeta_header *smh)
 {
 	struct savemeta_header *smh_be = (void *)buf;
 
+	if (be32_to_cpu(smh_be->sh_magic) != SAVEMETA_MAGIC ||
+	    be32_to_cpu(smh_be->sh_format) > SAVEMETA_FORMAT) {
+		printf("No valid file header found. Falling back to old format...\n");
+		return -1;
+	}
 	smh->sh_magic = be32_to_cpu(smh_be->sh_magic);
 	smh->sh_format = be32_to_cpu(smh_be->sh_format);
 	smh->sh_time = be64_to_cpu(smh_be->sh_time);
 	smh->sh_fs_bytes = be64_to_cpu(smh_be->sh_fs_bytes);
-}
-
-static int check_header(struct savemeta_header *smh)
-{
-	if (smh->sh_magic != SAVEMETA_MAGIC || smh->sh_format > SAVEMETA_FORMAT)
-		return -1;
 	printf("Metadata saved at %s", ctime((time_t *)&smh->sh_time)); /* ctime() adds \n */
 	printf("File system size %.2fGB\n", smh->sh_fs_bytes / ((float)(1 << 30)));
 	return 0;
@@ -1155,14 +1154,10 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 		return -1;
 	}
 	bp = restore_buf;
-	parse_header(bp, smh);
-	if (check_header(smh) != 0)
-		printf("No valid file header found. Falling back to old format...\n");
-	else {
+	if (parse_header(bp, smh) == 0) {
 		bp = restore_buf + sizeof(*smh);
 		restore_off = sizeof(*smh);
 	}
-
 	/* Scan for the position of the superblock. Required to support old formats(?). */
 	end = &restore_buf[256 + sizeof(struct saved_metablock) + sizeof(*sbmh)];
 	while (bp <= end) {
-- 
2.24.1

[Cluster-devel] [PATCH 12/13] restoremeta: Add bzip2 support

[Andrew Price]

Implement the required functions for adding bzip2 support and add
functions for reading the initial chunk so that we can try each
compression method. gzip has to be tried last as zlib's read function
doesn't report if the file is not in gzip format, it just returns the
data as-is. Adds a dependency on libbzip2.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 configure.ac          |   1 +
 gfs2/edit/Makefile.am |   2 +
 gfs2/edit/savemeta.c  | 136 +++++++++++++++++++++++++++++++-----------
 3 files changed, 105 insertions(+), 34 deletions(-)

diff --git a/configure.ac b/configure.ac
index 0c1b0192..de9d5264 100644
--- a/configure.ac
+++ b/configure.ac
@@ -104,6 +104,7 @@ PKG_CHECK_MODULES([check], [check >= 0.9.8],
 AM_CONDITIONAL([HAVE_CHECK], [test "x$have_check" = "xyes"])
 
 PKG_CHECK_MODULES([zlib],[zlib])
+PKG_CHECK_MODULES([bzip2],[bzip2])
 PKG_CHECK_MODULES([blkid],[blkid])
 PKG_CHECK_MODULES([uuid],[uuid],
                   [have_uuid=yes],
diff --git a/gfs2/edit/Makefile.am b/gfs2/edit/Makefile.am
index 4a89502c..53a7dea6 100644
--- a/gfs2/edit/Makefile.am
+++ b/gfs2/edit/Makefile.am
@@ -23,11 +23,13 @@ gfs2_edit_CPPFLAGS = \
 gfs2_edit_CFLAGS = \
 	$(ncurses_CFLAGS) \
 	$(zlib_CFLAGS) \
+	$(bzip2_CFLAGS) \
 	$(uuid_CFLAGS)
 
 gfs2_edit_LDADD = \
 	$(top_builddir)/gfs2/libgfs2/libgfs2.la \
 	$(ncurses_LIBS) \
 	$(zlib_LIBS) \
+	$(bzip2_LIBS) \
 	$(uuid_LIBS)
 
diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index 7e47b810..e20b405d 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -19,6 +19,7 @@
 #include <sys/time.h>
 #include <linux/gfs2_ondisk.h>
 #include <zlib.h>
+#include <bzlib.h>
 #include <time.h>
 
 #include <logging.h>
@@ -52,6 +53,7 @@ struct saved_metablock {
 struct metafd {
 	int fd;
 	gzFile gzfd;
+	BZFILE *bzfd;
 	const char *filename;
 	int gziplevel;
 	int eof;
@@ -60,6 +62,31 @@ struct metafd {
 	const char* (*strerr)(struct metafd *mfd);
 };
 
+char *restore_buf;
+ssize_t restore_left;
+off_t restore_off;
+#define RESTORE_BUF_SIZE (2 * 1024 * 1024)
+
+static char *restore_buf_next(struct metafd *mfd, size_t required_len)
+{
+	if (restore_left < required_len) {
+		char *tail = restore_buf + restore_off;
+		int ret;
+
+		memmove(restore_buf, tail, restore_left);
+		ret = mfd->read(mfd, restore_buf + restore_left, RESTORE_BUF_SIZE - restore_left);
+		if (ret < (int)required_len - restore_left)
+			return NULL;
+		restore_left += ret;
+		restore_off = 0;
+	}
+	restore_left -= required_len;
+	restore_off += required_len;
+	return &restore_buf[restore_off - required_len];
+}
+
+/* gzip compression method */
+
 static const char *gz_strerr(struct metafd *mfd)
 {
 	int err;
@@ -83,6 +110,76 @@ static void gz_close(struct metafd *mfd)
 	gzclose(mfd->gzfd);
 }
 
+/* This should be tried last because gzip doesn't distinguish between
+   decompressing a gzip file and reading an uncompressed file */
+static int restore_try_gzip(struct metafd *mfd)
+{
+	mfd->read = gz_read;
+	mfd->close = gz_close;
+	mfd->strerr = gz_strerr;
+	lseek(mfd->fd, 0, SEEK_SET);
+	mfd->gzfd = gzdopen(mfd->fd, "rb");
+	if (!mfd->gzfd)
+		return 1;
+	restore_left = mfd->read(mfd, restore_buf, RESTORE_BUF_SIZE);
+	if (restore_left < 512)
+		return -1;
+	return 0;
+}
+
+/* bzip2 compression method */
+
+static const char *bz_strerr(struct metafd *mfd)
+{
+	int err;
+	const char *errstr = BZ2_bzerror(mfd->bzfd, &err);
+
+	if (err == BZ_IO_ERROR)
+		return strerror(errno);
+	return errstr;
+}
+
+static int bz_read(struct metafd *mfd, void *buf, unsigned len)
+{
+	int bzerr = BZ_OK;
+	int ret;
+
+	ret = BZ2_bzRead(&bzerr, mfd->bzfd, buf, len);
+	if (bzerr == BZ_OK)
+		return ret;
+	if (bzerr == BZ_STREAM_END) {
+		mfd->eof = 1;
+		return ret;
+	}
+	return -1;
+}
+
+static void bz_close(struct metafd *mfd)
+{
+	BZ2_bzclose(mfd->bzfd);
+}
+
+static int restore_try_bzip(struct metafd *mfd)
+{
+	int bzerr;
+	FILE *f;
+
+	f = fdopen(mfd->fd, "r");
+	if (f == NULL)
+		return 1;
+
+	mfd->read = bz_read;
+	mfd->close = bz_close;
+	mfd->strerr = bz_strerr;
+	mfd->bzfd = BZ2_bzReadOpen(&bzerr, f, 0, 0, NULL, 0);
+	if (!mfd->bzfd)
+		return 1;
+	restore_left = mfd->read(mfd, restore_buf, RESTORE_BUF_SIZE);
+	if (restore_left < 512)
+		return -1;
+	return 0;
+}
+
 static uint64_t blks_saved;
 static uint64_t journal_blocks[MAX_JOURNALS_SAVED];
 static uint64_t gfs1_journal_size = 0; /* in blocks */
@@ -345,11 +442,13 @@ static void warm_fuzzy_stuff(uint64_t wfsblock, int force)
  */
 static struct metafd savemetaopen(char *out_fn, int gziplevel)
 {
-	struct metafd mfd = {-1, NULL, NULL, gziplevel};
+	struct metafd mfd = {0};
 	char gzmode[3] = "w9";
 	char dft_fn[] = DFT_SAVE_FILE;
 	mode_t mask = umask(S_IXUSR | S_IRWXG | S_IRWXO);
 
+	mfd.gziplevel = gziplevel;
+
 	if (!out_fn) {
 		out_fn = dft_fn;
 		mfd.fd = mkstemp(out_fn);
@@ -959,29 +1058,6 @@ void savemeta(char *out_fn, int saveoption, int gziplevel)
 	exit(0);
 }
 
-char *restore_buf;
-ssize_t restore_left;
-off_t restore_off;
-#define RESTORE_BUF_SIZE (2 * 1024 * 1024)
-
-static char *restore_buf_next(struct metafd *mfd, size_t required_len)
-{
-	if (restore_left < required_len) {
-		char *tail = restore_buf + restore_off;
-		int ret;
-
-		memmove(restore_buf, tail, restore_left);
-		ret = mfd->read(mfd, restore_buf + restore_left, RESTORE_BUF_SIZE - restore_left);
-		if (ret < required_len - restore_left)
-			return NULL;
-		restore_left += ret;
-		restore_off = 0;
-	}
-	restore_left -= required_len;
-	restore_off += required_len;
-	return &restore_buf[restore_off - required_len];
-}
-
 static int restore_block(struct metafd *mfd, struct saved_metablock *svb, char **buf, uint16_t maxlen)
 {
 	struct saved_metablock *svb_be;
@@ -1140,16 +1216,8 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 		perror("Could not open metadata file");
 		return 1;
 	}
-	mfd->read = gz_read;
-	mfd->close = gz_close;
-	mfd->strerr = gz_strerr;
-	mfd->gzfd = gzdopen(mfd->fd, "rb");
-	if (!mfd->gzfd) {
-		perror("gzdopen");
-		return 1;
-	}
-	restore_left = mfd->read(mfd, restore_buf, RESTORE_BUF_SIZE);
-	if (restore_left < 512) {
+	if (restore_try_bzip(mfd) != 0 &&
+	    restore_try_gzip(mfd) != 0) {
 		fprintf(stderr, "Failed to read metadata file header and superblock\n");
 		return -1;
 	}
-- 
2.24.1

[Cluster-devel] [PATCH 13/13] restoremeta: Skip the right number of bytes for the superblock

[Andrew Price]

Fix a bug where the second saved block was reached by skipping over the
size of a gfs2 superblock, which is not always the correct offset.
Instead, skip over the length specified by the saved superblock's
descriptor.

Signed-off-by: Andrew Price <anprice@redhat.com>
---
 gfs2/edit/savemeta.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gfs2/edit/savemeta.c b/gfs2/edit/savemeta.c
index e20b405d..63cb91af 100644
--- a/gfs2/edit/savemeta.c
+++ b/gfs2/edit/savemeta.c
@@ -1198,6 +1198,7 @@ static void complain(const char *complaint)
 static int restore_init(const char *path, struct metafd *mfd, struct savemeta_header *smh, int printonly)
 {
 	struct gfs2_meta_header *sbmh;
+	uint16_t sb_siglen;
 	char *end;
 	char *bp;
 	int ret;
@@ -1229,6 +1230,7 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 	/* Scan for the position of the superblock. Required to support old formats(?). */
 	end = &restore_buf[256 + sizeof(struct saved_metablock) + sizeof(*sbmh)];
 	while (bp <= end) {
+		sb_siglen = be16_to_cpu(((struct saved_metablock *)bp)->siglen);
 		sbmh = (struct gfs2_meta_header *)(bp + sizeof(struct saved_metablock));
 		if (sbmh->mh_magic == cpu_to_be32(GFS2_MAGIC) &&
 		    sbmh->mh_type == cpu_to_be32(GFS2_METATYPE_SB))
@@ -1243,7 +1245,7 @@ static int restore_init(const char *path, struct metafd *mfd, struct savemeta_he
 	if (ret != 0)
 		return ret;
 
-	bp += sizeof(struct saved_metablock) + sizeof(sbd.sd_sb);
+	bp += sizeof(struct saved_metablock) + sb_siglen;
 	restore_off = bp - restore_buf;
 	restore_left -= restore_off;
 	return 0;
-- 
2.24.1