Recent changes (master)

Jens Axboe
2018-06-22T12:00:16Z

The following changes since commit 5de1d4ba1e6ae82bb4ad559463801cb6b7096ac3:

Merge branch 'readonly-trim' of https://github.com/vincentkfu/fio (2018-06-18 13:58:26 -0600)

are available in the git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 9caaf02b06deecf0b0a13c16b6d59dddc64b8c35:

Merge branch 'doc-norandommap' of https://github.com/larrystevenwise/fio (2018-06-21 11:22:41 -0600)

----------------------------------------------------------------
Jens Axboe (2):
Merge branch 'wip-single-glfs-instance' of https://github.com/zhanghuan/fio
Merge branch 'doc-norandommap' of https://github.com/larrystevenwise/fio

Steve Wise (1):
doc: add text about possibly verify errors with norandommap

Zhang Huan (1):
glusterfs: capable to test with one single glfs instance

HOWTO | 9 ++-
engines/gfapi.h | 1 +
engines/glusterfs.c | 179 +++++++++++++++++++++++++++++++++++++++++++---------
fio.1 | 9 ++-
4 files changed, 167 insertions(+), 31 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index e1399b4..70eed28 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1329,7 +1329,9 @@ I/O type
and that some blocks may be read/written more than once. If this option is
used with :option:`verify` and multiple blocksizes (via :option:`bsrange`),
only intact blocks are verified, i.e., partially-overwritten blocks are
- ignored.
+ ignored. With an async I/O engine and an I/O depth > 1, it is possible for
+ the same block to be overwritten, which can cause verification errors. Either
+ do not use norandommap in this case, or also use the lfsr random generator.

.. option:: softrandommap=bool

@@ -2663,6 +2665,11 @@ Verification
previously written file. If the data direction includes any form of write,
the verify will be of the newly written data.

+ To avoid false verification errors, do not use the norandommap option when
+ verifying data with async I/O engines and I/O depths > 1. Or use the
+ norandommap and the lfsr random generator together to avoid writing to the
+ same offset with muliple outstanding I/Os.
+
.. option:: verify_offset=int

Swap the verification header with data somewhere else in the block before
diff --git a/engines/gfapi.h b/engines/gfapi.h
index 1028431..e4cdbcb 100644
--- a/engines/gfapi.h
+++ b/engines/gfapi.h
@@ -5,6 +5,7 @@ struct gf_options {
void *pad;
char *gf_vol;
char *gf_brick;
+ int gf_single_instance;
};

struct gf_data {
diff --git a/engines/glusterfs.c b/engines/glusterfs.c
index 981dfa3..d0250b7 100644
--- a/engines/glusterfs.c
+++ b/engines/glusterfs.c
@@ -28,16 +28,159 @@ struct fio_option gfapi_options[] = {
.group = FIO_OPT_G_GFAPI,
},
{
+ .name = "single-instance",
+ .lname = "Single glusterfs instance",
+ .type = FIO_OPT_BOOL,
+ .help = "Only one glusterfs instance",
+ .off1 = offsetof(struct gf_options, gf_single_instance),
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_GFAPI,
+ },
+ {
.name = NULL,
},
};

-int fio_gf_setup(struct thread_data *td)
+struct glfs_info {
+ struct flist_head list;
+ char *volume;
+ char *brick;
+ glfs_t *fs;
+ int refcount;
+};
+
+static pthread_mutex_t glfs_lock = PTHREAD_MUTEX_INITIALIZER;
+static FLIST_HEAD(glfs_list_head);
+
+static glfs_t *fio_gf_new_fs(char *volume, char *brick)
{
int r = 0;
+ glfs_t *fs;
+ struct stat sb = { 0, };
+
+ fs = glfs_new(volume);
+ if (!fs) {
+ log_err("glfs_new failed.\n");
+ goto out;
+ }
+ glfs_set_logging(fs, "/tmp/fio_gfapi.log", 7);
+ /* default to tcp */
+ r = glfs_set_volfile_server(fs, "tcp", brick, 0);
+ if (r) {
+ log_err("glfs_set_volfile_server failed.\n");
+ goto out;
+ }
+ r = glfs_init(fs);
+ if (r) {
+ log_err("glfs_init failed. Is glusterd running on brick?\n");
+ goto out;
+ }
+ sleep(2);
+ r = glfs_lstat(fs, ".", &sb);
+ if (r) {
+ log_err("glfs_lstat failed.\n");
+ goto out;
+ }
+
+out:
+ if (r) {
+ glfs_fini(fs);
+ fs = NULL;
+ }
+ return fs;
+}
+
+static glfs_t *fio_gf_get_glfs(struct gf_options *opt,
+ char *volume, char *brick)
+{
+ struct glfs_info *glfs = NULL;
+ struct glfs_info *tmp;
+ struct flist_head *entry;
+
+ if (!opt->gf_single_instance)
+ return fio_gf_new_fs(volume, brick);
+
+ pthread_mutex_lock (&glfs_lock);
+
+ flist_for_each(entry, &glfs_list_head) {
+ tmp = flist_entry(entry, struct glfs_info, list);
+ if (!strcmp(volume, tmp->volume) &&
+ !strcmp(brick, tmp->brick)) {
+ glfs = tmp;
+ break;
+ }
+ }
+
+ if (glfs) {
+ glfs->refcount++;
+ } else {
+ glfs = malloc(sizeof(*glfs));
+ if (!glfs)
+ goto out;
+ INIT_FLIST_HEAD(&glfs->list);
+ glfs->refcount = 0;
+ glfs->volume = strdup(volume);
+ glfs->brick = strdup(brick);
+ glfs->fs = fio_gf_new_fs(volume, brick);
+ if (!glfs->fs) {
+ free(glfs);
+ glfs = NULL;
+ goto out;
+ }
+
+ flist_add_tail(&glfs->list, &glfs_list_head);
+ glfs->refcount = 1;
+ }
+
+out:
+ pthread_mutex_unlock (&glfs_lock);
+
+ if (glfs)
+ return glfs->fs;
+ return NULL;
+}
+
+static void fio_gf_put_glfs(struct gf_options *opt, glfs_t *fs)
+{
+ struct glfs_info *glfs = NULL;
+ struct glfs_info *tmp;
+ struct flist_head *entry;
+
+ if (!opt->gf_single_instance) {
+ glfs_fini(fs);
+ return;
+ }
+
+ pthread_mutex_lock (&glfs_lock);
+
+ flist_for_each(entry, &glfs_list_head) {
+ tmp = flist_entry(entry, struct glfs_info, list);
+ if (tmp->fs == fs) {
+ glfs = tmp;
+ break;
+ }
+ }
+
+ if (!glfs) {
+ log_err("glfs not found to fini.\n");
+ } else {
+ glfs->refcount--;
+
+ if (glfs->refcount == 0) {
+ glfs_fini(glfs->fs);
+ free(glfs->volume);
+ free(glfs->brick);
+ flist_del(&glfs->list);
+ }
+ }
+
+ pthread_mutex_unlock (&glfs_lock);
+}
+
+int fio_gf_setup(struct thread_data *td)
+{
struct gf_data *g = NULL;
struct gf_options *opt = td->eo;
- struct stat sb = { 0, };

dprint(FD_IO, "fio setup\n");

@@ -49,42 +192,20 @@ int fio_gf_setup(struct thread_data *td)
log_err("malloc failed.\n");
return -ENOMEM;
}
- g->fs = NULL;
g->fd = NULL;
g->aio_events = NULL;

- g->fs = glfs_new(opt->gf_vol);
- if (!g->fs) {
- log_err("glfs_new failed.\n");
- goto cleanup;
- }
- glfs_set_logging(g->fs, "/tmp/fio_gfapi.log", 7);
- /* default to tcp */
- r = glfs_set_volfile_server(g->fs, "tcp", opt->gf_brick, 0);
- if (r) {
- log_err("glfs_set_volfile_server failed.\n");
+ g->fs = fio_gf_get_glfs(opt, opt->gf_vol, opt->gf_brick);
+ if (!g->fs)
goto cleanup;
- }
- r = glfs_init(g->fs);
- if (r) {
- log_err("glfs_init failed. Is glusterd running on brick?\n");
- goto cleanup;
- }
- sleep(2);
- r = glfs_lstat(g->fs, ".", &sb);
- if (r) {
- log_err("glfs_lstat failed.\n");
- goto cleanup;
- }
+
dprint(FD_FILE, "fio setup %p\n", g->fs);
td->io_ops_data = g;
return 0;
cleanup:
- if (g->fs)
- glfs_fini(g->fs);
free(g);
td->io_ops_data = NULL;
- return r;
+ return -EIO;
}

void fio_gf_cleanup(struct thread_data *td)
@@ -97,7 +218,7 @@ void fio_gf_cleanup(struct thread_data *td)
if (g->fd)
glfs_close(g->fd);
if (g->fs)
- glfs_fini(g->fs);
+ fio_gf_put_glfs(td->eo, g->fs);
free(g);
td->io_ops_data = NULL;
}
diff --git a/fio.1 b/fio.1
index c744d1a..6d2eba6 100644
--- a/fio.1
+++ b/fio.1
@@ -1120,7 +1120,9 @@ at past I/O history. This means that some blocks may not be read or written,
and that some blocks may be read/written more than once. If this option is
used with \fBverify\fR and multiple blocksizes (via \fBbsrange\fR),
only intact blocks are verified, i.e., partially\-overwritten blocks are
-ignored.
+ignored. With an async I/O engine and an I/O depth > 1, it is possible for
+the same block to be overwritten, which can cause verification errors. Either
+do not use norandommap in this case, or also use the lfsr random generator.
.TP
.BI softrandommap \fR=\fPbool
See \fBnorandommap\fR. If fio runs with the random block map enabled and
@@ -2370,6 +2372,11 @@ that the written data is also correctly read back. If the data direction
given is a read or random read, fio will assume that it should verify a
previously written file. If the data direction includes any form of write,
the verify will be of the newly written data.
+.P
+To avoid false verification errors, do not use the norandommap option when
+verifying data with async I/O engines and I/O depths > 1. Or use the
+norandommap and the lfsr random generator together to avoid writing to the
+same offset with muliple outstanding I/Os.
.RE
.TP
.BI verify_offset \fR=\fPint