Recent changes (master)

Jens Axboe
2021-05-19T12:01:26Z

The following changes since commit dfecde6a4b49bd299b2a7192c10533b9beb4820d:

Merge branch '2021-05-13/stat-fix-integer-overflow' of https://github.com/flx42/fio (2021-05-14 09:36:59 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to b54e0d80c52e626021aacd0ae4d9875940cff9aa:

Merge branch 'taras/nfs-upstream' of https://github.com/tarasglek/fio-1 (2021-05-18 17:34:38 -0600)

----------------------------------------------------------------
Jens Axboe (1):
Merge branch 'taras/nfs-upstream' of https://github.com/tarasglek/fio-1

Taras Glek (6):
NFS engine
NFS configure fixes
C-style comments
single line bodies
skip skeleton comments
clean up nfs example

HOWTO | 13 ++-
Makefile | 6 ++
configure | 29 +++++
engines/nfs.c | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
examples/nfs.fio | 22 ++++
fio.1 | 10 ++
optgroup.c | 4 +
optgroup.h | 2 +
options.c | 5 +
9 files changed, 404 insertions(+), 1 deletion(-)
create mode 100644 engines/nfs.c
create mode 100644 examples/nfs.fio

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index f5681c0d..86fb2964 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1171,7 +1171,7 @@ I/O type

**1**
Backward-compatible alias for **mixed**.
-
+
**2**
Alias for **both**.

@@ -2103,6 +2103,12 @@ I/O engine
I/O engine supporting asynchronous read and write operations to the
DAOS File System (DFS) via libdfs.

+ **nfs**
+ I/O engine supporting asynchronous read and write operations to
+ NFS filesystems from userspace via libnfs. This is useful for
+ achieving higher concurrency and thus throughput than is possible
+ via kernel NFS.
+
I/O engine specific parameters
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -2525,6 +2531,11 @@ with the caveat that when used on the command line, they must come after the
Specificy a different object class for the dfs file.
Use DAOS container's object class by default.

+.. option:: nfs_url=str : [nfs]
+
+ URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]
+ Refer to the libnfs README for more details.
+
I/O depth
~~~~~~~~~

diff --git a/Makefile b/Makefile
index ba027b2e..ef317373 100644
--- a/Makefile
+++ b/Makefile
@@ -79,6 +79,12 @@ ifdef CONFIG_LIBNBD
ENGINES += nbd
endif

+ifdef CONFIG_LIBNFS
+ CFLAGS += $(LIBNFS_CFLAGS)
+ LIBS += $(LIBNFS_LIBS)
+ SOURCE += engines/nfs.c
+endif
+
ifdef CONFIG_64BIT
CPPFLAGS += -DBITS_PER_LONG=64
else ifdef CONFIG_32BIT
diff --git a/configure b/configure
index e886bdc8..8b763700 100755
--- a/configure
+++ b/configure
@@ -170,6 +170,7 @@ disable_native="no"
march_set="no"
libiscsi="no"
libnbd="no"
+libnfs="no"
libzbc=""
dfs=""
dynamic_engines="no"
@@ -241,6 +242,8 @@ for opt do
;;
--disable-tcmalloc) disable_tcmalloc="yes"
;;
+ --disable-nfs) disable_nfs="yes"
+ ;;
--dynamic-libengines) dynamic_engines="yes"
;;
--disable-dfs) dfs="no"
@@ -271,8 +274,10 @@ if test "$show_help" = "yes" ; then
echo "--disable-rados Disable Rados support even if found"
echo "--disable-rbd Disable Rados Block Device even if found"
echo "--disable-http Disable HTTP support even if found"
+ echo "--disable-nfs Disable userspace NFS support even if found"
echo "--disable-gfapi Disable gfapi"
echo "--enable-libhdfs Enable hdfs support"
+ echo "--enable-libnfs Enable nfs support"
echo "--disable-lex Disable use of lex/yacc for math"
echo "--disable-pmem Disable pmem based engines even if found"
echo "--enable-lex Enable use of lex/yacc for math"
@@ -2277,6 +2282,21 @@ EOF
fi
print_config "DAOS File System (dfs) Engine" "$dfs"

+##########################################
+# Check if we have libnfs (for userspace nfs support).
+if test "$disable_nfs" != "yes"; then
+ if $(pkg-config libnfs); then
+ libnfs="yes"
+ libnfs_cflags=$(pkg-config --cflags libnfs)
+ libnfs_libs=$(pkg-config --libs libnfs)
+ else
+ if test "$libnfs" = "yes" ; then
+ echo "libnfs" "Install libnfs"
+ fi
+ fi
+fi
+print_config "NFS engine" "$libnfs"
+
##########################################
# Check if we have lex/yacc available
yacc="no"
@@ -3101,6 +3121,9 @@ fi
if test "$dfs" = "yes" ; then
output_sym "CONFIG_DFS"
fi
+if test "$libnfs" = "yes" ; then
+ output_sym "CONFIG_NFS"
+fi
if test "$march_set" = "no" && test "$build_native" = "yes" ; then
output_sym "CONFIG_BUILD_NATIVE"
fi
@@ -3140,6 +3163,12 @@ if test "$libnbd" = "yes" ; then
echo "LIBNBD_CFLAGS=$libnbd_cflags" >> $config_host_mak
echo "LIBNBD_LIBS=$libnbd_libs" >> $config_host_mak
fi
+if test "$libnfs" = "yes" ; then
+ output_sym "CONFIG_LIBNFS"
+ echo "CONFIG_LIBNFS=m" >> $config_host_mak
+ echo "LIBNFS_CFLAGS=$libnfs_cflags" >> $config_host_mak
+ echo "LIBNFS_LIBS=$libnfs_libs" >> $config_host_mak
+fi
if test "$dynamic_engines" = "yes" ; then
output_sym "CONFIG_DYNAMIC_ENGINES"
fi
diff --git a/engines/nfs.c b/engines/nfs.c
new file mode 100644
index 00000000..21be8833
--- /dev/null
+++ b/engines/nfs.c
@@ -0,0 +1,314 @@
+#include <stdlib.h>
+#include <poll.h>
+#include <nfsc/libnfs.h>
+#include <nfsc/libnfs-raw.h>
+#include <nfsc/libnfs-raw-mount.h>
+
+#include "../fio.h"
+#include "../optgroup.h"
+
+enum nfs_op_type {
+ NFS_READ_WRITE = 0,
+ NFS_STAT_MKDIR_RMDIR,
+ NFS_STAT_TOUCH_RM,
+};
+
+struct fio_libnfs_options {
+ struct nfs_context *context;
+ char *nfs_url;
+ unsigned int queue_depth; /* nfs_callback needs this info, but doesn't have fio td structure to pull it from */
+ /* the following implement a circular queue of outstanding IOs */
+ int outstanding_events; /* IOs issued to libnfs, that have not returned yet */
+ int prev_requested_event_index; /* event last returned via fio_libnfs_event */
+ int next_buffered_event; /* round robin-pointer within events[] */
+ int buffered_event_count; /* IOs completed by libnfs, waiting for FIO */
+ int free_event_buffer_index; /* next free buffer */
+ struct io_u**events;
+};
+
+struct nfs_data {
+ struct nfsfh *nfsfh;
+ struct fio_libnfs_options *options;
+};
+
+static struct fio_option options[] = {
+ {
+ .name = "nfs_url",
+ .lname = "nfs_url",
+ .type = FIO_OPT_STR_STORE,
+ .help = "URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]",
+ .off1 = offsetof(struct fio_libnfs_options, nfs_url),
+ .category = FIO_OPT_C_ENGINE,
+ .group = __FIO_OPT_G_NFS,
+ },
+ {
+ .name = NULL,
+ },
+};
+
+static struct io_u *fio_libnfs_event(struct thread_data *td, int event)
+{
+ struct fio_libnfs_options *o = td->eo;
+ struct io_u *io_u = o->events[o->next_buffered_event];
+ assert(o->events[o->next_buffered_event]);
+ o->events[o->next_buffered_event] = NULL;
+ o->next_buffered_event = (o->next_buffered_event + 1) % td->o.iodepth;
+ /* validate our state machine */
+ assert(o->buffered_event_count);
+ o->buffered_event_count--;
+ assert(io_u);
+ /* assert that fio_libnfs_event is being called in sequential fashion */
+ assert(event == 0 || o->prev_requested_event_index + 1 == event);
+ if (o->buffered_event_count == 0) {
+ o->prev_requested_event_index = -1;
+ } else {
+ o->prev_requested_event_index = event;
+ }
+ return io_u;
+}
+
+static int nfs_event_loop(struct thread_data *td, bool flush) {
+ struct fio_libnfs_options *o = td->eo;
+ struct pollfd pfds[1]; /* nfs:0 */
+ /* we already have stuff queued for fio, no need to waste cpu on poll() */
+ if (o->buffered_event_count)
+ return o->buffered_event_count;
+ /* fio core logic seems to stop calling this event-loop if we ever return with 0 events */
+ #define SHOULD_WAIT() (o->outstanding_events == td->o.iodepth || (flush && o->outstanding_events))
+
+ do {
+ int timeout = SHOULD_WAIT() ? -1 : 0;
+ int ret = 0;
+ pfds[0].fd = nfs_get_fd(o->context);
+ pfds[0].events = nfs_which_events(o->context);
+ ret = poll(&pfds[0], 1, timeout);
+ if (ret < 0) {
+ if (errno == EINTR || errno == EAGAIN) {
+ continue;
+ }
+ log_err("nfs: failed to poll events: %s.\n",
+ strerror(errno));
+ break;
+ }
+
+ ret = nfs_service(o->context, pfds[0].revents);
+ if (ret < 0) {
+ log_err("nfs: socket is in an unrecoverable error state.\n");
+ break;
+ }
+ } while (SHOULD_WAIT());
+ return o->buffered_event_count;
+#undef SHOULD_WAIT
+}
+
+static int fio_libnfs_getevents(struct thread_data *td, unsigned int min,
+ unsigned int max, const struct timespec *t)
+{
+ return nfs_event_loop(td, false);
+}
+
+static void nfs_callback(int res, struct nfs_context *nfs, void *data,
+ void *private_data)
+{
+ struct io_u *io_u = private_data;
+ struct nfs_data *nfs_data = io_u->file->engine_data;
+ struct fio_libnfs_options *o = nfs_data->options;
+ if (res < 0) {
+ log_err("Failed NFS operation(code:%d): %s\n", res, nfs_get_error(o->context));
+ io_u->error = -res;
+ /* res is used for read math below, don't wanna pass negative there */
+ res = 0;
+ } else if (io_u->ddir == DDIR_READ) {
+ memcpy(io_u->buf, data, res);
+ if (res == 0)
+ log_err("Got NFS EOF, this is probably not expected\n");
+ }
+ /* fio uses resid to track remaining data */
+ io_u->resid = io_u->xfer_buflen - res;
+
+ assert(!o->events[o->free_event_buffer_index]);
+ o->events[o->free_event_buffer_index] = io_u;
+ o->free_event_buffer_index = (o->free_event_buffer_index + 1) % o->queue_depth;
+ o->outstanding_events--;
+ o->buffered_event_count++;
+}
+
+static int queue_write(struct fio_libnfs_options *o, struct io_u *io_u) {
+ struct nfs_data *nfs_data = io_u->engine_data;
+ return nfs_pwrite_async(o->context, nfs_data->nfsfh,
+ io_u->offset, io_u->buflen, io_u->buf, nfs_callback,
+ io_u);
+}
+
+static int queue_read(struct fio_libnfs_options *o, struct io_u *io_u) {
+ struct nfs_data *nfs_data = io_u->engine_data;
+ return nfs_pread_async(o->context, nfs_data->nfsfh, io_u->offset, io_u->buflen, nfs_callback, io_u);
+}
+
+static enum fio_q_status fio_libnfs_queue(struct thread_data *td,
+ struct io_u *io_u)
+{
+ struct nfs_data *nfs_data = io_u->file->engine_data;
+ struct fio_libnfs_options *o = nfs_data->options;
+ struct nfs_context *nfs = o->context;
+ int err;
+ enum fio_q_status ret = FIO_Q_QUEUED;
+
+ io_u->engine_data = nfs_data;
+ switch(io_u->ddir) {
+ case DDIR_WRITE:
+ err = queue_write(o, io_u);
+ break;
+ case DDIR_READ:
+ err = queue_read(o, io_u);
+ break;
+ case DDIR_TRIM:
+ log_err("nfs: trim is not supported");
+ err = -1;
+ break;
+ default:
+ log_err("nfs: unhandled io %d\n", io_u->ddir);
+ err = -1;
+ }
+ if (err) {
+ log_err("nfs: Failed to queue nfs op: %s\n", nfs_get_error(nfs));
+ td->error = 1;
+ return FIO_Q_COMPLETED;
+ }
+ o->outstanding_events++;
+ return ret;
+}
+
+/*
+ * Do a mount if one has not been done before
+ */
+static int do_mount(struct thread_data *td, const char *url)
+{
+ size_t event_size = sizeof(struct io_u **) * td->o.iodepth;
+ struct fio_libnfs_options *options = td->eo;
+ struct nfs_url *nfs_url = NULL;
+ int ret = 0;
+ int path_len = 0;
+ char *mnt_dir = NULL;
+
+ if (options->context)
+ return 0;
+
+ options->context = nfs_init_context();
+ if (options->context == NULL) {
+ log_err("nfs: failed to init nfs context\n");
+ return -1;
+ }
+
+ options->events = malloc(event_size);
+ memset(options->events, 0, event_size);
+
+ options->prev_requested_event_index = -1;
+ options->queue_depth = td->o.iodepth;
+
+ nfs_url = nfs_parse_url_full(options->context, url);
+ path_len = strlen(nfs_url->path);
+ mnt_dir = malloc(path_len + strlen(nfs_url->file) + 1);
+ strcpy(mnt_dir, nfs_url->path);
+ strcpy(mnt_dir + strlen(nfs_url->path), nfs_url->file);
+ ret = nfs_mount(options->context, nfs_url->server, mnt_dir);
+ free(mnt_dir);
+ nfs_destroy_url(nfs_url);
+ return ret;
+}
+
+static int fio_libnfs_setup(struct thread_data *td)
+{
+ /* Using threads with libnfs causes fio to hang on exit, lower performance */
+ td->o.use_thread = 0;
+ return 0;
+}
+
+static void fio_libnfs_cleanup(struct thread_data *td)
+{
+ struct fio_libnfs_options *o = td->eo;
+ nfs_umount(o->context);
+ nfs_destroy_context(o->context);
+ free(o->events);
+}
+
+static int fio_libnfs_open(struct thread_data *td, struct fio_file *f)
+{
+ int ret;
+ struct fio_libnfs_options *options = td->eo;
+ struct nfs_data *nfs_data = NULL;
+ int flags = 0;
+
+ if (!options->nfs_url) {
+ log_err("nfs: nfs_url is a required parameter\n");
+ return -1;
+ }
+
+ ret = do_mount(td, options->nfs_url);
+
+ if (ret != 0) {
+ log_err("nfs: Failed to mount %s with code %d: %s\n", options->nfs_url, ret, nfs_get_error(options->context));
+ return ret;
+ }
+ nfs_data = malloc(sizeof(struct nfs_data));
+ memset(nfs_data, 0, sizeof(struct nfs_data));
+ nfs_data->options = options;
+
+ if (td->o.td_ddir == TD_DDIR_WRITE) {
+ flags |= O_CREAT | O_RDWR;
+ } else {
+ flags |= O_RDWR;
+ }
+ ret = nfs_open(options->context, f->file_name, flags, &nfs_data->nfsfh);
+
+ if (ret != 0)
+ log_err("Failed to open %s: %s\n", f->file_name, nfs_get_error(options->context));
+ f->engine_data = nfs_data;
+ return ret;
+}
+
+static int fio_libnfs_close(struct thread_data *td, struct fio_file *f)
+{
+ struct nfs_data *nfs_data = f->engine_data;
+ struct fio_libnfs_options *o = nfs_data->options;
+ int ret = 0;
+ if (nfs_data->nfsfh)
+ ret = nfs_close(o->context, nfs_data->nfsfh);
+ free(nfs_data);
+ f->engine_data = NULL;
+ return ret;
+}
+
+/*
+ * Hook for writing out outstanding data.
+ */
+static int fio_libnfs_commit(struct thread_data *td) {
+ nfs_event_loop(td, true);
+ return 0;
+}
+
+struct ioengine_ops ioengine = {
+ .name = "nfs",
+ .version = FIO_IOOPS_VERSION,
+ .setup = fio_libnfs_setup,
+ .queue = fio_libnfs_queue,
+ .getevents = fio_libnfs_getevents,
+ .event = fio_libnfs_event,
+ .cleanup = fio_libnfs_cleanup,
+ .open_file = fio_libnfs_open,
+ .close_file = fio_libnfs_close,
+ .commit = fio_libnfs_commit,
+ .flags = FIO_DISKLESSIO | FIO_NOEXTEND | FIO_NODISKUTIL,
+ .options = options,
+ .option_struct_size = sizeof(struct fio_libnfs_options),
+};
+
+static void fio_init fio_nfs_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_nfs_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
diff --git a/examples/nfs.fio b/examples/nfs.fio
new file mode 100644
index 00000000..f856cebf
--- /dev/null
+++ b/examples/nfs.fio
@@ -0,0 +1,22 @@
+[global]
+nfs_url=nfs://127.0.0.1/nfs
+blocksize=524288
+iodepth=10
+ioengine=nfs
+size=104857600
+lat_percentiles=1
+group_reporting
+numjobs=10
+ramp_time=5s
+filename_format=myfiles.$clientuid.$jobnum.$filenum
+time_based=1
+
+[write]
+rw=write
+runtime=10s
+stonewall
+
+[read]
+wait_for=write
+rw=randread
+runtime=10s
diff --git a/fio.1 b/fio.1
index 533bcf6a..ab08cb01 100644
--- a/fio.1
+++ b/fio.1
@@ -1901,6 +1901,12 @@ not be \fBcudamalloc\fR. This ioengine defines engine specific options.
.B dfs
I/O engine supporting asynchronous read and write operations to the DAOS File
System (DFS) via libdfs.
+.TP
+.B nfs
+I/O engine supporting asynchronous read and write operations to
+NFS filesystems from userspace via libnfs. This is useful for
+achieving higher concurrency and thus throughput than is possible
+via kernel NFS.
.SS "I/O engine specific parameters"
In addition, there are some parameters which are only valid when a specific
\fBioengine\fR is in use. These are used identically to normal parameters,
@@ -2283,6 +2289,10 @@ Use DAOS container's chunk size by default.
.BI (dfs)object_class
Specificy a different object class for the dfs file.
Use DAOS container's object class by default.
+.TP
+.BI (nfs)nfs_url
+URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]
+Refer to the libnfs README for more details.
.SS "I/O depth"
.TP
.BI iodepth \fR=\fPint
diff --git a/optgroup.c b/optgroup.c
index 15a16229..bebb4a51 100644
--- a/optgroup.c
+++ b/optgroup.c
@@ -185,6 +185,10 @@ static const struct opt_group fio_opt_cat_groups[] = {
.name = "DAOS File System (dfs) I/O engine", /* dfs */
.mask = FIO_OPT_G_DFS,
},
+ {
+ .name = "NFS I/O engine", /* nfs */
+ .mask = FIO_OPT_G_NFS,
+ },
{
.name = NULL,
},
diff --git a/optgroup.h b/optgroup.h
index ff748629..1fb84a29 100644
--- a/optgroup.h
+++ b/optgroup.h
@@ -70,6 +70,7 @@ enum opt_category_group {
__FIO_OPT_G_NR,
__FIO_OPT_G_LIBCUFILE,
__FIO_OPT_G_DFS,
+ __FIO_OPT_G_NFS,

FIO_OPT_G_RATE = (1ULL << __FIO_OPT_G_RATE),
FIO_OPT_G_ZONE = (1ULL << __FIO_OPT_G_ZONE),
@@ -110,6 +111,7 @@ enum opt_category_group {
FIO_OPT_G_INVALID = (1ULL << __FIO_OPT_G_NR),
FIO_OPT_G_ISCSI = (1ULL << __FIO_OPT_G_ISCSI),
FIO_OPT_G_NBD = (1ULL << __FIO_OPT_G_NBD),
+ FIO_OPT_G_NFS = (1ULL << __FIO_OPT_G_NFS),
FIO_OPT_G_IOURING = (1ULL << __FIO_OPT_G_IOURING),
FIO_OPT_G_FILESTAT = (1ULL << __FIO_OPT_G_FILESTAT),
FIO_OPT_G_LIBCUFILE = (1ULL << __FIO_OPT_G_LIBCUFILE),
diff --git a/options.c b/options.c
index ddabaa82..b82a10aa 100644
--- a/options.c
+++ b/options.c
@@ -2025,6 +2025,11 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
{ .ival = "dfs",
.help = "DAOS File System (dfs) IO engine",
},
+#endif
+#ifdef CONFIG_NFS
+ { .ival = "nfs",
+ .help = "NFS IO engine",
+ },
#endif
},
},