Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit dfecde6a4b49bd299b2a7192c10533b9beb4820d:

  Merge branch '2021-05-13/stat-fix-integer-overflow' of https://github.com/flx42/fio (2021-05-14 09:36:59 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to b54e0d80c52e626021aacd0ae4d9875940cff9aa:

  Merge branch 'taras/nfs-upstream' of https://github.com/tarasglek/fio-1 (2021-05-18 17:34:38 -0600)

----------------------------------------------------------------
Jens Axboe (1):
      Merge branch 'taras/nfs-upstream' of https://github.com/tarasglek/fio-1

Taras Glek (6):
      NFS engine
      NFS configure fixes
      C-style comments
      single line bodies
      skip skeleton comments
      clean up nfs example

 HOWTO            |  13 ++-
 Makefile         |   6 ++
 configure        |  29 +++++
 engines/nfs.c    | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 examples/nfs.fio |  22 ++++
 fio.1            |  10 ++
 optgroup.c       |   4 +
 optgroup.h       |   2 +
 options.c        |   5 +
 9 files changed, 404 insertions(+), 1 deletion(-)
 create mode 100644 engines/nfs.c
 create mode 100644 examples/nfs.fio

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index f5681c0d..86fb2964 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1171,7 +1171,7 @@ I/O type
 
 		**1**
 			Backward-compatible alias for **mixed**.
-		
+
 		**2**
 			Alias for **both**.
 
@@ -2103,6 +2103,12 @@ I/O engine
 			I/O engine supporting asynchronous read and write operations to the
 			DAOS File System (DFS) via libdfs.
 
+		**nfs**
+			I/O engine supporting asynchronous read and write operations to
+			NFS filesystems from userspace via libnfs. This is useful for
+			achieving higher concurrency and thus throughput than is possible
+			via kernel NFS.
+
 I/O engine specific parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -2525,6 +2531,11 @@ with the caveat that when used on the command line, they must come after the
 	Specificy a different object class for the dfs file.
 	Use DAOS container's object class by default.
 
+.. option:: nfs_url=str : [nfs]
+
+	URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]
+	Refer to the libnfs README for more details.
+
 I/O depth
 ~~~~~~~~~
 
diff --git a/Makefile b/Makefile
index ba027b2e..ef317373 100644
--- a/Makefile
+++ b/Makefile
@@ -79,6 +79,12 @@ ifdef CONFIG_LIBNBD
   ENGINES += nbd
 endif
 
+ifdef CONFIG_LIBNFS
+  CFLAGS += $(LIBNFS_CFLAGS)
+  LIBS += $(LIBNFS_LIBS)
+  SOURCE += engines/nfs.c
+endif
+
 ifdef CONFIG_64BIT
   CPPFLAGS += -DBITS_PER_LONG=64
 else ifdef CONFIG_32BIT
diff --git a/configure b/configure
index e886bdc8..8b763700 100755
--- a/configure
+++ b/configure
@@ -170,6 +170,7 @@ disable_native="no"
 march_set="no"
 libiscsi="no"
 libnbd="no"
+libnfs="no"
 libzbc=""
 dfs=""
 dynamic_engines="no"
@@ -241,6 +242,8 @@ for opt do
   ;;
   --disable-tcmalloc) disable_tcmalloc="yes"
   ;;
+  --disable-nfs) disable_nfs="yes"
+  ;;
   --dynamic-libengines) dynamic_engines="yes"
   ;;
   --disable-dfs) dfs="no"
@@ -271,8 +274,10 @@ if test "$show_help" = "yes" ; then
   echo "--disable-rados         Disable Rados support even if found"
   echo "--disable-rbd           Disable Rados Block Device even if found"
   echo "--disable-http          Disable HTTP support even if found"
+  echo "--disable-nfs           Disable userspace NFS support even if found"
   echo "--disable-gfapi         Disable gfapi"
   echo "--enable-libhdfs        Enable hdfs support"
+  echo "--enable-libnfs         Enable nfs support"
   echo "--disable-lex           Disable use of lex/yacc for math"
   echo "--disable-pmem          Disable pmem based engines even if found"
   echo "--enable-lex            Enable use of lex/yacc for math"
@@ -2277,6 +2282,21 @@ EOF
 fi
 print_config "DAOS File System (dfs) Engine" "$dfs"
 
+##########################################
+# Check if we have libnfs (for userspace nfs support).
+if test "$disable_nfs" != "yes"; then
+  if $(pkg-config libnfs); then
+    libnfs="yes"
+    libnfs_cflags=$(pkg-config --cflags libnfs)
+    libnfs_libs=$(pkg-config --libs libnfs)
+  else
+    if test "$libnfs" = "yes" ; then
+      echo "libnfs" "Install libnfs"
+    fi
+  fi
+fi
+print_config "NFS engine" "$libnfs"
+
 ##########################################
 # Check if we have lex/yacc available
 yacc="no"
@@ -3101,6 +3121,9 @@ fi
 if test "$dfs" = "yes" ; then
   output_sym "CONFIG_DFS"
 fi
+if test "$libnfs" = "yes" ; then
+  output_sym "CONFIG_NFS"
+fi
 if test "$march_set" = "no" && test "$build_native" = "yes" ; then
   output_sym "CONFIG_BUILD_NATIVE"
 fi
@@ -3140,6 +3163,12 @@ if test "$libnbd" = "yes" ; then
   echo "LIBNBD_CFLAGS=$libnbd_cflags" >> $config_host_mak
   echo "LIBNBD_LIBS=$libnbd_libs" >> $config_host_mak
 fi
+if test "$libnfs" = "yes" ; then
+  output_sym "CONFIG_LIBNFS"
+  echo "CONFIG_LIBNFS=m" >> $config_host_mak
+  echo "LIBNFS_CFLAGS=$libnfs_cflags" >> $config_host_mak
+  echo "LIBNFS_LIBS=$libnfs_libs" >> $config_host_mak
+fi
 if test "$dynamic_engines" = "yes" ; then
   output_sym "CONFIG_DYNAMIC_ENGINES"
 fi
diff --git a/engines/nfs.c b/engines/nfs.c
new file mode 100644
index 00000000..21be8833
--- /dev/null
+++ b/engines/nfs.c
@@ -0,0 +1,314 @@
+#include <stdlib.h>
+#include <poll.h>
+#include <nfsc/libnfs.h>
+#include <nfsc/libnfs-raw.h>
+#include <nfsc/libnfs-raw-mount.h>
+
+#include "../fio.h"
+#include "../optgroup.h"
+
+enum nfs_op_type {
+	NFS_READ_WRITE = 0,
+	NFS_STAT_MKDIR_RMDIR,
+	NFS_STAT_TOUCH_RM,
+};
+
+struct fio_libnfs_options {
+	struct nfs_context *context;
+	char *nfs_url;
+	unsigned int queue_depth; /* nfs_callback needs this info, but doesn't have fio td structure to pull it from */
+	/* the following implement a circular queue of outstanding IOs */
+	int outstanding_events; /* IOs issued to libnfs, that have not returned yet */
+	int prev_requested_event_index; /* event last returned via fio_libnfs_event */
+	int next_buffered_event; /* round robin-pointer within events[] */
+	int buffered_event_count; /* IOs completed by libnfs, waiting for FIO */
+	int free_event_buffer_index; /* next free buffer */
+	struct io_u**events;
+};
+
+struct nfs_data {
+	struct nfsfh *nfsfh;
+	struct fio_libnfs_options *options;
+};
+
+static struct fio_option options[] = {
+	{
+		.name     = "nfs_url",
+		.lname    = "nfs_url",
+		.type     = FIO_OPT_STR_STORE,
+		.help	= "URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]",
+		.off1     = offsetof(struct fio_libnfs_options, nfs_url),
+		.category = FIO_OPT_C_ENGINE,
+		.group	= __FIO_OPT_G_NFS,
+	},
+	{
+		.name     = NULL,
+	},
+};
+
+static struct io_u *fio_libnfs_event(struct thread_data *td, int event)
+{
+	struct fio_libnfs_options *o = td->eo;
+	struct io_u *io_u = o->events[o->next_buffered_event];
+	assert(o->events[o->next_buffered_event]);
+	o->events[o->next_buffered_event] = NULL;
+	o->next_buffered_event = (o->next_buffered_event + 1) % td->o.iodepth;
+	/* validate our state machine */
+	assert(o->buffered_event_count);
+	o->buffered_event_count--;
+	assert(io_u);
+	/* assert that fio_libnfs_event is being called in sequential fashion */
+	assert(event == 0 || o->prev_requested_event_index + 1 == event);
+	if (o->buffered_event_count == 0) {
+		o->prev_requested_event_index = -1;
+	} else {
+		o->prev_requested_event_index = event;
+	}
+	return io_u;
+}
+
+static int nfs_event_loop(struct thread_data *td, bool flush) {
+	struct fio_libnfs_options *o = td->eo;
+	struct pollfd pfds[1]; /* nfs:0 */
+	/* we already have stuff queued for fio, no need to waste cpu on poll() */
+	if (o->buffered_event_count)
+		return o->buffered_event_count;
+	/* fio core logic seems to stop calling this event-loop if we ever return with 0 events */
+	#define SHOULD_WAIT() (o->outstanding_events == td->o.iodepth || (flush && o->outstanding_events))
+
+	do {
+		int timeout = SHOULD_WAIT() ? -1 : 0;
+		int ret = 0;
+		pfds[0].fd = nfs_get_fd(o->context);
+		pfds[0].events = nfs_which_events(o->context);
+		ret = poll(&pfds[0], 1, timeout);
+		if (ret < 0) {
+			if (errno == EINTR || errno == EAGAIN) {
+				continue;
+			}
+			log_err("nfs: failed to poll events: %s.\n",
+				strerror(errno));
+			break;
+		}
+
+		ret = nfs_service(o->context, pfds[0].revents);
+		if (ret < 0) {
+			log_err("nfs: socket is in an unrecoverable error state.\n");
+			break;
+		}
+	} while (SHOULD_WAIT());
+	return o->buffered_event_count;
+#undef SHOULD_WAIT
+}
+
+static int fio_libnfs_getevents(struct thread_data *td, unsigned int min,
+				  unsigned int max, const struct timespec *t)
+{
+	return nfs_event_loop(td, false);
+}
+
+static void nfs_callback(int res, struct nfs_context *nfs, void *data,
+                       void *private_data)
+{
+	struct io_u *io_u = private_data;
+	struct nfs_data *nfs_data = io_u->file->engine_data;
+	struct fio_libnfs_options *o = nfs_data->options;
+	if (res < 0) {
+		log_err("Failed NFS operation(code:%d): %s\n", res, nfs_get_error(o->context));
+		io_u->error = -res;
+		/* res is used for read math below, don't wanna pass negative there */
+		res = 0;
+	} else if (io_u->ddir == DDIR_READ) {
+		memcpy(io_u->buf, data, res);
+		if (res == 0)
+			log_err("Got NFS EOF, this is probably not expected\n");
+	}
+	/* fio uses resid to track remaining data */
+	io_u->resid = io_u->xfer_buflen - res;
+
+	assert(!o->events[o->free_event_buffer_index]);
+	o->events[o->free_event_buffer_index] = io_u;
+	o->free_event_buffer_index = (o->free_event_buffer_index + 1) % o->queue_depth;
+	o->outstanding_events--;
+	o->buffered_event_count++;
+}
+
+static int queue_write(struct fio_libnfs_options *o, struct io_u *io_u) {
+	struct nfs_data *nfs_data = io_u->engine_data;
+	return nfs_pwrite_async(o->context, nfs_data->nfsfh,
+                           io_u->offset, io_u->buflen, io_u->buf, nfs_callback,
+                           io_u);
+}
+
+static int queue_read(struct fio_libnfs_options *o, struct io_u *io_u) {
+	struct nfs_data *nfs_data = io_u->engine_data;
+	return nfs_pread_async(o->context,  nfs_data->nfsfh, io_u->offset, io_u->buflen, nfs_callback,  io_u);
+}
+
+static enum fio_q_status fio_libnfs_queue(struct thread_data *td,
+					    struct io_u *io_u)
+{
+	struct nfs_data *nfs_data = io_u->file->engine_data;
+	struct fio_libnfs_options *o = nfs_data->options;
+	struct nfs_context *nfs = o->context;
+	int err;
+	enum fio_q_status ret = FIO_Q_QUEUED;
+
+	io_u->engine_data = nfs_data;
+	switch(io_u->ddir) {
+		case DDIR_WRITE:
+			err = queue_write(o, io_u);
+			break;
+		case DDIR_READ:
+			err = queue_read(o, io_u);
+			break;
+		case DDIR_TRIM:
+			log_err("nfs: trim is not supported");
+			err = -1;
+			break;
+		default:
+			log_err("nfs: unhandled io %d\n", io_u->ddir);
+			err = -1;
+	}
+	if (err) {
+		log_err("nfs: Failed to queue nfs op: %s\n", nfs_get_error(nfs));
+		td->error = 1;
+		return FIO_Q_COMPLETED;
+	}
+	o->outstanding_events++;
+	return ret;
+}
+
+/*
+ * Do a mount if one has not been done before 
+ */
+static int do_mount(struct thread_data *td, const char *url)
+{
+	size_t event_size = sizeof(struct io_u **) * td->o.iodepth;
+	struct fio_libnfs_options *options = td->eo;
+	struct nfs_url *nfs_url = NULL;
+	int ret = 0;
+	int path_len = 0;
+	char *mnt_dir = NULL;
+
+	if (options->context)
+		return 0;
+
+	options->context = nfs_init_context();
+	if (options->context == NULL) {
+		log_err("nfs: failed to init nfs context\n");
+		return -1;
+	}
+
+	options->events = malloc(event_size);
+	memset(options->events, 0, event_size);
+
+	options->prev_requested_event_index = -1;
+	options->queue_depth = td->o.iodepth;
+
+	nfs_url = nfs_parse_url_full(options->context, url);
+	path_len = strlen(nfs_url->path);
+	mnt_dir = malloc(path_len + strlen(nfs_url->file) + 1);
+	strcpy(mnt_dir, nfs_url->path);
+	strcpy(mnt_dir + strlen(nfs_url->path), nfs_url->file);
+	ret = nfs_mount(options->context, nfs_url->server, mnt_dir);
+	free(mnt_dir);
+	nfs_destroy_url(nfs_url);
+	return ret;
+}
+
+static int fio_libnfs_setup(struct thread_data *td)
+{
+	/* Using threads with libnfs causes fio to hang on exit, lower performance */
+	td->o.use_thread = 0;
+	return 0;
+}
+
+static void fio_libnfs_cleanup(struct thread_data *td)
+{
+	struct fio_libnfs_options *o = td->eo;
+	nfs_umount(o->context);
+	nfs_destroy_context(o->context);
+	free(o->events);
+}
+
+static int fio_libnfs_open(struct thread_data *td, struct fio_file *f)
+{
+	int ret;
+	struct fio_libnfs_options *options = td->eo;
+	struct nfs_data *nfs_data = NULL;
+	int flags = 0;
+
+	if (!options->nfs_url) {
+		log_err("nfs: nfs_url is a required parameter\n");
+		return -1;
+	}
+
+	ret = do_mount(td, options->nfs_url);
+
+	if (ret != 0) {
+		log_err("nfs: Failed to mount %s with code %d: %s\n", options->nfs_url, ret, nfs_get_error(options->context));
+		return ret;
+	}
+	nfs_data = malloc(sizeof(struct nfs_data));
+	memset(nfs_data, 0, sizeof(struct nfs_data));
+	nfs_data->options = options;
+
+	if (td->o.td_ddir == TD_DDIR_WRITE) {
+		flags |= O_CREAT | O_RDWR;
+	} else {
+		flags |= O_RDWR;
+	}
+	ret = nfs_open(options->context, f->file_name, flags, &nfs_data->nfsfh);
+
+	if (ret != 0)
+		log_err("Failed to open %s: %s\n", f->file_name, nfs_get_error(options->context));
+	f->engine_data = nfs_data;
+	return ret;
+}
+
+static int fio_libnfs_close(struct thread_data *td, struct fio_file *f)
+{
+	struct nfs_data *nfs_data = f->engine_data;
+	struct fio_libnfs_options *o = nfs_data->options;
+	int ret = 0;
+	if (nfs_data->nfsfh)
+		ret = nfs_close(o->context, nfs_data->nfsfh);
+	free(nfs_data);
+	f->engine_data = NULL;
+	return ret;
+}
+
+/*
+ * Hook for writing out outstanding data.
+ */
+static int fio_libnfs_commit(struct thread_data *td) {
+	nfs_event_loop(td, true);
+	return 0;
+}
+
+struct ioengine_ops ioengine = {
+	.name		= "nfs",
+	.version	= FIO_IOOPS_VERSION,
+	.setup		= fio_libnfs_setup,
+	.queue		= fio_libnfs_queue,
+	.getevents	= fio_libnfs_getevents,
+	.event		= fio_libnfs_event,
+	.cleanup	= fio_libnfs_cleanup,
+	.open_file	= fio_libnfs_open,
+	.close_file	= fio_libnfs_close,
+	.commit     = fio_libnfs_commit,
+	.flags      = FIO_DISKLESSIO | FIO_NOEXTEND | FIO_NODISKUTIL,
+	.options	= options,
+	.option_struct_size	= sizeof(struct fio_libnfs_options),
+};
+
+static void fio_init fio_nfs_register(void)
+{
+	register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_nfs_unregister(void)
+{
+	unregister_ioengine(&ioengine);
+}
diff --git a/examples/nfs.fio b/examples/nfs.fio
new file mode 100644
index 00000000..f856cebf
--- /dev/null
+++ b/examples/nfs.fio
@@ -0,0 +1,22 @@
+[global]
+nfs_url=nfs://127.0.0.1/nfs
+blocksize=524288
+iodepth=10
+ioengine=nfs
+size=104857600
+lat_percentiles=1
+group_reporting
+numjobs=10
+ramp_time=5s
+filename_format=myfiles.$clientuid.$jobnum.$filenum
+time_based=1
+
+[write]
+rw=write
+runtime=10s
+stonewall
+
+[read]
+wait_for=write
+rw=randread
+runtime=10s
diff --git a/fio.1 b/fio.1
index 533bcf6a..ab08cb01 100644
--- a/fio.1
+++ b/fio.1
@@ -1901,6 +1901,12 @@ not be \fBcudamalloc\fR. This ioengine defines engine specific options.
 .B dfs
 I/O engine supporting asynchronous read and write operations to the DAOS File
 System (DFS) via libdfs.
+.TP
+.B nfs
+I/O engine supporting asynchronous read and write operations to
+NFS filesystems from userspace via libnfs. This is useful for
+achieving higher concurrency and thus throughput than is possible
+via kernel NFS.
 .SS "I/O engine specific parameters"
 In addition, there are some parameters which are only valid when a specific
 \fBioengine\fR is in use. These are used identically to normal parameters,
@@ -2283,6 +2289,10 @@ Use DAOS container's chunk size by default.
 .BI (dfs)object_class
 Specificy a different object class for the dfs file.
 Use DAOS container's object class by default.
+.TP
+.BI (nfs)nfs_url
+URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]
+Refer to the libnfs README for more details.
 .SS "I/O depth"
 .TP
 .BI iodepth \fR=\fPint
diff --git a/optgroup.c b/optgroup.c
index 15a16229..bebb4a51 100644
--- a/optgroup.c
+++ b/optgroup.c
@@ -185,6 +185,10 @@ static const struct opt_group fio_opt_cat_groups[] = {
 		.name	= "DAOS File System (dfs) I/O engine", /* dfs */
 		.mask	= FIO_OPT_G_DFS,
 	},
+	{
+		.name	= "NFS I/O engine", /* nfs */
+		.mask	= FIO_OPT_G_NFS,
+	},
 	{
 		.name	= NULL,
 	},
diff --git a/optgroup.h b/optgroup.h
index ff748629..1fb84a29 100644
--- a/optgroup.h
+++ b/optgroup.h
@@ -70,6 +70,7 @@ enum opt_category_group {
 	__FIO_OPT_G_NR,
 	__FIO_OPT_G_LIBCUFILE,
 	__FIO_OPT_G_DFS,
+	__FIO_OPT_G_NFS,
 
 	FIO_OPT_G_RATE		= (1ULL << __FIO_OPT_G_RATE),
 	FIO_OPT_G_ZONE		= (1ULL << __FIO_OPT_G_ZONE),
@@ -110,6 +111,7 @@ enum opt_category_group {
 	FIO_OPT_G_INVALID	= (1ULL << __FIO_OPT_G_NR),
 	FIO_OPT_G_ISCSI         = (1ULL << __FIO_OPT_G_ISCSI),
 	FIO_OPT_G_NBD		= (1ULL << __FIO_OPT_G_NBD),
+	FIO_OPT_G_NFS		= (1ULL << __FIO_OPT_G_NFS),
 	FIO_OPT_G_IOURING	= (1ULL << __FIO_OPT_G_IOURING),
 	FIO_OPT_G_FILESTAT	= (1ULL << __FIO_OPT_G_FILESTAT),
 	FIO_OPT_G_LIBCUFILE	= (1ULL << __FIO_OPT_G_LIBCUFILE),
diff --git a/options.c b/options.c
index ddabaa82..b82a10aa 100644
--- a/options.c
+++ b/options.c
@@ -2025,6 +2025,11 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 			  { .ival = "dfs",
 			    .help = "DAOS File System (dfs) IO engine",
 			  },
+#endif
+#ifdef CONFIG_NFS
+			  { .ival = "nfs",
+			    .help = "NFS IO engine",
+			  },
 #endif
 		},
 	},