Recent changes (master)

Jens Axboe
2017-07-04T12:00:24Z

The following changes since commit 5283741f7be708fbbb3feb2cd5ca5187f3a964d1:

gettime: reduce test CPU clock entries to 1000 (2017-07-02 16:21:47 -0600)

are available in the git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 1be7afd7329ebdca520d637f571d2b31c33f6ba1:

Merge branch 'fallocate_native' of https://github.com/sitsofe/fio (2017-07-03 16:51:31 -0600)

----------------------------------------------------------------
Jens Axboe (1):
Merge branch 'fallocate_native' of https://github.com/sitsofe/fio

Sitsofe Wheeler (2):
filesetup: add native fallocate
HOWTO: indent v3 terse output definition

HOWTO | 11 ++++++++---
file.h | 1 +
filesetup.c | 30 ++++++++++++++++++++++++++++--
fio.1 | 9 +++++++--
options.c | 14 +++++++++++---
os/os-linux.h | 18 ++++++++++++++++++
os/os-mac.h | 12 ++++++++++++
os/os.h | 8 ++++++++
8 files changed, 93 insertions(+), 10 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index 6e53cff..92c3b73 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1049,6 +1049,10 @@ I/O type
**none**
Do not pre-allocate space.

+ **native**
+ Use a platform's native pre-allocation call but fall back to
+ **none** behavior if it fails/is not implemented.
+
**posix**
Pre-allocate via :manpage:`posix_fallocate(3)`.

@@ -1063,8 +1067,9 @@ I/O type
Backward-compatible alias for **posix**.

May not be available on all supported platforms. **keep** is only available
- on Linux. If using ZFS on Solaris this must be set to **none** because ZFS
- doesn't support it. Default: **posix**.
+ on Linux. If using ZFS on Solaris this cannot be set to **posix**
+ because ZFS doesn't support pre-allocation. Default: **native** if any
+ pre-allocation methods are available, **none** if not.

.. option:: fadvise_hint=str

@@ -3328,7 +3333,7 @@ will be a disk utilization section.
Below is a single line containing short names for each of the fields in the
minimal output v3, separated by semicolons::

-terse_version_3;fio_version;jobname;groupid;error;read_kb;read_bandwidth;read_iops;read_runtime_ms;read_slat_min;read_slat_max;read_slat_mean;read_slat_dev;read_clat_min;read_clat_max;read_clat_mean;read_clat_dev;read_clat_pct01;read_clat_pct02;read_clat_pct03;read_clat_pct04;read_clat_pct05;read_clat_pct06;read_clat_pct07;read_clat_pct08;read_clat_pct09;read_clat_pct10;read_clat_pct11;read_clat_pct12;read_clat_pct13;read_clat_pct14;read_clat_pct15;read_clat_pct16;read_clat_pct17;read_clat_pct18;read_clat_pct19;read_clat_pct20;read_tlat_min;read_lat_max;read_lat_mean;read_lat_dev;read_bw_min;read_bw_max;read_bw_agg_pct;read_bw_mean;read_bw_dev;write_kb;write_bandwidth;write_iops;write_runtime_ms;write_slat_min;write_slat_max;write_slat_mean;write_slat_dev;write_clat_min;write_clat_max;write_clat_mean;write_clat_dev;write_clat_pct01;write_clat_pct02;write_clat_pct03;write_clat_pct04;write_clat_pct05;write_clat_pct06;write_clat_pct07;write_clat_pct08;write_clat_pct09;write_clat_pct10;
write_clat_pct11;write_clat_pct12;write_clat_pct13;write_clat_pct14;write_clat_pct15;write_clat_pct16;write_clat_pct17;write_clat_pct18;write_clat_pct19;write_clat_pct20;write_tlat_min;write_lat_max;write_lat_mean;write_lat_dev;write_bw_min;write_bw_max;write_bw_agg_pct;write_bw_mean;write_bw_dev;cpu_user;cpu_sys;cpu_csw;cpu_mjf;cpu_minf;iodepth_1;iodepth_2;iodepth_4;iodepth_8;iodepth_16;iodepth_32;iodepth_64;lat_2us;lat_4us;lat_10us;lat_20us;lat_50us;lat_100us;lat_250us;lat_500us;lat_750us;lat_1000us;lat_2ms;lat_4ms;lat_10ms;lat_20ms;lat_50ms;lat_100ms;lat_250ms;lat_500ms;lat_750ms;lat_1000ms;lat_2000ms;lat_over_2000ms;disk_name;disk_read_iops;disk_write_iops;disk_read_merges;disk_write_merges;disk_read_ticks;write_ticks;disk_queue_time;disk_util
+ terse_version_3;fio_version;jobname;groupid;error;read_kb;read_bandwidth;read_iops;read_runtime_ms;read_slat_min;read_slat_max;read_slat_mean;read_slat_dev;read_clat_min;read_clat_max;read_clat_mean;read_clat_dev;read_clat_pct01;read_clat_pct02;read_clat_pct03;read_clat_pct04;read_clat_pct05;read_clat_pct06;read_clat_pct07;read_clat_pct08;read_clat_pct09;read_clat_pct10;read_clat_pct11;read_clat_pct12;read_clat_pct13;read_clat_pct14;read_clat_pct15;read_clat_pct16;read_clat_pct17;read_clat_pct18;read_clat_pct19;read_clat_pct20;read_tlat_min;read_lat_max;read_lat_mean;read_lat_dev;read_bw_min;read_bw_max;read_bw_agg_pct;read_bw_mean;read_bw_dev;write_kb;write_bandwidth;write_iops;write_runtime_ms;write_slat_min;write_slat_max;write_slat_mean;write_slat_dev;write_clat_min;write_clat_max;write_clat_mean;write_clat_dev;write_clat_pct01;write_clat_pct02;write_clat_pct03;write_clat_pct04;write_clat_pct05;write_clat_pct06;write_clat_pct07;write_clat_pct08;write_clat_pct09;write_clat_pct10
;write_clat_pct11;write_clat_pct12;write_clat_pct13;write_clat_pct14;write_clat_pct15;write_clat_pct16;write_clat_pct17;write_clat_pct18;write_clat_pct19;write_clat_pct20;write_tlat_min;write_lat_max;write_lat_mean;write_lat_dev;write_bw_min;write_bw_max;write_bw_agg_pct;write_bw_mean;write_bw_dev;cpu_user;cpu_sys;cpu_csw;cpu_mjf;cpu_minf;iodepth_1;iodepth_2;iodepth_4;iodepth_8;iodepth_16;iodepth_32;iodepth_64;lat_2us;lat_4us;lat_10us;lat_20us;lat_50us;lat_100us;lat_250us;lat_500us;lat_750us;lat_1000us;lat_2ms;lat_4ms;lat_10ms;lat_20ms;lat_50ms;lat_100ms;lat_250ms;lat_500ms;lat_750ms;lat_1000ms;lat_2000ms;lat_over_2000ms;disk_name;disk_read_iops;disk_write_iops;disk_read_merges;disk_write_merges;disk_read_ticks;write_ticks;disk_queue_time;disk_util

Trace file format
diff --git a/file.h b/file.h
index 9801bb5..84daa5f 100644
--- a/file.h
+++ b/file.h
@@ -63,6 +63,7 @@ enum fio_fallocate_mode {
FIO_FALLOCATE_NONE = 1,
FIO_FALLOCATE_POSIX = 2,
FIO_FALLOCATE_KEEP_SIZE = 3,
+ FIO_FALLOCATE_NATIVE = 4,
};

/*
diff --git a/filesetup.c b/filesetup.c
index f3e3865..38ad9ed 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -38,6 +38,25 @@ static inline void clear_error(struct thread_data *td)
td->verror[0] = '\0';
}

+static inline int native_fallocate(struct thread_data *td, struct fio_file *f)
+{
+ bool success;
+
+ success = fio_fallocate(f, 0, f->real_file_size);
+ dprint(FD_FILE, "native fallocate of file %s size %llu was "
+ "%ssuccessful\n", f->file_name,
+ (unsigned long long) f->real_file_size,
+ !success ? "un": "");
+
+ if (success)
+ return 0;
+
+ if (errno == ENOSYS)
+ dprint(FD_FILE, "native fallocate is not implemented\n");
+
+ return -1;
+}
+
static void fallocate_file(struct thread_data *td, struct fio_file *f)
{
int r;
@@ -45,10 +64,16 @@ static void fallocate_file(struct thread_data *td, struct fio_file *f)
if (td->o.fill_device)
return;

-#ifdef CONFIG_POSIX_FALLOCATE
switch (td->o.fallocate_mode) {
+ case FIO_FALLOCATE_NATIVE:
+ r = native_fallocate(td, f);
+ if (r != 0)
+ log_err("fio: native_fallocate call failed: %s\n",
+ strerror(errno));
+ break;
case FIO_FALLOCATE_NONE:
break;
+#ifdef CONFIG_POSIX_FALLOCATE
case FIO_FALLOCATE_POSIX:
dprint(FD_FILE, "posix_fallocate file %s size %llu\n",
f->file_name,
@@ -58,6 +83,7 @@ static void fallocate_file(struct thread_data *td, struct fio_file *f)
if (r > 0)
log_err("fio: posix_fallocate fails: %s\n", strerror(r));
break;
+#endif /* CONFIG_POSIX_FALLOCATE */
#ifdef CONFIG_LINUX_FALLOCATE
case FIO_FALLOCATE_KEEP_SIZE:
dprint(FD_FILE, "fallocate(FALLOC_FL_KEEP_SIZE) "
@@ -74,7 +100,7 @@ static void fallocate_file(struct thread_data *td, struct fio_file *f)
log_err("fio: unknown fallocate mode: %d\n", td->o.fallocate_mode);
assert(0);
}
-#endif /* CONFIG_POSIX_FALLOCATE */
+
}

/*
diff --git a/fio.1 b/fio.1
index ab04208..9783646 100644
--- a/fio.1
+++ b/fio.1
@@ -436,6 +436,10 @@ are:
.B none
Do not pre-allocate space.
.TP
+.B native
+Use a platform's native pre-allocation call but fall back to 'none' behavior if
+it fails/is not implemented.
+.TP
.B posix
Pre-allocate via \fBposix_fallocate\fR\|(3).
.TP
@@ -450,8 +454,9 @@ Backward-compatible alias for 'posix'.
.RE
.P
May not be available on all supported platforms. 'keep' is only
-available on Linux. If using ZFS on Solaris this must be set to 'none'
-because ZFS doesn't support it. Default: 'posix'.
+available on Linux. If using ZFS on Solaris this cannot be set to 'posix'
+because ZFS doesn't support it. Default: 'native' if any pre-allocation methods
+are available, 'none' if not.
.RE
.TP
.BI fadvise_hint \fR=\fPstr
diff --git a/options.c b/options.c
index 09a21af..b21f09a 100644
--- a/options.c
+++ b/options.c
@@ -2289,14 +2289,14 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.parent = "nrfiles",
.hide = 1,
},
-#ifdef CONFIG_POSIX_FALLOCATE
+#if defined(CONFIG_POSIX_FALLOCATE) || defined(FIO_HAVE_NATIVE_FALLOCATE)
{
.name = "fallocate",
.lname = "Fallocate",
.type = FIO_OPT_STR,
.off1 = offsetof(struct thread_options, fallocate_mode),
.help = "Whether pre-allocation is performed when laying out files",
- .def = "posix",
+ .def = "native",
.category = FIO_OPT_C_FILE,
.group = FIO_OPT_G_INVALID,
.posval = {
@@ -2304,10 +2304,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.oval = FIO_FALLOCATE_NONE,
.help = "Do not pre-allocate space",
},
+ { .ival = "native",
+ .oval = FIO_FALLOCATE_NATIVE,
+ .help = "Use native pre-allocation if possible",
+ },
+#ifdef CONFIG_POSIX_FALLOCATE
{ .ival = "posix",
.oval = FIO_FALLOCATE_POSIX,
.help = "Use posix_fallocate()",
},
+#endif
#ifdef CONFIG_LINUX_FALLOCATE
{ .ival = "keep",
.oval = FIO_FALLOCATE_KEEP_SIZE,
@@ -2319,10 +2325,12 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.oval = FIO_FALLOCATE_NONE,
.help = "Alias for 'none'",
},
+#ifdef CONFIG_POSIX_FALLOCATE
{ .ival = "1",
.oval = FIO_FALLOCATE_POSIX,
.help = "Alias for 'posix'",
},
+#endif
},
},
#else /* CONFIG_POSIX_FALLOCATE */
@@ -2332,7 +2340,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.type = FIO_OPT_UNSUPPORTED,
.help = "Your platform does not support fallocate",
},
-#endif /* CONFIG_POSIX_FALLOCATE */
+#endif /* CONFIG_POSIX_FALLOCATE || FIO_HAVE_NATIVE_FALLOCATE */
{
.name = "fadvise_hint",
.lname = "Fadvise hint",
diff --git a/os/os-linux.h b/os/os-linux.h
index 8c1e93b..e7d600d 100644
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -392,4 +392,22 @@ static inline int shm_attach_to_open_removed(void)
return 1;
}

+#ifdef CONFIG_LINUX_FALLOCATE
+#define FIO_HAVE_NATIVE_FALLOCATE
+static inline bool fio_fallocate(struct fio_file *f, uint64_t offset,
+ uint64_t len)
+{
+ int ret;
+ ret = fallocate(f->fd, 0, 0, len);
+ if (ret == 0)
+ return true;
+
+ /* Work around buggy old glibc versions... */
+ if (ret > 0)
+ errno = ret;
+
+ return false;
+}
+#endif
+
#endif
diff --git a/os/os-mac.h b/os/os-mac.h
index 7de36ea..a1536c7 100644
--- a/os/os-mac.h
+++ b/os/os-mac.h
@@ -20,6 +20,7 @@
#define FIO_USE_GENERIC_INIT_RANDOM_STATE
#define FIO_HAVE_GETTID
#define FIO_HAVE_CHARDEV_SIZE
+#define FIO_HAVE_NATIVE_FALLOCATE

#define OS_MAP_ANON MAP_ANON

@@ -101,4 +102,15 @@ static inline int gettid(void)
*/
extern int fdatasync(int fd);

+static inline bool fio_fallocate(struct fio_file *f, uint64_t offset, uint64_t len)
+{
+ fstore_t store = {F_ALLOCATEALL, F_PEOFPOSMODE, offset, len};
+ if (fcntl(f->fd, F_PREALLOCATE, &store) != -1) {
+ if (ftruncate(f->fd, len) == 0)
+ return true;
+ }
+
+ return false;
+}
+
#endif
diff --git a/os/os.h b/os/os.h
index 1d400c8..afee9f9 100644
--- a/os/os.h
+++ b/os/os.h
@@ -361,4 +361,12 @@ static inline int shm_attach_to_open_removed(void)
}
#endif

+#ifndef FIO_HAVE_NATIVE_FALLOCATE
+static inline bool fio_fallocate(struct fio_file *f, uint64_t offset, uint64_t len)
+{
+ errno = ENOSYS;
+ return false;
+}
+#endif
+
#endif