Recent changes (master)

Jens Axboe
2020-05-21T12:00:09Z

The following changes since commit 26949a8bbc544094740c61f79b4ea778978e7d8e:

Merge branch '32-bit-fixes' of https://github.com/sitsofe/fio (2020-05-19 16:14:19 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 0e21f5c6f64a73bcede20e1a29a885845e453b8e:

Merge branch 'testing' of https://github.com/vincentkfu/fio (2020-05-20 14:19:12 -0600)

----------------------------------------------------------------
Jens Axboe (2):
Merge branch 'issue-989' of https://github.com/Nordix/fio
Merge branch 'testing' of https://github.com/vincentkfu/fio

Lars Ekman (1):
Corrected scope of for-loop

Vincent Fu (6):
t/jsonplus2csv_test: reduce file size
t/run-fio-tests: better catch file errors
t/latency_percentiles: run cmdprio_percentage tests only if root
docs: update cmdprio_percentage with note about root user
testing: use max-jobs to speed up testing
t/zbd: improve error handling for test scripts

HOWTO | 3 ++-
fio.1 | 3 ++-
t/jsonplus2csv_test.py | 5 ++++-
t/latency_percentiles.py | 5 +++--
t/readonly.py | 1 +
t/run-fio-tests.py | 37 +++++++++++++++++++++++------------
t/steadystate_tests.py | 2 +-
t/strided.py | 1 +
t/zbd/run-tests-against-regular-nullb | 4 ++--
t/zbd/run-tests-against-zoned-nullb | 2 +-
tools/plot/fio2gnuplot | 8 ++++----
11 files changed, 45 insertions(+), 26 deletions(-)

---

Diff of recent changes:

diff --git a/HOWTO b/HOWTO
index 430c7b62..cd628552 100644
--- a/HOWTO
+++ b/HOWTO
@@ -2040,7 +2040,8 @@ with the caveat that when used on the command line, they must come after the
the priority bit. Non-read I/O is likely unaffected by ``cmdprio_percentage``.
This option cannot be used with the `prio` or `prioclass` options. For this
option to set the priority bit properly, NCQ priority must be supported and
- enabled and :option:`direct`\=1 option must be used.
+ enabled and :option:`direct`\=1 option must be used. fio must also be run as
+ the root user.

.. option:: fixedbufs : [io_uring]

diff --git a/fio.1 b/fio.1
index a2379f98..9e9e1b1a 100644
--- a/fio.1
+++ b/fio.1
@@ -1806,7 +1806,8 @@ Set the percentage of I/O that will be issued with higher priority by setting
the priority bit. Non-read I/O is likely unaffected by ``cmdprio_percentage``.
This option cannot be used with the `prio` or `prioclass` options. For this
option to set the priority bit properly, NCQ priority must be supported and
-enabled and `direct=1' option must be used.
+enabled and `direct=1' option must be used. fio must also be run as the root
+user.
.TP
.BI (io_uring)fixedbufs
If fio is asked to do direct IO, then Linux will map pages for each IO call, and
diff --git a/t/jsonplus2csv_test.py b/t/jsonplus2csv_test.py
index 2b34ef25..f01a9f32 100755
--- a/t/jsonplus2csv_test.py
+++ b/t/jsonplus2csv_test.py
@@ -44,6 +44,7 @@ def run_fio(fio):
fio path to fio executable.
"""

+# We need an async ioengine to get submission latencies
if platform.system() == 'Linux':
aio = 'libaio'
elif platform.system() == 'Windows':
@@ -52,13 +53,14 @@ def run_fio(fio):
aio = 'posixaio'

fio_args = [
+ "--max-jobs=4",
"--output=fio-output.json",
"--output-format=json+",
"--filename=fio_jsonplus_clat2csv.test",
"--ioengine=" + aio,
"--time_based",
"--runtime=3s",
- "--size=1G",
+ "--size=1M",
"--slat_percentiles=1",
"--clat_percentiles=1",
"--lat_percentiles=1",
@@ -87,6 +89,7 @@ def check_output(fio_output, script_path):
"""

if fio_output.returncode != 0:
+ print("ERROR: fio run failed")
return False

if platform.system() == 'Windows':
diff --git a/t/latency_percentiles.py b/t/latency_percentiles.py
index 5cdd49cf..6ce4579a 100755
--- a/t/latency_percentiles.py
+++ b/t/latency_percentiles.py
@@ -109,6 +109,7 @@ class FioLatTest():
"""Run a test."""

fio_args = [
+ "--max-jobs=16",
"--name=latency",
"--randrepeat=0",
"--norandommap",
@@ -1303,9 +1304,9 @@ def main():
(args.run_only and test['test_id'] not in args.run_only):
skipped = skipped + 1
outcome = 'SKIPPED (User request)'
- elif platform.system() != 'Linux' and 'cmdprio_percentage' in test:
+ elif (platform.system() != 'Linux' or os.geteuid() != 0) and 'cmdprio_percentage' in test:
skipped = skipped + 1
- outcome = 'SKIPPED (Linux required for cmdprio_percentage tests)'
+ outcome = 'SKIPPED (Linux root required for cmdprio_percentage tests)'
else:
test_obj = test['test_obj'](artifact_root, test, args.debug)
status = test_obj.run_fio(fio)
diff --git a/t/readonly.py b/t/readonly.py
index 43686c9c..464847c6 100755
--- a/t/readonly.py
+++ b/t/readonly.py
@@ -36,6 +36,7 @@ def parse_args():

def run_fio(fio, test, index):
fio_args = [
+ "--max-jobs=16",
"--name=readonly",
"--ioengine=null",
"--time_based",
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 8e326ed5..763e0103 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -226,6 +226,7 @@ class FioJobTest(FioExeTest):
self.json_data = None
self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
self.fio_args = [
+ "--max-jobs=16",
"--output-format={0}".format(self.output_format),
"--output={0}".format(self.fio_output),
self.fio_job,
@@ -273,6 +274,20 @@ class FioJobTest(FioExeTest):
else:
logging.debug("Test %d: precondition step failed", self.testnum)

+ @classmethod
+ def get_file(cls, filename):
+ """Safely read a file."""
+ file_data = ''
+ success = True
+
+ try:
+ with open(filename, "r") as output_file:
+ file_data = output_file.read()
+ except OSError:
+ success = False
+
+ return file_data, success
+
def check_result(self):
"""Check fio job results."""

@@ -289,10 +304,8 @@ class FioJobTest(FioExeTest):
if 'json' not in self.output_format:
return

- try:
- with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
- file_data = output_file.read()
- except EnvironmentError:
+ file_data, success = self.get_file(os.path.join(self.test_dir, self.fio_output))
+ if not success:
self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
self.passed = False
return
@@ -457,11 +470,9 @@ class Requirements(object):
Requirements._linux = platform.system() == "Linux"

if Requirements._linux:
- try:
- config_file = os.path.join(fio_root, "config-host.h")
- with open(config_file, "r") as config:
- contents = config.read()
- except Exception:
+ config_file = os.path.join(fio_root, "config-host.h")
+ contents, success = FioJobTest.get_file(config_file)
+ if not success:
print("Unable to open {0} to check requirements".format(config_file))
Requirements._zbd = True
else:
@@ -899,10 +910,10 @@ def main():
else:
result = "FAILED: {0}".format(test.failure_reason)
failed = failed + 1
- with open(test.stderr_file, "r") as stderr_file:
- logging.debug("Test %d: stderr:\n%s", config['test_id'], stderr_file.read())
- with open(test.stdout_file, "r") as stdout_file:
- logging.debug("Test %d: stdout:\n%s", config['test_id'], stdout_file.read())
+ contents, _ = FioJobTest.get_file(test.stderr_file)
+ logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
+ contents, _ = FioJobTest.get_file(test.stdout_file)
+ logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
print("Test {0} {1}".format(config['test_id'], result))

print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
diff --git a/t/steadystate_tests.py b/t/steadystate_tests.py
index b55a67ac..e99b655c 100755
--- a/t/steadystate_tests.py
+++ b/t/steadystate_tests.py
@@ -122,7 +122,7 @@ if __name__ == '__main__':
for job in reads:

tf = "steadystate_job{0}.json".format(jobnum)
- parameters = [ "--name=job{0}".format(jobnum) ]
+ parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ]
parameters.extend([ "--thread",
"--output-format=json",
"--output={0}".format(tf),
diff --git a/t/strided.py b/t/strided.py
index aac15d10..6d34be8a 100755
--- a/t/strided.py
+++ b/t/strided.py
@@ -52,6 +52,7 @@ def parse_args():
def run_fio(fio, test, index):
filename = "strided"
fio_args = [
+ "--max-jobs=16",
"--name=strided",
"--zonemode=strided",
"--log_offset=1",
diff --git a/t/zbd/run-tests-against-regular-nullb b/t/zbd/run-tests-against-regular-nullb
index 0f6e4b66..5b7b4009 100755
--- a/t/zbd/run-tests-against-regular-nullb
+++ b/t/zbd/run-tests-against-regular-nullb
@@ -8,7 +8,7 @@ scriptdir="$(cd "$(dirname "$0")" && pwd)"

for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d"; done
modprobe -r null_blk
-modprobe null_blk nr_devices=0 || return $?
+modprobe null_blk nr_devices=0 || exit $?
for d in /sys/kernel/config/nullb/*; do
[ -d "$d" ] && rmdir "$d"
done
@@ -22,6 +22,6 @@ modprobe null_blk nr_devices=0 &&
echo 4096 > blocksize &&
echo 1024 > size &&
echo 1 > memory_backed &&
- echo 1 > power
+ echo 1 > power || exit $?

"${scriptdir}"/test-zbd-support "$@" /dev/nullb0
diff --git a/t/zbd/run-tests-against-zoned-nullb b/t/zbd/run-tests-against-zoned-nullb
index 0952011c..53aee3e8 100755
--- a/t/zbd/run-tests-against-zoned-nullb
+++ b/t/zbd/run-tests-against-zoned-nullb
@@ -8,7 +8,7 @@ scriptdir="$(cd "$(dirname "$0")" && pwd)"

for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d"; done
modprobe -r null_blk
-modprobe null_blk nr_devices=0 || return $?
+modprobe null_blk nr_devices=0 || exit $?
for d in /sys/kernel/config/nullb/*; do
[ -d "$d" ] && rmdir "$d"
done
diff --git a/tools/plot/fio2gnuplot b/tools/plot/fio2gnuplot
index cc4ea4c7..69aa791e 100755
--- a/tools/plot/fio2gnuplot
+++ b/tools/plot/fio2gnuplot
@@ -93,10 +93,10 @@ set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green"
compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg));
compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg));

- pos=0
- # Let's create a temporary file for each selected fio file
- for file in fio_data_file:
- tmp_filename = "gnuplot_temp_file.%d" % pos
+ pos=0
+ # Let's create a temporary file for each selected fio file
+ for file in fio_data_file:
+ tmp_filename = "gnuplot_temp_file.%d" % pos

# Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces
if len(fio_data_file) > 1: