Recent changes (master)

Jens Axboe
2020-03-27T12:00:16Z

The following changes since commit 3bd2078bdd1c173f9d02bc20e2d630302555a8a0:

zbd: add test for stressing zone locking (2020-03-17 20:05:54 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 983319d02626347d178b34320ac5835ae7ecad02:

Merge branch 'jsonplus2csv' of https://github.com/vincentkfu/fio (2020-03-26 09:03:40 -0600)

----------------------------------------------------------------
Jens Axboe (1):
Merge branch 'jsonplus2csv' of https://github.com/vincentkfu/fio

Vincent Fu (3):
tools/fio_jsonplus2csv: accommodate multiple lat measurements
t/jsonplus2csv_test.py: test script for tools/fio_jsonplus_clat2csv
.travis.yml: remove pip line from xcode11.2 config

.appveyor.yml | 2 +-
.travis.yml | 3 +-
t/jsonplus2csv_test.py | 150 ++++++++++++++++++
t/run-fio-tests.py | 8 +
tools/fio_jsonplus_clat2csv | 379 +++++++++++++++++++++++++++++++-------------
5 files changed, 428 insertions(+), 114 deletions(-)
create mode 100755 t/jsonplus2csv_test.py

---

Diff of recent changes:

diff --git a/.appveyor.yml b/.appveyor.yml
index bf0978ad..2f962c4b 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -15,7 +15,7 @@ environment:
install:
- '%CYG_ROOT%\setup-x86_64.exe --quiet-mode --no-shortcuts --only-site --site "%CYG_MIRROR%" --packages "mingw64-%PACKAGE_ARCH%-zlib,mingw64-%PACKAGE_ARCH%-CUnit" > NUL'
- SET PATH=C:\Python38-x64;%CYG_ROOT%\bin;%PATH% # NB: Changed env variables persist to later sections
- - python.exe -m pip install scipy
+ - python.exe -m pip install scipy six

build_script:
- 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --disable-native --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe'
diff --git a/.travis.yml b/.travis.yml
index 77c31b77..6b710cc3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,8 +48,9 @@ before_install:
brew install cunit;
if [[ "$TRAVIS_OSX_IMAGE" == "xcode11.2" ]]; then
pip3 install scipy;
+ else
+ pip install scipy;
fi;
- pip install scipy;
fi;
script:
- ./configure --extra-cflags="${EXTRA_CFLAGS}" && make
diff --git a/t/jsonplus2csv_test.py b/t/jsonplus2csv_test.py
new file mode 100755
index 00000000..2b34ef25
--- /dev/null
+++ b/t/jsonplus2csv_test.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (c) 2020 Western Digital Corporation or its affiliates.
+#
+"""
+jsonplus2csv-test.py
+
+Do one basic test of tools/fio_jsonplus2csv
+
+USAGE
+python jsonplus2csv-test.py [-f fio-executable] [-s script-location]
+
+EXAMPLES
+python t/jsonplus2csv-test.py
+python t/jsonplus2csv-test.py -f ./fio -s tools
+
+REQUIREMENTS
+Python 3.5+
+"""
+
+import os
+import sys
+import platform
+import argparse
+import subprocess
+
+
+def parse_args():
+ """Parse command-line arguments."""
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-f', '--fio',
+ help='path to fio executable (e.g., ./fio)')
+ parser.add_argument('-s', '--script',
+ help='directory containing fio_jsonplus2csv script')
+ return parser.parse_args()
+
+
+def run_fio(fio):
+ """Run fio to generate json+ data.
+
+ Parameters:
+ fio path to fio executable.
+ """
+
+ if platform.system() == 'Linux':
+ aio = 'libaio'
+ elif platform.system() == 'Windows':
+ aio = 'windowsaio'
+ else:
+ aio = 'posixaio'
+
+ fio_args = [
+ "--output=fio-output.json",
+ "--output-format=json+",
+ "--filename=fio_jsonplus_clat2csv.test",
+ "--ioengine=" + aio,
+ "--time_based",
+ "--runtime=3s",
+ "--size=1G",
+ "--slat_percentiles=1",
+ "--clat_percentiles=1",
+ "--lat_percentiles=1",
+ "--thread=1",
+ "--name=test1",
+ "--rw=randrw",
+ "--name=test2",
+ "--rw=read",
+ "--name=test3",
+ "--rw=write",
+ ]
+
+ output = subprocess.run([fio] + fio_args, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ return output
+
+
+def check_output(fio_output, script_path):
+ """Run t/fio_jsonplus_clat2csv and validate the generated CSV files
+ against the original json+ fio output.
+
+ Parameters:
+ fio_output subprocess.run object describing fio run.
+ script_path path to fio_jsonplus_clat2csv script.
+ """
+
+ if fio_output.returncode != 0:
+ return False
+
+ if platform.system() == 'Windows':
+ script = ['python.exe', script_path]
+ else:
+ script = [script_path]
+
+ script_args = ["fio-output.json", "fio-output.csv"]
+ script_args_validate = script_args + ["--validate"]
+
+ script_output = subprocess.run(script + script_args)
+ if script_output.returncode != 0:
+ return False
+
+ script_output = subprocess.run(script + script_args_validate)
+ if script_output.returncode != 0:
+ return False
+
+ return True
+
+
+def main():
+ """Entry point for this script."""
+
+ args = parse_args()
+
+ index = 1
+ passed = 0
+ failed = 0
+
+ if args.fio:
+ fio_path = args.fio
+ else:
+ fio_path = os.path.join(os.path.dirname(__file__), '../fio')
+ if not os.path.exists(fio_path):
+ fio_path = 'fio'
+ print("fio path is", fio_path)
+
+ if args.script:
+ script_path = args.script
+ else:
+ script_path = os.path.join(os.path.dirname(__file__), '../tools/fio_jsonplus_clat2csv')
+ if not os.path.exists(script_path):
+ script_path = 'fio_jsonplus_clat2csv'
+ print("script path is", script_path)
+
+ fio_output = run_fio(fio_path)
+ status = check_output(fio_output, script_path)
+ print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
+ if status:
+ passed = passed + 1
+ else:
+ failed = failed + 1
+ index = index + 1
+
+ print("{0} tests passed, {1} failed".format(passed, failed))
+
+ sys.exit(failed)
+
+if __name__ == '__main__':
+ main()
diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py
index 36fcb2f4..ea5abc4e 100755
--- a/t/run-fio-tests.py
+++ b/t/run-fio-tests.py
@@ -757,6 +757,14 @@ TEST_LIST = [
'success': SUCCESS_DEFAULT,
'requirements': [],
},
+ {
+ 'test_id': 1011,
+ 'test_class': FioExeTest,
+ 'exe': 't/jsonplus2csv_test.py',
+ 'parameters': ['-f', '{fio_path}'],
+ 'success': SUCCESS_DEFAULT,
+ 'requirements': [],
+ },
]

diff --git a/tools/fio_jsonplus_clat2csv b/tools/fio_jsonplus_clat2csv
index 78a007e5..9544ab74 100755
--- a/tools/fio_jsonplus_clat2csv
+++ b/tools/fio_jsonplus_clat2csv
@@ -1,76 +1,97 @@
#!/usr/bin/python2.7
# Note: this script is python2 and python3 compatible.
-#
-# fio_jsonplus_clat2csv
-#
-# This script converts fio's json+ completion latency data to CSV format.
-#
-# For example:
-#
-# Run the following fio jobs:
-# ../fio --output=fio-jsonplus.output --output-format=json+ --name=test1
-# --ioengine=null --time_based --runtime=5s --size=1G --rw=randrw
-# --name=test2 --ioengine=null --time_based --runtime=3s --size=1G
-# --rw=read --name=test3 --ioengine=null --time_based --runtime=4s
-# --size=8G --rw=write
-#
-# Then run:
-# fio_jsonplus_clat2csv fio-jsonplus.output fio-latency.csv
-#
-# You will end up with the following 3 files
-#
-# -rw-r--r-- 1 root root 6467 Jun 27 14:57 fio-latency_job0.csv
-# -rw-r--r-- 1 root root 3985 Jun 27 14:57 fio-latency_job1.csv
-# -rw-r--r-- 1 root root 4490 Jun 27 14:57 fio-latency_job2.csv
-#
-# fio-latency_job0.csv will look something like:
-#
-# clat_nsec, read_count, read_cumulative, read_percentile, write_count,
-# write_cumulative, write_percentile, trim_count, trim_cumulative,
-# trim_percentile,
-# 25, 1, 1, 1.50870705013e-07, , , , , , ,
-# 26, 12, 13, 1.96131916517e-06, 947, 947, 0.000142955890032, , , ,
-# 27, 843677, 843690, 0.127288105112, 838347, 839294, 0.126696959629, , , ,
-# 28, 1877982, 2721672, 0.410620573454, 1870189, 2709483, 0.409014312345, , , ,
-# 29, 4471, 2726143, 0.411295116376, 7718, 2717201, 0.410179395301, , , ,
-# 30, 2142885, 4869028, 0.734593687087, 2138164, 4855365, 0.732949340025, , , ,
-# ...
-# 2544, , , , 2, 6624404, 0.999997433738, , , ,
-# 2576, 3, 6628178, 0.99999788781, 4, 6624408, 0.999998037564, , , ,
-# 2608, 4, 6628182, 0.999998491293, 4, 6624412, 0.999998641391, , , ,
-# 2640, 3, 6628185, 0.999998943905, 2, 6624414, 0.999998943304, , , ,
-# 2672, 1, 6628186, 0.999999094776, 3, 6624417, 0.999999396174, , , ,
-# 2736, 1, 6628187, 0.999999245646, 1, 6624418, 0.99999954713, , , ,
-# 2768, 2, 6628189, 0.999999547388, 1, 6624419, 0.999999698087, , , ,
-# 2800, , , , 1, 6624420, 0.999999849043, , , ,
-# 2832, 1, 6628190, 0.999999698259, , , , , , ,
-# 4192, 1, 6628191, 0.999999849129, , , , , , ,
-# 5792, , , , 1, 6624421, 1.0, , , ,
-# 10304, 1, 6628192, 1.0, , , , , , ,
-#
-# The first line says that you had one read IO with 25ns clat,
-# the cumulative number of read IOs at or below 25ns is 1, and
-# 25ns is the 0.00001509th percentile for read latency
-#
-# The job had 2 write IOs complete in 2544ns,
-# 6624404 write IOs completed in 2544ns or less,
-# and this represents the 99.99974th percentile for write latency
-#
-# The last line says that one read IO had 10304ns clat,
-# 6628192 read IOs had 10304ns or shorter clat, and
-# 10304ns is the 100th percentile for read latency
-#
+
+"""
+fio_jsonplus_clat2csv
+
+This script converts fio's json+ latency data to CSV format.
+
+For example:
+
+Run the following fio jobs:
+$ fio --output=fio-jsonplus.output --output-format=json+ --ioengine=null \
+ --time_based --runtime=3s --size=1G --slat_percentiles=1 \
+ --clat_percentiles=1 --lat_percentiles=1 \
+ --name=test1 --rw=randrw \
+ --name=test2 --rw=read \
+ --name=test3 --rw=write
+
+Then run:
+$ fio_jsonplus_clat2csv fio-jsonplus.output fio-jsonplus.csv
+
+You will end up with the following 3 files:
+
+-rw-r--r-- 1 root root 77547 Mar 24 15:17 fio-jsonplus_job0.csv
+-rw-r--r-- 1 root root 65413 Mar 24 15:17 fio-jsonplus_job1.csv
+-rw-r--r-- 1 root root 63291 Mar 24 15:17 fio-jsonplus_job2.csv
+
+fio-jsonplus_job0.csv will look something like:
+
+nsec, read_slat_ns_count, read_slat_ns_cumulative, read_slat_ns_percentile, read_clat_ns_count, read_clat_ns_cumulative, read_clat_ns_percentile, read_lat_ns_count, read_lat_ns_cumulative, read_lat_ns_percentile, write_slat_ns_count, write_slat_ns_cumulative, write_slat_ns_percentile, write_clat_ns_count, write_clat_ns_cumulative, write_clat_ns_percentile, write_lat_ns_count, write_lat_ns_cumulative, write_lat_ns_percentile, trim_slat_ns_count, trim_slat_ns_cumulative, trim_slat_ns_percentile, trim_clat_ns_count, trim_clat_ns_cumulative, trim_clat_ns_percentile, trim_lat_ns_count, trim_lat_ns_cumulative, trim_lat_ns_percentile,
+12, , , , 3, 3, 6.11006798673e-07, , , , , , , 2, 2, 4.07580840603e-07, , , , , , , , , , , , ,
+13, , , , 1364, 1367, 0.000278415431262, , , , , , , 1776, 1778, 0.000362339367296, , , , , , , , , , , , ,
+14, , , , 181872, 183239, 0.037320091594, , , , , , , 207436, 209214, 0.0426358089929, , , , , , , , , , , , ,
+15, , , , 1574811, 1758050, 0.358060167469, , , , , , , 1661435, 1870649, 0.381220345946, , , , , , , , , , , , ,
+16, , , , 2198478, 3956528, 0.805821835713, , , , , , , 2154571, 4025220, 0.820301275606, , , , , , , , , , , , ,
+17, , , , 724335, 4680863, 0.953346372218, , , , , , , 645351, 4670571, 0.951817627138, , , , , , , , , , , , ,
+18, , , , 71837, 4752700, 0.96797733735, , , , , , , 61084, 4731655, 0.964265961171, , , , , , , , , , , , ,
+19, , , , 15915, 4768615, 0.971218728417, , , , , , , 18419, 4750074, 0.968019576923, , , , , , , , , , , , ,
+20, , , , 12651, 4781266, 0.973795344087, , , , , , , 14176, 4764250, 0.970908509921, , , , , , , , , , , , ,
+...
+168960, , , , , , , , , , , , , 1, 4906999, 0.999999388629, 1, 4906997, 0.999998981048, , , , , , , , , ,
+177152, , , , , , , , , , , , , 1, 4907000, 0.999999592419, 1, 4906998, 0.999999184838, , , , , , , , , ,
+183296, , , , , , , , , , , , , 1, 4907001, 0.99999979621, 1, 4906999, 0.999999388629, , , , , , , , , ,
+189440, , , , , , , 1, 4909925, 0.999999185324, , , , , , , , , , , , , , , , , , ,
+214016, , , , 1, 4909928, 0.999999796331, 2, 4909927, 0.999999592662, , , , , , , , , , , , , , , , , , ,
+246784, , , , , , , , , , , , , , , , 1, 4907000, 0.999999592419, , , , , , , , , ,
+272384, , , , 1, 4909929, 1.0, 1, 4909928, 0.999999796331, , , , , , , , , , , , , , , , , , ,
+329728, , , , , , , , , , , , , 1, 4907002, 1.0, 1, 4907001, 0.99999979621, , , , , , , , , ,
+1003520, , , , , , , , , , , , , , , , 1, 4907002, 1.0, , , , , , , , , ,
+1089536, , , , , , , 1, 4909929, 1.0, , , , , , , , , , , , , , , , , , ,
+
+The first line says that there were three read IOs with 12ns clat,
+the cumulative number of read IOs at or below 12ns was two, and
+12ns was the 0.0000611th percentile for read latency. There were
+two write IOs with 12ns clat, the cumulative number of write IOs
+at or below 12ns was two, and 12ns was the 0.0000408th percentile
+for write latency.
+
+The job had one write IO complete at 168960ns and 4906999 write IOs
+completed at or below this duration. Also this duration was the
+99.99994th percentile for write latency. There was one write IO
+with a total latency of 168960ns, this duration had a cumulative
+frequency of 4906997 write IOs and was the 99.9998981048th percentile
+for write total latency.
+
+The last line says that one read IO had 1089536ns total latency, this
+duration had a cumulative frequency of 4909929 and represented the 100th
+percentile for read total latency.
+
+Running the following:
+
+$ fio_jsonplus_clat2csv fio-jsonplus.output fio-jsonplus.csv --validate
+fio-jsonplus_job0.csv validated
+fio-jsonplus_job1.csv validated
+fio-jsonplus_job2.csv validated
+
+will check the CSV data against the json+ output to confirm that the CSV
+data matches.
+"""

from __future__ import absolute_import
from __future__ import print_function
import os
import json
import argparse
+import itertools
import six
-from six.moves import range

+DDIR_LIST = ['read', 'write', 'trim']
+LAT_LIST = ['slat_ns', 'clat_ns', 'lat_ns']

def parse_args():
+ """Parse command-line arguments."""
+
parser = argparse.ArgumentParser()
parser.add_argument('source',
help='fio json+ output file containing completion '
@@ -78,12 +99,26 @@ def parse_args():
parser.add_argument('dest',
help='destination file stub for latency data in CSV '
'format. job number will be appended to filename')
+ parser.add_argument('--debug', '-d', action='store_true',
+ help='enable debug prints')
+ parser.add_argument('--validate', action='store_true',
+ help='validate CSV against JSON output')
args = parser.parse_args()

return args

def percentile(idx, run_total):
+ """Return a percentile for a specified index based on a running total.
+
+ Parameters:
+ idx index for which to generate percentile.
+ run_total list of cumulative sums.
+
+ Returns:
+ Percentile represented by the specified index.
+ """
+
total = run_total[len(run_total)-1]
if total == 0:
return 0
@@ -91,7 +126,18 @@ def percentile(idx, run_total):
return float(run_total[idx]) / total

-def more_lines(indices, bins):
+def more_bins(indices, bins):
+ """Determine whether we have more bins to process.
+
+ Parameters:
+ indices a dict containing the last index processed in each bin.
+ bins a dict contaiing a set of bins to process.
+
+ Returns:
+ True if the indices do not yet point to the end of each bin in bins.
+ False if the indices point beyond their repsective bins.
+ """
+
for key, value in six.iteritems(indices):
if value < len(bins[key]):
return True
@@ -99,78 +145,187 @@ def more_lines(indices, bins):
return False

+def debug_print(debug, *args):
+ """Print debug messages.
+
+ Parameters:
+ debug emit messages if True.
+ *args arguments for print().
+ """
+
+ if debug:
+ print(*args)
+
+
+def get_csvfile(dest, jobnum):
+ """Generate CSV filename from command-line arguments and job numbers.
+
+ Paramaters:
+ dest file specification for CSV filename.
+ jobnum job number.
+
+ Returns:
+ A string that is a new filename that incorporates the job number.
+ """
+
+ stub, ext = os.path.splitext(dest)
+ return stub + '_job' + str(jobnum) + ext
+
+
+def validate(args, jsondata, col_labels):
+ """Validate CSV data against json+ output.
+
+ This function checks the CSV data to make sure that it was correctly
+ generated from the original json+ output. json+ 'bins' objects are
+ constructed from the CSV data and then compared to the corresponding
+ objects in the json+ data. An AssertionError will appear if a mismatch
+ is found.
+
+ Percentiles and cumulative counts are not checked.
+
+ Parameters:
+ args command-line arguments for this script.
+ jsondata json+ output to compare against.
+ col_labels column labels for CSV data.
+
+ Returns
+ 0 if no mismatches found.
+ """
+
+ colnames = [c.strip() for c in col_labels.split(',')]
+
+ for jobnum in range(len(jsondata['jobs'])):
+ job_data = jsondata['jobs'][jobnum]
+ csvfile = get_csvfile(args.dest, jobnum)
+
+ with open(csvfile, 'r') as csvsource:
+ csvlines = csvsource.read().split('\n')
+
+ assert csvlines[0] == col_labels
+ debug_print(args.debug, 'col_labels match for', csvfile)
+
+ # create 'bins' objects from the CSV data
+ counts = {}
+ for ddir in DDIR_LIST:
+ counts[ddir] = {}
+ for lat in LAT_LIST:
+ counts[ddir][lat] = {}
+
+ csvlines.pop(0)
+ for line in csvlines:
+ if line.strip() == "":
+ continue
+ values = line.split(',')
+ nsec = values[0]
+ for col in colnames:
+ if 'count' in col:
+ val = values[colnames.index(col)]
+ if val.strip() != "":
+ count = int(val)
+ ddir, lat, _, _ = col.split('_')
+ lat = lat + '_ns'
+ counts[ddir][lat][nsec] = count
+ try:
+ assert count == job_data[ddir][lat]['bins'][nsec]
+ except Exception:
+ print("mismatch:", csvfile, ddir, lat, nsec, "ns")
+ return 1
+
+ # compare 'bins' objects created from the CSV data
+ # with corresponding 'bins' objects in the json+ output
+ for ddir in DDIR_LIST:
+ for lat in LAT_LIST:
+ if lat in job_data[ddir] and 'bins' in job_data[ddir][lat]:
+ assert job_data[ddir][lat]['bins'] == counts[ddir][lat]
+ debug_print(args.debug, csvfile, ddir, lat, "bins match")
+ else:
+ assert counts[ddir][lat] == {}
+ debug_print(args.debug, csvfile, ddir, lat, "bins empty")
+
+ print(csvfile, "validated")
+
+ return 0
+
+
def main():
+ """Starting point for this script.
+
+ In standard mode, this script will generate CSV data from fio json+ output.
+ In validation mode it will check to make sure that counts in CSV files
+ match the counts in the json+ data.
+ """
+
args = parse_args()

with open(args.source, 'r') as source:
jsondata = json.loads(source.read())

+ ddir_lat_list = list(ddir + '_' + lat for ddir, lat in itertools.product(DDIR_LIST, LAT_LIST))
+ debug_print(args.debug, 'ddir_lat_list: ', ddir_lat_list)
+ col_labels = 'nsec, '
+ for ddir_lat in ddir_lat_list:
+ col_labels += "{0}_count, {0}_cumulative, {0}_percentile, ".format(ddir_lat)
+ debug_print(args.debug, 'col_labels: ', col_labels)
+
+ if args.validate:
+ return validate(args, jsondata, col_labels)
+
for jobnum in range(0, len(jsondata['jobs'])):
bins = {}
run_total = {}
- ddir_set = set(['read', 'write', 'trim'])
-
- prev_ddir = None
- for ddir in ddir_set:
- if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']:
- bins_loc = 'clat_ns'
- elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']:
- bins_loc = 'lat_ns'
- else:
- raise RuntimeError("Latency bins not found. "
- "Are you sure you are using json+ output?")
-
- bins[ddir] = [[int(key), value] for key, value in
- six.iteritems(jsondata['jobs'][jobnum][ddir][bins_loc]
- ['bins'])]
- bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0])
-
- run_total[ddir] = [0 for x in range(0, len(bins[ddir]))]
- if len(bins[ddir]) > 0:
- run_total[ddir][0] = bins[ddir][0][1]
- for x in range(1, len(bins[ddir])):
- run_total[ddir][x] = run_total[ddir][x-1] + \
- bins[ddir][x][1]
-
- stub, ext = os.path.splitext(args.dest)
- outfile = stub + '_job' + str(jobnum) + ext
-
- with open(outfile, 'w') as output:
- output.write("{0}ec, ".format(bins_loc))
- ddir_list = list(ddir_set)
- for ddir in ddir_list:
- output.write("{0}_count, {0}_cumulative, {0}_percentile, ".
- format(ddir))
- output.write("\n")
+
+ for ddir in DDIR_LIST:
+ ddir_data = jsondata['jobs'][jobnum][ddir]
+ for lat in LAT_LIST:
+ ddir_lat = ddir + '_' + lat
+ if lat not in ddir_data or 'bins' not in ddir_data[lat]:
+ bins[ddir_lat] = []
+ debug_print(args.debug, 'job', jobnum, ddir_lat, 'not found')
+ continue
+
+ debug_print(args.debug, 'job', jobnum, ddir_lat, 'processing')
+ bins[ddir_lat] = [[int(key), value] for key, value in
+ six.iteritems(ddir_data[lat]['bins'])]
+ bins[ddir_lat] = sorted(bins[ddir_lat], key=lambda bin: bin[0])
+
+ run_total[ddir_lat] = [0 for x in range(0, len(bins[ddir_lat]))]
+ run_total[ddir_lat][0] = bins[ddir_lat][0][1]
+ for index in range(1, len(bins[ddir_lat])):
+ run_total[ddir_lat][index] = run_total[ddir_lat][index-1] + \
+ bins[ddir_lat][index][1]
+
+ csvfile = get_csvfile(args.dest, jobnum)
+ with open(csvfile, 'w') as output:
+ output.write(col_labels + "\n")

#
-# Have a counter for each ddir
+# Have a counter for each ddir_lat pairing
# In each round, pick the shortest remaining duration
# and output a line with any values for that duration
#
- indices = {x: 0 for x in ddir_list}
- while more_lines(indices, bins):
+ indices = {x: 0 for x in ddir_lat_list}
+ while more_bins(indices, bins):
+ debug_print(args.debug, 'indices: ', indices)
min_lat = 17112760320
- for ddir in ddir_list:
- if indices[ddir] < len(bins[ddir]):
- min_lat = min(bins[ddir][indices[ddir]][0], min_lat)
+ for ddir_lat in ddir_lat_list:
+ if indices[ddir_lat] < len(bins[ddir_lat]):
+ min_lat = min(bins[ddir_lat][indices[ddir_lat]][0], min_lat)

output.write("{0}, ".format(min_lat))

- for ddir in ddir_list:
- if indices[ddir] < len(bins[ddir]) and \
- min_lat == bins[ddir][indices[ddir]][0]:
- count = bins[ddir][indices[ddir]][1]
- cumulative = run_total[ddir][indices[ddir]]
- ptile = percentile(indices[ddir], run_total[ddir])
- output.write("{0}, {1}, {2}, ".format(count,
- cumulative, ptile))
- indices[ddir] += 1
+ for ddir_lat in ddir_lat_list:
+ if indices[ddir_lat] < len(bins[ddir_lat]) and \
+ min_lat == bins[ddir_lat][indices[ddir_lat]][0]:
+ count = bins[ddir_lat][indices[ddir_lat]][1]
+ cumulative = run_total[ddir_lat][indices[ddir_lat]]
+ ptile = percentile(indices[ddir_lat], run_total[ddir_lat])
+ output.write("{0}, {1}, {2}, ".format(count, cumulative, ptile))
+ indices[ddir_lat] += 1
else:
output.write(", , , ")
output.write("\n")

- print("{0} generated".format(outfile))
+ print("{0} generated".format(csvfile))