Recent changes (master)

Jens Axboe
2020-09-15T12:00:08Z

The following changes since commit 695611a9d4cd554d44d8b2ec5da2811061950a2e:

Allow offload with FAKEIO engines (2020-09-11 09:58:15 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 7064f8942a3b8070acf60b8e5fabc16f8221ae40:

Merge branch 'msys2' of https://github.com/sitsofe/fio into master (2020-09-14 19:43:39 -0600)

----------------------------------------------------------------
Bart Van Assche (2):
backend: Remove two superfluous casts
backend: Use asprintf() instead of strlen() + sprintf()

Jens Axboe (2):
Merge branch 'backend' of https://github.com/bvanassche/fio into master
Merge branch 'msys2' of https://github.com/sitsofe/fio into master

Sitsofe Wheeler (12):
configure: pass non-null pointer to (v)asprintf
net: coerce the result of htonl before printing
windows: fix wrong format strings
windows: fix DWORD format string complaints
configure: be explicit about "XP" Windows API version
configure/Makefile: add option to generate pdb symbols
Makefile/ci: Don't pass CFLAGS when linking
appveyor: cleanup and add separate install script
Makefile: introduce FIO_CFLAGS
memlock: avoid type confusion in format string
configure: cleanup lex/yacc tests
travis: cleanup build script

.appveyor.yml | 40 ++++++++++++++++--------
Makefile | 77 ++++++++++++++++++++++++++---------------------
backend.c | 15 ++++-----
ci/appveyor-install.sh | 41 +++++++++++++++++++++++++
ci/travis-build.sh | 8 +++--
ci/travis-install.sh | 5 ++-
configure | 60 ++++++++++++++++++++++++++----------
engines/net.c | 5 +--
engines/windowsaio.c | 8 ++---
os/windows/cpu-affinity.c | 26 ++++++++--------
os/windows/dobuild.cmd | 10 ++++--
os/windows/install.wxs | 8 +++++
os/windows/posix.c | 5 +--
t/memlock.c | 2 +-
14 files changed, 211 insertions(+), 99 deletions(-)
create mode 100755 ci/appveyor-install.sh

---

Diff of recent changes:

diff --git a/.appveyor.yml b/.appveyor.yml
index 352caeee..fad16326 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -5,33 +5,49 @@ image:

environment:
CYG_MIRROR: http://cygwin.mirror.constant.com
- CYG_ROOT: C:\cygwin64
- MAKEFLAGS: -j 2
matrix:
- - platform: x64
- PACKAGE_ARCH: x86_64
+ - ARCHITECTURE: x64
+ CC: clang
+ CONFIGURE_OPTIONS: --enable-pdb
+ DISTRO: msys2
+# Skip 32 bit clang build
+# - ARCHITECTURE: x86
+# CC: clang
+# CONFIGURE_OPTIONS: --enable-pdb
+# DISTRO: msys2
+ - ARCHITECTURE: x64
CONFIGURE_OPTIONS:
- - platform: x86
- PACKAGE_ARCH: i686
+ DISTRO: cygwin
+ - ARCHITECTURE: x86
CONFIGURE_OPTIONS: --build-32bit-win --target-win-ver=xp
+ DISTRO: cygwin

install:
- - '%CYG_ROOT%\setup-x86_64.exe --quiet-mode --no-shortcuts --only-site --site "%CYG_MIRROR%" --packages "mingw64-%PACKAGE_ARCH%-zlib,mingw64-%PACKAGE_ARCH%-CUnit" > NUL'
- - SET PATH=C:\Python38-x64;%CYG_ROOT%\bin;%PATH% # NB: Changed env variables persist to later sections
+ - if %DISTRO%==cygwin (
+ SET "PATH=C:\cygwin64\bin;C:\cygwin64;%PATH%"
+ )
+ - if %DISTRO%==msys2 if %ARCHITECTURE%==x86 (
+ SET "PATH=C:\msys64\mingw32\bin;C:\msys64\usr\bin;%PATH%"
+ )
+ - if %DISTRO%==msys2 if %ARCHITECTURE%==x64 (
+ SET "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%"
+ )
+ - SET PATH=C:\Python38-x64;%PATH% # NB: Changed env variables persist to later sections
- SET PYTHONUNBUFFERED=TRUE
- - python.exe -m pip install scipy six
+ - bash.exe ci\appveyor-install.sh

build_script:
- - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --disable-native --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe'
+ - bash.exe configure --extra-cflags=-Werror --disable-native %CONFIGURE_OPTIONS%
+ - make.exe -j2

after_build:
- file.exe fio.exe
- make.exe test
- - 'cd os\windows && dobuild.cmd %PLATFORM% && cd ..'
+ - 'cd os\windows && dobuild.cmd %ARCHITECTURE% && cd ..'
- ps: Get-ChildItem .\os\windows\*.msi | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name -DeploymentName fio.msi }

test_script:
- - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -f fio.exe ] && python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug'
+ - python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug

on_finish:
- 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -d test-artifacts ] && 7z a -t7z test-artifacts.7z test-artifacts -xr!foo.0.0 -xr!latency.?.0 -xr!fio_jsonplus_clat2csv.test && appveyor PushArtifact test-artifacts.7z'
diff --git a/Makefile b/Makefile
index b00daca2..5ed8a808 100644
--- a/Makefile
+++ b/Makefile
@@ -22,16 +22,22 @@ endif
DEBUGFLAGS = -DFIO_INC_DEBUG
CPPFLAGS= -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DFIO_INTERNAL $(DEBUGFLAGS)
OPTFLAGS= -g -ffast-math
-CFLAGS := -std=gnu99 -Wwrite-strings -Wall -Wdeclaration-after-statement $(OPTFLAGS) $(EXTFLAGS) $(BUILD_CFLAGS) -I. -I$(SRCDIR) $(CFLAGS)
+FIO_CFLAGS= -std=gnu99 -Wwrite-strings -Wall -Wdeclaration-after-statement $(OPTFLAGS) $(EXTFLAGS) $(BUILD_CFLAGS) -I. -I$(SRCDIR)
LIBS += -lm $(EXTLIBS)
PROGS = fio
SCRIPTS = $(addprefix $(SRCDIR)/,tools/fio_generate_plots tools/plot/fio2gnuplot tools/genfio tools/fiologparser.py tools/hist/fiologparser_hist.py tools/hist/fio-histo-log-pctiles.py tools/fio_jsonplus_clat2csv)

ifndef CONFIG_FIO_NO_OPT
- CFLAGS := -O3 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $(CFLAGS)
+ FIO_CFLAGS += -O3 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2
endif
ifdef CONFIG_BUILD_NATIVE
- CFLAGS := -march=native $(CFLAGS)
+ FIO_CFLAGS += -march=native
+endif
+
+ifdef CONFIG_PDB
+ LINK_PDBFILE ?= -Wl,-pdb,$(dir $@)/$(basename $(@F)).pdb
+ FIO_CFLAGS += -gcodeview
+ LDFLAGS += -fuse-ld=lld $(LINK_PDBFILE)
endif

ifdef CONFIG_GFIO
@@ -55,7 +61,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
ifdef CONFIG_LIBHDFS
HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
HDFSLIB= -Wl,-rpath $(JAVA_HOME)/jre/lib/$(FIO_HDFS_CPU)/server -L$(JAVA_HOME)/jre/lib/$(FIO_HDFS_CPU)/server $(FIO_LIBHDFS_LIB)/libhdfs.a -ljvm
- CFLAGS := $(HDFSFLAGS) $(CFLAGS)
+ FIO_CFLAGS += $(HDFSFLAGS)
SOURCE += engines/libhdfs.c
endif

@@ -74,10 +80,9 @@ ifdef CONFIG_LIBNBD
endif

ifdef CONFIG_64BIT
- CFLAGS := -DBITS_PER_LONG=64 $(CFLAGS)
-endif
-ifdef CONFIG_32BIT
- CFLAGS := -DBITS_PER_LONG=32 $(CFLAGS)
+ CPPFLAGS += -DBITS_PER_LONG=64
+else ifdef CONFIG_32BIT
+ CPPFLAGS += -DBITS_PER_LONG=32
endif
ifdef CONFIG_LIBAIO
aio_SRCS = engines/libaio.c
@@ -155,7 +160,7 @@ ifdef CONFIG_GFAPI
SOURCE += engines/glusterfs_async.c
LIBS += -lgfapi -lglusterfs
ifdef CONFIG_GF_FADVISE
- CFLAGS := "-DGFAPI_USE_FADVISE" $(CFLAGS)
+ FIO_CFLAGS += "-DGFAPI_USE_FADVISE"
endif
endif
ifdef CONFIG_MTD
@@ -234,7 +239,7 @@ ifeq ($(CONFIG_TARGET_OS), AIX)
endif
ifeq ($(CONFIG_TARGET_OS), HP-UX)
LIBS += -lpthread -ldl -lrt
- CFLAGS := -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE_EXTENDED $(CFLAGS)
+ FIO_CFLAGS += -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE_EXTENDED
endif
ifeq ($(CONFIG_TARGET_OS), Darwin)
LIBS += -lpthread -ldl
@@ -243,24 +248,21 @@ ifneq (,$(findstring CYGWIN,$(CONFIG_TARGET_OS)))
SOURCE += os/windows/cpu-affinity.c os/windows/posix.c
WINDOWS_OBJS = os/windows/cpu-affinity.o os/windows/posix.o lib/hweight.o
LIBS += -lpthread -lpsapi -lws2_32 -lssp
- CFLAGS := -DPSAPI_VERSION=1 -Ios/windows/posix/include -Wno-format $(CFLAGS)
+ FIO_CFLAGS += -DPSAPI_VERSION=1 -Ios/windows/posix/include -Wno-format
endif

ifdef CONFIG_DYNAMIC_ENGINES
DYNAMIC_ENGS := $(ENGINES)
define engine_template =
$(1)_OBJS := $$($(1)_SRCS:.c=.o)
-$$($(1)_OBJS): CFLAGS := -fPIC $$($(1)_CFLAGS) $(CFLAGS)
-engines/lib$(1).so: $$($(1)_OBJS)
- $$(QUIET_LINK)$(CC) -shared -rdynamic -fPIC -Wl,-soname,lib$(1).so.1 $$($(1)_LIBS) -o $$@ $$<
+$$($(1)_OBJS): FIO_CFLAGS += -fPIC $$($(1)_CFLAGS)
ENGS_OBJS += engines/lib$(1).so
-all install: $(ENGS_OBJS)
endef
else # !CONFIG_DYNAMIC_ENGINES
define engine_template =
SOURCE += $$($(1)_SRCS)
LIBS += $$($(1)_LIBS)
-CFLAGS := $$($(1)_CFLAGS) $(CFLAGS)
+CFLAGS += $$($(1)_CFLAGS)
endef
endif

@@ -427,7 +429,7 @@ mandir = $(prefix)/man
sharedir = $(prefix)/share/fio
endif

-all: $(PROGS) $(T_TEST_PROGS) $(UT_PROGS) $(SCRIPTS) FORCE
+all: $(PROGS) $(T_TEST_PROGS) $(UT_PROGS) $(SCRIPTS) $(ENGS_OBJS) FORCE

.PHONY: all install clean test
.PHONY: FORCE cscope
@@ -436,7 +438,7 @@ FIO-VERSION-FILE: FORCE
@$(SHELL) $(SRCDIR)/FIO-VERSION-GEN
-include FIO-VERSION-FILE

-override CFLAGS := -DFIO_VERSION='"$(FIO_VERSION)"' $(CFLAGS)
+override CFLAGS := -DFIO_VERSION='"$(FIO_VERSION)"' $(FIO_CFLAGS) $(CFLAGS)

%.o : %.c
@mkdir -p $(dir $@)
@@ -478,7 +480,7 @@ lexer.h: lex.yy.c
exp/test-expression-parser.o: exp/test-expression-parser.c
$(QUIET_CC)$(CC) -o $@ $(CFLAGS) $(CPPFLAGS) -c $<
exp/test-expression-parser: exp/test-expression-parser.o
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) $< y.tab.o lex.yy.o -o $@ $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) $< y.tab.o lex.yy.o -o $@ $(LIBS)

parse.o: lex.yy.o y.tab.o
endif
@@ -514,55 +516,60 @@ printing.o: printing.c printing.h

t/io_uring.o: os/linux/io_uring.h
t/io_uring: $(T_IOU_RING_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_IOU_RING_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_IOU_RING_OBJS) $(LIBS)

t/read-to-pipe-async: $(T_PIPE_ASYNC_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_PIPE_ASYNC_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_PIPE_ASYNC_OBJS) $(LIBS)

t/memlock: $(T_MEMLOCK_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_MEMLOCK_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_MEMLOCK_OBJS) $(LIBS)

t/stest: $(T_SMALLOC_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_SMALLOC_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_SMALLOC_OBJS) $(LIBS)

t/ieee754: $(T_IEEE_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_IEEE_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_IEEE_OBJS) $(LIBS)

fio: $(FIO_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(FIO_OBJS) $(LIBS) $(HDFSLIB)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(FIO_OBJS) $(LIBS) $(HDFSLIB)

gfio: $(GFIO_OBJS)
$(QUIET_LINK)$(CC) $(filter-out -static, $(LDFLAGS)) -o gfio $(GFIO_OBJS) $(LIBS) $(GFIO_LIBS) $(GTK_LDFLAGS) $(HDFSLIB)

t/fio-genzipf: $(T_ZIPF_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_ZIPF_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_ZIPF_OBJS) $(LIBS)

t/axmap: $(T_AXMAP_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_AXMAP_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_AXMAP_OBJS) $(LIBS)

t/lfsr-test: $(T_LFSR_TEST_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_LFSR_TEST_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_LFSR_TEST_OBJS) $(LIBS)

t/gen-rand: $(T_GEN_RAND_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_GEN_RAND_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_GEN_RAND_OBJS) $(LIBS)

ifeq ($(CONFIG_TARGET_OS), Linux)
t/fio-btrace2fio: $(T_BTRACE_FIO_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_BTRACE_FIO_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_BTRACE_FIO_OBJS) $(LIBS)
endif

t/fio-dedupe: $(T_DEDUPE_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_DEDUPE_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_DEDUPE_OBJS) $(LIBS)

t/fio-verify-state: $(T_VS_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_VS_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_VS_OBJS) $(LIBS)

t/time-test: $(T_TT_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_TT_OBJS) $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_TT_OBJS) $(LIBS)

ifdef CONFIG_HAVE_CUNIT
unittests/unittest: $(UT_OBJS) $(UT_TARGET_OBJS)
- $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(UT_OBJS) $(UT_TARGET_OBJS) -lcunit $(LIBS)
+ $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(UT_OBJS) $(UT_TARGET_OBJS) -lcunit $(LIBS)
+endif
+
+ifdef CONFIG_DYNAMIC_ENGINES
+engines/lib$(1).so: $$($(1)_OBJS)
+ $$(QUIET_LINK)$(CC) -shared -rdynamic -fPIC -Wl,-soname,lib$(1).so.1 $$($(1)_LIBS) -o $$@ $$<
endif

clean: FORCE
@@ -603,7 +610,7 @@ fulltest:
sudo t/zbd/run-tests-against-zoned-nullb; \
fi

-install: $(PROGS) $(SCRIPTS) tools/plot/fio2gnuplot.1 FORCE
+install: $(PROGS) $(SCRIPTS) $(ENGS_OBJS) tools/plot/fio2gnuplot.1 FORCE
$(INSTALL) -m 755 -d $(DESTDIR)$(bindir)
$(INSTALL) $(PROGS) $(SCRIPTS) $(DESTDIR)$(bindir)
ifdef CONFIG_DYNAMIC_ENGINES
diff --git a/backend.c b/backend.c
index 05453ae2..f91f3caf 100644
--- a/backend.c
+++ b/backend.c
@@ -1458,16 +1458,17 @@ static bool keep_running(struct thread_data *td)
return false;
}

-static int exec_string(struct thread_options *o, const char *string, const char *mode)
+static int exec_string(struct thread_options *o, const char *string,
+ const char *mode)
{
- size_t newlen = strlen(string) + strlen(o->name) + strlen(mode) + 13 + 1;
int ret;
char *str;

- str = malloc(newlen);
- sprintf(str, "%s > %s.%s.txt 2>&1", string, o->name, mode);
+ if (asprintf(&str, "%s > %s.%s.txt 2>&1", string, o->name, mode) < 0)
+ return -1;

- log_info("%s : Saving output of %s in %s.%s.txt\n",o->name, mode, o->name, mode);
+ log_info("%s : Saving output of %s in %s.%s.txt\n", o->name, mode,
+ o->name, mode);
ret = system(str);
if (ret == -1)
log_err("fio: exec of cmd <%s> failed\n", str);
@@ -1731,7 +1732,7 @@ static void *thread_main(void *data)
if (!init_random_map(td))
goto err;

- if (o->exec_prerun && exec_string(o, o->exec_prerun, (const char *)"prerun"))
+ if (o->exec_prerun && exec_string(o, o->exec_prerun, "prerun"))
goto err;

if (o->pre_read && !pre_read_files(td))
@@ -1890,7 +1891,7 @@ static void *thread_main(void *data)
rate_submit_exit(td);

if (o->exec_postrun)
- exec_string(o, o->exec_postrun, (const char *)"postrun");
+ exec_string(o, o->exec_postrun, "postrun");

if (exitall_on_terminate || (o->exitall_error && td->error))
fio_terminate_threads(td->groupid, td->o.exit_what);
diff --git a/ci/appveyor-install.sh b/ci/appveyor-install.sh
new file mode 100755
index 00000000..c73e4cb5
--- /dev/null
+++ b/ci/appveyor-install.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# The PATH to appropriate distro commands must already be set before invoking
+# this script
+# The following environment variables must be set:
+# PLATFORM={i686,x64}
+# DISTRO={cygwin,msys2}
+# The following environment can optionally be set:
+# CYG_MIRROR=<URL>
+set -eu
+
+case "${ARCHITECTURE}" in
+ "x64")
+ PACKAGE_ARCH="x86_64"
+ ;;
+ "x86")
+ PACKAGE_ARCH="i686"
+ ;;
+esac
+
+echo "Installing packages..."
+case "${DISTRO}" in
+ "cygwin")
+ CYG_MIRROR=${CYG_MIRROR:-"http://cygwin.mirror.constant.com"}
+ setup-x86_64.exe --quiet-mode --no-shortcuts --only-site \
+ --site "${CYG_MIRROR}" --packages \
+ "mingw64-${PACKAGE_ARCH}-CUnit,mingw64-${PACKAGE_ARCH}-zlib"
+ ;;
+ "msys2")
+ #pacman --noconfirm -Syuu # MSYS2 core update
+ #pacman --noconfirm -Syuu # MSYS2 normal update
+ pacman.exe --noconfirm -S \
+ mingw-w64-${PACKAGE_ARCH}-clang \
+ mingw-w64-${PACKAGE_ARCH}-cunit \
+ mingw-w64-${PACKAGE_ARCH}-lld
+ ;;
+esac
+
+python.exe -m pip install scipy six
+
+echo "Python3 path: $(type -p python3 2>&1)"
+echo "Python3 version: $(python3 -V 2>&1)"
diff --git a/ci/travis-build.sh b/ci/travis-build.sh
index 231417e2..923d882d 100755
--- a/ci/travis-build.sh
+++ b/ci/travis-build.sh
@@ -1,8 +1,9 @@
#!/bin/bash
+set -eu

CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}"
EXTRA_CFLAGS="-Werror"
-PYTHONUNBUFFERED=TRUE
+export PYTHONUNBUFFERED=TRUE
CONFIGURE_FLAGS=()

case "$TRAVIS_OS_NAME" in
@@ -11,6 +12,7 @@ case "$TRAVIS_OS_NAME" in
case "$CI_TARGET_ARCH" in
"x86")
EXTRA_CFLAGS="${EXTRA_CFLAGS} -m32"
+ export LDFLAGS="-m32"
;;
"amd64")
CONFIGURE_FLAGS+=(--enable-cuda)
@@ -24,7 +26,7 @@ CONFIGURE_FLAGS+=(--extra-cflags="${EXTRA_CFLAGS}")
make &&
make test &&
if [[ "$CI_TARGET_ARCH" == "arm64" ]]; then
- sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"
+ sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"
else
- sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug
+ sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug
fi
diff --git a/ci/travis-install.sh b/ci/travis-install.sh
index b6895e82..103695dc 100755
--- a/ci/travis-install.sh
+++ b/ci/travis-install.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -e
+set -eu

CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}"
case "$TRAVIS_OS_NAME" in
@@ -51,6 +51,5 @@ case "$TRAVIS_OS_NAME" in
;;
esac

-echo "Python version: $(/usr/bin/python -V 2>&1)"
-echo "Python3 path: $(which python3 2>&1)"
+echo "Python3 path: $(type -p python3 2>&1)"
echo "Python3 version: $(python3 -V 2>&1)"
diff --git a/configure b/configure
index 08571fb0..12b7cb58 100755
--- a/configure
+++ b/configure
@@ -193,6 +193,8 @@ for opt do
;;
--target-win-ver=*) target_win_ver="$optarg"
;;
+ --enable-pdb) pdb="yes"
+ ;;
--build-static) build_static="yes"
;;
--enable-gfio) gfio_check="yes"
@@ -256,6 +258,7 @@ if test "$show_help" = "yes" ; then
echo "--extra-cflags= Specify extra CFLAGS to pass to compiler"
echo "--build-32bit-win Enable 32-bit build on Windows"
echo "--target-win-ver= Minimum version of Windows to target (XP or 7)"
+ echo "--enable-pdb Enable Windows PDB symbols generation (needs clang/lld)"
echo "--build-static Build a static fio"
echo "--esx Configure build options for esx"
echo "--enable-gfio Enable building of gtk gfio"
@@ -394,6 +397,8 @@ CYGWIN*)
fi
if test "$target_win_ver" = "XP"; then
output_sym "CONFIG_WINDOWS_XP"
+ # Technically the below is targeting 2003
+ CFLAGS="$CFLAGS -D_WIN32_WINNT=0x0502"
elif test "$target_win_ver" = "7"; then
output_sym "CONFIG_WINDOWS_7"
CFLAGS="$CFLAGS -D_WIN32_WINNT=0x0601"
@@ -939,7 +944,8 @@ cat > $TMPC << EOF

int main(int argc, char **argv)
{
- return asprintf(NULL, "%s", "str") == 0;
+ char *buf;
+ return asprintf(&buf, "%s", "str") == 0;
}
EOF
if compile_prog "" "" "have_asprintf"; then
@@ -956,7 +962,8 @@ cat > $TMPC << EOF
int main(int argc, char **argv)
{
va_list ap;
- return vasprintf(NULL, "%s", ap) == 0;
+ char *buf;
+ return vasprintf(&buf, "%s", ap) == 0;
}
EOF
if compile_prog "" "" "have_vasprintf"; then
@@ -2234,19 +2241,14 @@ lex="no"
arith="no"
if test "$disable_lex" = "no" || test -z "$disable_lex" ; then
if test "$targetos" != "SunOS" ; then
-LEX=$(which lex 2> /dev/null)
-if test -x "$LEX" ; then
+if has lex; then
lex="yes"
fi
-YACC=$(which bison 2> /dev/null)
-if test -x "$YACC" ; then
+if has bison; then
yacc="yes"
yacc_is_bison="yes"
-else
- YACC=$(which yacc 2> /dev/null)
- if test -x "$YACC" ; then
- yacc="yes"
- fi
+elif has yacc; then
+ yacc="yes"
fi
if test "$yacc" = "yes" && test "$lex" = "yes" ; then
arith="yes"
@@ -2262,7 +2264,9 @@ int main(int argc, char **argv)
return 0;
}
EOF
-if compile_prog "" "-ll" "lex"; then
+if compile_prog "" "-lfl" "flex"; then
+ LIBS="-lfl $LIBS"
+elif compile_prog "" "-ll" "lex"; then
LIBS="-ll $LIBS"
else
arith="no"
@@ -2276,8 +2280,7 @@ if test "$arith" = "yes" ; then
if test "$force_no_lex_o" = "yes" ; then
lex_use_o="no"
else
-$LEX -o lex.yy.c exp/expression-parser.l 2> /dev/null
-if test "$?" = "0" ; then
+if lex -o lex.yy.c exp/expression-parser.l 2> /dev/null; then
lex_use_o="yes"
else
lex_use_o="no"
@@ -2698,6 +2701,27 @@ if compile_prog "" "" "statx_syscall"; then
fi
print_config "statx(2)/syscall" "$statx_syscall"

+##########################################
+# check for Windows PDB generation support
+if test "pdb" != "no" ; then
+ cat > $TMPC <<EOF
+int main(void)
+{
+ return 0;
+}
+EOF
+ if compile_prog "-g -gcodeview" "-fuse-ld=lld -Wl,-pdb,$TMPO" "pdb"; then
+ pdb=yes
+ else
+ if test "$pdb" = "yes"; then
+ feature_not_found "PDB" "clang and lld"
+ fi
+ pdb=no
+ fi
+else
+ pdb=no
+fi
+print_config "Windows PDB generation" "$pdb"
#############################################################################

if test "$wordsize" = "64" ; then
@@ -2931,9 +2955,9 @@ fi
if test "$arith" = "yes" ; then
output_sym "CONFIG_ARITHMETIC"
if test "$yacc_is_bison" = "yes" ; then
- echo "YACC=$YACC -y" >> $config_host_mak
+ echo "YACC=bison -y" >> $config_host_mak
else
- echo "YACC=$YACC" >> $config_host_mak
+ echo "YACC=yacc" >> $config_host_mak
fi
if test "$lex_use_o" = "yes" ; then
echo "CONFIG_LEX_USE_O=y" >> $config_host_mak
@@ -3020,6 +3044,10 @@ fi
if test "$dynamic_engines" = "yes" ; then
output_sym "CONFIG_DYNAMIC_ENGINES"
fi
+if test "$pdb" = yes; then
+ output_sym "CONFIG_PDB"
+fi
+
print_config "Lib-based ioengines dynamic" "$dynamic_engines"
cat > $TMPC << EOF
int main(int argc, char **argv)
diff --git a/engines/net.c b/engines/net.c
index 91f25774..c6cec584 100644
--- a/engines/net.c
+++ b/engines/net.c
@@ -938,8 +938,9 @@ static int fio_netio_udp_recv_open(struct thread_data *td, struct fio_file *f)

if (ntohl(msg.magic) != FIO_LINK_OPEN_CLOSE_MAGIC ||
ntohl(msg.cmd) != FIO_LINK_OPEN) {
- log_err("fio: bad udp open magic %x/%x\n", ntohl(msg.magic),
- ntohl(msg.cmd));
+ log_err("fio: bad udp open magic %x/%x\n",
+ (unsigned int) ntohl(msg.magic),
+ (unsigned int) ntohl(msg.cmd));
return -1;
}

diff --git a/engines/windowsaio.c b/engines/windowsaio.c
index 5c7e7964..9868e816 100644
--- a/engines/windowsaio.c
+++ b/engines/windowsaio.c
@@ -161,15 +161,15 @@ static int windowsaio_invalidate_cache(struct fio_file *f)
if (ihFile != INVALID_HANDLE_VALUE) {
if (!CloseHandle(ihFile)) {
error = GetLastError();
- log_info("windowsaio: invalidation fd close %s "
- "failed: error %d\n", f->file_name, error);
+ log_info("windowsaio: invalidation fd close %s failed: error %lu\n",
+ f->file_name, error);
rc = 1;
}
} else {
error = GetLastError();
if (error != ERROR_FILE_NOT_FOUND) {
- log_info("windowsaio: cache invalidation of %s failed: "
- "error %d\n", f->file_name, error);
+ log_info("windowsaio: cache invalidation of %s failed: error %lu\n",
+ f->file_name, error);
rc = 1;
}
}
diff --git a/os/windows/cpu-affinity.c b/os/windows/cpu-affinity.c
index 69997b24..46fd048d 100644
--- a/os/windows/cpu-affinity.c
+++ b/os/windows/cpu-affinity.c
@@ -14,7 +14,7 @@ int fio_setaffinity(int pid, os_cpu_mask_t cpumask)
bSuccess = SetThreadAffinityMask(h, cpumask);
if (!bSuccess)
log_err("fio_setaffinity failed: failed to set thread affinity (pid %d, mask %.16llx)\n",
- pid, cpumask);
+ pid, (long long unsigned) cpumask);

CloseHandle(h);
} else {
@@ -83,7 +83,7 @@ unsigned int cpus_online(void)
static void print_mask(os_cpu_mask_t *cpumask)
{
for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
- dprint(FD_PROCESS, "cpumask[%d]=%lu\n", i, cpumask->row[i]);
+ dprint(FD_PROCESS, "cpumask[%d]=%" PRIu64 "\n", i, cpumask->row[i]);
}

/* Return the index of the least significant set CPU in cpumask or -1 if no
@@ -99,7 +99,7 @@ int first_set_cpu(os_cpu_mask_t *cpumask)
int row_first_cpu;

row_first_cpu = __builtin_ffsll(cpumask->row[row]) - 1;
- dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%lu\n",
+ dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
row_first_cpu, row, cpumask->row[row]);
if (row_first_cpu > -1) {
mask_first_cpu = cpus_offset + row_first_cpu;
@@ -136,7 +136,7 @@ static int last_set_cpu(os_cpu_mask_t *cpumask)
row_last_cpu++;
}

- dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%lu\n",
+ dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
row_last_cpu, row, cpumask->row[row]);
if (row_last_cpu > -1) {
mask_last_cpu = cpus_offset + row_last_cpu;
@@ -213,13 +213,17 @@ static int mask_to_group_mask(os_cpu_mask_t *cpumask, int *processor_group, uint
needed_shift = FIO_CPU_MASK_STRIDE - bit_offset;
needed_mask_shift = FIO_CPU_MASK_STRIDE - needed;
needed_mask = (uint64_t)-1 >> needed_mask_shift;
- dprint(FD_PROCESS, "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%ld needed_mask_shift=%d\n", bit_offset, end, needed, needed_shift, needed_mask, needed_mask_shift);
+ dprint(FD_PROCESS,
+ "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%" PRIu64 "needed_mask_shift=%d\n",
+ bit_offset, end, needed, needed_shift, needed_mask,
+ needed_mask_shift);
group_cpumask |= (cpumask->row[row + 1] & needed_mask) << needed_shift;
}
group_cpumask &= (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - group_size);

/* Return group and mask */
- dprint(FD_PROCESS, "Returning group=%d group_mask=%lu\n", group, group_cpumask);
+ dprint(FD_PROCESS, "Returning group=%d group_mask=%" PRIu64 "\n",
+ group, group_cpumask);
*processor_group = group;
*affinity_mask = group_cpumask;

@@ -257,10 +261,8 @@ int fio_setaffinity(int pid, os_cpu_mask_t cpumask)
if (SetThreadGroupAffinity(handle, &new_group_affinity, NULL) != 0)
ret = 0;
else {
- log_err("fio_setaffinity: failed to set thread affinity "
- "(pid %d, group %d, mask %" PRIx64 ", "
- "GetLastError=%d)\n", pid, group, group_mask,
- GetLastError());
+ log_err("fio_setaffinity: failed to set thread affinity (pid %d, group %d, mask %" PRIx64 ", GetLastError=%lu)\n",
+ pid, group, group_mask, GetLastError());
goto err;
}

@@ -319,7 +321,7 @@ int fio_getaffinity(int pid, os_cpu_mask_t *mask)
goto err;
}
if (!GetProcessGroupAffinity(handle, &group_count, current_groups)) {
- log_err("%s: failed to get single group affinity for pid %d (%d)\n",
+ log_err("%s: failed to get single group affinity for pid %d (%lu)\n",
__func__, pid, GetLastError());
goto err;
}
@@ -329,7 +331,7 @@ int fio_getaffinity(int pid, os_cpu_mask_t *mask)
goto err;
}
if (!GetProcessAffinityMask(handle, &process_mask, &system_mask)) {
- log_err("%s: GetProcessAffinityMask() failed for pid\n",
+ log_err("%s: GetProcessAffinityMask() failed for pid %d\n",
__func__, pid);
goto err;
}
diff --git a/os/windows/dobuild.cmd b/os/windows/dobuild.cmd
index d06a2afa..08df3e87 100644
--- a/os/windows/dobuild.cmd
+++ b/os/windows/dobuild.cmd
@@ -34,7 +34,13 @@ if defined SIGN_FIO (
signtool sign /as /n "%SIGNING_CN%" /tr http://timestamp.digicert.com /td sha256 /fd sha256 ..\..\t\*.exe
)

-"%WIX%bin\candle" -nologo -arch %FIO_ARCH% -dFioVersionNumbers="%FIO_VERSION_NUMBERS%" install.wxs
+if exist ..\..\fio.pdb (
+ set FIO_PDB=true
+) else (
+ set FIO_PDB=false
+)
+
+"%WIX%bin\candle" -nologo -arch %FIO_ARCH% -dFioVersionNumbers="%FIO_VERSION_NUMBERS%" -dFioPDB="%FIO_PDB%" install.wxs
@if ERRORLEVEL 1 goto end
"%WIX%bin\candle" -nologo -arch %FIO_ARCH% examples.wxs
@if ERRORLEVEL 1 goto end
@@ -43,4 +49,4 @@ if defined SIGN_FIO (

if defined SIGN_FIO (
signtool sign /n "%SIGNING_CN%" /tr http://timestamp.digicert.com /td sha256 /fd sha256 %FIO_VERSION%-%FIO_ARCH%.msi
-)
\ No newline at end of file
+)
diff --git a/os/windows/install.wxs b/os/windows/install.wxs
index dcb8c92c..f73ec5e2 100755
--- a/os/windows/install.wxs
+++ b/os/windows/install.wxs
@@ -27,6 +27,11 @@
<File Source="..\..\fio.exe"/>
<Environment Action="set" Part="last" Id="PATH" Name="PATH" Value="[INSTALLDIR]fio\" System="yes"/>
</Component>
+ <?if $(var.FioPDB) = true?>
+ <Component>
+ <File Id="fio.pdb" Name="fio.pdb" Source="..\..\fio.pdb"/>
+ </Component>
+ <?endif?>
<Component>
<File Id="README" Name="README.txt" Source="..\..\README"/>
</Component>
@@ -76,6 +81,9 @@

<Feature Id="AlwaysInstall" Absent="disallow" ConfigurableDirectory="INSTALLDIR" Display="hidden" Level="1" Title="Flexible I/O Tester">
<ComponentRef Id="fio.exe"/>
+ <?if $(var.FioPDB) = true?>
+ <ComponentRef Id="fio.pdb"/>
+ <?endif?>
<ComponentRef Id="HOWTO"/>
<ComponentRef Id="README"/>
<ComponentRef Id="REPORTING_BUGS"/>
diff --git a/os/windows/posix.c b/os/windows/posix.c
index 31271de0..9e9f12ef 100644
--- a/os/windows/posix.c
+++ b/os/windows/posix.c
@@ -168,7 +168,7 @@ int win_to_posix_error(DWORD winerr)
case ERROR_FILE_INVALID:
return ENXIO;
default:
- log_err("fio: windows error %d not handled\n", winerr);
+ log_err("fio: windows error %lu not handled\n", winerr);
return EIO;
}

@@ -188,7 +188,8 @@ int GetNumLogicalProcessors(void)
if (error == ERROR_INSUFFICIENT_BUFFER)
processor_info = malloc(len);
else {
- log_err("Error: GetLogicalProcessorInformation failed: %d\n", error);
+ log_err("Error: GetLogicalProcessorInformation failed: %lu\n",
+ error);
return -1;
}

diff --git a/t/memlock.c b/t/memlock.c
index 418dc3c4..9f5a3ea8 100644
--- a/t/memlock.c
+++ b/t/memlock.c
@@ -22,7 +22,7 @@ static void *worker(void *data)
for (index = 0; index + 4096 < size; index += 4096)
memset(&buf[index+512], 0x89, 512);
if (first) {
- printf("loop%d: did %lu MiB\n", i+1, size/(1024UL*1024UL));
+ printf("loop%d: did %lu MiB\n", i+1, td->mib);
first = 0;
}
}