Recent changes (master)

Jens Axboe
2020-07-27T12:00:09Z

The following changes since commit e235e74dfd0627b17690194f957da509b8ace808:

Merge branch 'test-cleanup' of https://github.com/vincentkfu/fio (2020-07-25 13:55:16 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 11b280c158db01744effde3863bfe9c65f7af090:

t/jobs/t001[1-2].fio: run for 10 seconds instead of 3 (2020-07-26 22:19:47 -0600)

----------------------------------------------------------------
Jens Axboe (4):
Merge branch 'travis_cleanup' of https://github.com/sitsofe/fio
Merge branch 'enable_or_exit' of https://github.com/sitsofe/fio
Merge branch 'fix_travis_libiscsi' of https://github.com/sitsofe/fio
t/jobs/t001[1-2].fio: run for 10 seconds instead of 3

Sitsofe Wheeler (12):
travis: simplify yml file
travis: improve installation dependencies
configure: check for Debian/Ubuntu tcmalloc_minimal
travis: use install section instead of before_install section
travis: add dependency for cuda ioengine
travis: enable libiscsi and cuda ioengines
travis: make CI install script bail out on first error
memory: fix incorrect pointer comparison when freeing cuda memory
Makefile: fix incorrectly set libiscsi cflags
configure: fail when explicit enabling doesn't succeed
travis: remove unneeded dependency
travis: fix x86 libiscsi detection

.travis.yml | 18 ++++----------
Makefile | 2 +-
ci/travis-build.sh | 23 ++++++++++++++----
ci/travis-install.sh | 30 ++++++++++++++++-------
configure | 62 +++++++++++++++++++++++++++++------------------
memory.c | 2 +-
t/jobs/t0011-5d2788d5.fio | 2 +-
t/jobs/t0012.fio | 2 +-
8 files changed, 87 insertions(+), 54 deletions(-)

---

Diff of recent changes:

diff --git a/.travis.yml b/.travis.yml
index b64f0a95..e35aff39 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,36 +9,28 @@ arch:
- amd64
- arm64
env:
- matrix:
- - BUILD_ARCH="x86"
- - BUILD_ARCH="x86_64"
global:
- MAKEFLAGS="-j 2"
matrix:
include:
+ - os: linux
+ compiler: gcc
+ arch: amd64
+ env: BUILD_ARCH="x86" # Only do the gcc x86 build to reduce clutter
# Default xcode image
- os: osx
compiler: clang # Workaround travis setting CC=["clang", "gcc"]
- env: BUILD_ARCH="x86_64"
arch: amd64
# Latest xcode image (needs periodic updating)
- os: osx
compiler: clang
osx_image: xcode11.2
- env: BUILD_ARCH="x86_64"
arch: amd64
exclude:
- os: osx
compiler: gcc
- - os: linux
- compiler: clang
- arch: amd64
- env: BUILD_ARCH="x86" # Only do the gcc x86 build to reduce clutter
- - os: linux
- env: BUILD_ARCH="x86"
- arch: arm64

-before_install:
+install:
- ci/travis-install.sh

script:
diff --git a/Makefile b/Makefile
index f374ac84..8e1ebc90 100644
--- a/Makefile
+++ b/Makefile
@@ -62,7 +62,7 @@ endif
ifdef CONFIG_LIBISCSI
iscsi_SRCS = engines/libiscsi.c
iscsi_LIBS = $(LIBISCSI_LIBS)
- iscsi_CFLAGS = $(LIBISCSI_LIBS)
+ iscsi_CFLAGS = $(LIBISCSI_CFLAGS)
ENGINES += iscsi
endif

diff --git a/ci/travis-build.sh b/ci/travis-build.sh
index 06012e89..231417e2 100755
--- a/ci/travis-build.sh
+++ b/ci/travis-build.sh
@@ -1,16 +1,29 @@
#!/bin/bash

+CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}"
EXTRA_CFLAGS="-Werror"
PYTHONUNBUFFERED=TRUE
+CONFIGURE_FLAGS=()

-if [[ "$BUILD_ARCH" == "x86" ]]; then
- EXTRA_CFLAGS="${EXTRA_CFLAGS} -m32"
-fi
+case "$TRAVIS_OS_NAME" in
+ "linux")
+ CONFIGURE_FLAGS+=(--enable-libiscsi)
+ case "$CI_TARGET_ARCH" in
+ "x86")
+ EXTRA_CFLAGS="${EXTRA_CFLAGS} -m32"
+ ;;
+ "amd64")
+ CONFIGURE_FLAGS+=(--enable-cuda)
+ ;;
+ esac
+ ;;
+esac
+CONFIGURE_FLAGS+=(--extra-cflags="${EXTRA_CFLAGS}")

-./configure --extra-cflags="${EXTRA_CFLAGS}" &&
+./configure "${CONFIGURE_FLAGS[@]}" &&
make &&
make test &&
- if [[ "$TRAVIS_CPU_ARCH" == "arm64" ]]; then
+ if [[ "$CI_TARGET_ARCH" == "arm64" ]]; then
sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"
else
sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug
diff --git a/ci/travis-install.sh b/ci/travis-install.sh
index 232ab6b1..b6895e82 100755
--- a/ci/travis-install.sh
+++ b/ci/travis-install.sh
@@ -1,13 +1,15 @@
#!/bin/bash
+set -e

+CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}"
case "$TRAVIS_OS_NAME" in
"linux")
# Architecture-dependent packages.
pkgs=(
libaio-dev
- libcunit1
libcunit1-dev
- libgoogle-perftools4
+ libfl-dev
+ libgoogle-perftools-dev
libibverbs-dev
libiscsi-dev
libnuma-dev
@@ -15,15 +17,26 @@ case "$TRAVIS_OS_NAME" in
librdmacm-dev
libz-dev
)
- if [[ "$BUILD_ARCH" == "x86" ]]; then
- pkgs=("${pkgs[@]/%/:i386}")
- pkgs+=(gcc-multilib)
- else
- pkgs+=(glusterfs-common)
+ case "$CI_TARGET_ARCH" in
+ "x86")
+ pkgs=("${pkgs[@]/%/:i386}")
+ pkgs+=(
+ gcc-multilib
+ pkg-config:i386
+ )
+ ;;
+ "amd64")
+ pkgs+=(nvidia-cuda-dev)
+ ;;
+ esac
+ if [[ $CI_TARGET_ARCH != "x86" ]]; then
+ pkgs+=(glusterfs-common)
fi
# Architecture-independent packages and packages for which we don't
# care about the architecture.
pkgs+=(
+ bison
+ flex
python3
python3-scipy
python3-six
@@ -34,8 +47,7 @@ case "$TRAVIS_OS_NAME" in
"osx")
brew update >/dev/null 2>&1
brew install cunit
- pip3 install scipy
- pip3 install six
+ pip3 install scipy six
;;
esac

diff --git a/configure b/configure
index 25216c63..5925e94f 100755
--- a/configure
+++ b/configure
@@ -144,6 +144,7 @@ libhdfs="no"
pmemblk="no"
devdax="no"
pmem="no"
+cuda="no"
disable_lex=""
disable_pmem="no"
disable_native="no"
@@ -202,7 +203,7 @@ for opt do
;;
--disable-pmem) disable_pmem="yes"
;;
- --enable-cuda) enable_cuda="yes"
+ --enable-cuda) cuda="yes"
;;
--disable-native) disable_native="yes"
;;
@@ -626,8 +627,13 @@ int main(void)
return 0;
}
EOF
- if test "$libaio_uring" = "yes" && compile_prog "" "-luring" "libaio io_uring" ; then
- libaio=yes
+ if test "$libaio_uring" = "yes"; then
+ if compile_prog "" "-luring" "libaio io_uring" ; then
+ libaio=yes
+ LIBS="-luring $LIBS"
+ else
+ feature_not_found "libaio io_uring" ""
+ fi
elif compile_prog "" "-laio" "libaio" ; then
libaio=yes
libaio_uring=no
@@ -2052,7 +2058,7 @@ if test "$libhdfs" = "yes" ; then
hdfs_conf_error=1
fi
if test "$hdfs_conf_error" = "1" ; then
- exit 1
+ feature_not_found "libhdfs" ""
fi
FIO_HDFS_CPU=$cpu
if test "$FIO_HDFS_CPU" = "x86_64" ; then
@@ -2170,15 +2176,16 @@ fi
print_config "DDN's Infinite Memory Engine" "$libime"

##########################################
-# Check if we have required environment variables configured for libiscsi
-if test "$libiscsi" = "yes" ; then
- if $(pkg-config --atleast-version=1.9.0 libiscsi); then
+# Check if we have libiscsi
+if test "$libiscsi" != "no" ; then
+ minimum_libiscsi=1.9.0
+ if $(pkg-config --atleast-version=$minimum_libiscsi libiscsi); then
libiscsi="yes"
libiscsi_cflags=$(pkg-config --cflags libiscsi)
libiscsi_libs=$(pkg-config --libs libiscsi)
else
if test "$libiscsi" = "yes" ; then
- echo "libiscsi" "Install libiscsi >= 1.9.0"
+ feature_not_found "libiscsi" "libiscsi >= $minimum_libiscsi"
fi
libiscsi="no"
fi
@@ -2186,16 +2193,16 @@ fi
print_config "iscsi engine" "$libiscsi"

##########################################
-# Check if we have libnbd (for NBD support).
-minimum_libnbd=0.9.8
-if test "$libnbd" = "yes" ; then
+# Check if we have libnbd (for NBD support)
+if test "$libnbd" != "no" ; then
+ minimum_libnbd=0.9.8
if $(pkg-config --atleast-version=$minimum_libnbd libnbd); then
libnbd="yes"
libnbd_cflags=$(pkg-config --cflags libnbd)
libnbd_libs=$(pkg-config --libs libnbd)
else
if test "$libnbd" = "yes" ; then
- echo "libnbd" "Install libnbd >= $minimum_libnbd"
+ feature_not_found "libnbd" "libnbd >= $minimum_libnbd"
fi
libnbd="no"
fi
@@ -2506,9 +2513,7 @@ print_config "march_armv8_a_crc_crypto" "$march_armv8_a_crc_crypto"

##########################################
# cuda probe
-if test "$cuda" != "yes" ; then
- cuda="no"
-fi
+if test "$cuda" != "no" ; then
cat > $TMPC << EOF
#include <cuda.h>
int main(int argc, char **argv)
@@ -2516,9 +2521,15 @@ int main(int argc, char **argv)
return cuInit(0);
}
EOF
-if test "$enable_cuda" = "yes" && compile_prog "" "-lcuda" "cuda"; then
- cuda="yes"
- LIBS="-lcuda $LIBS"
+ if compile_prog "" "-lcuda" "cuda"; then
+ cuda="yes"
+ LIBS="-lcuda $LIBS"
+ else
+ if test "$cuda" = "yes" ; then
+ feature_not_found "cuda" ""
+ fi
+ cuda="no"
+ fi
fi
print_config "cuda" "$cuda"

@@ -3006,11 +3017,16 @@ int main(int argc, char **argv)
return 0;
}
EOF
-if test "$disable_tcmalloc" != "yes" && compile_prog "" "-ltcmalloc" "tcmalloc"; then
- LIBS="-ltcmalloc $LIBS"
- tcmalloc="yes"
-else
- tcmalloc="no"
+if test "$disable_tcmalloc" != "yes"; then
+ if compile_prog "" "-ltcmalloc" "tcmalloc"; then
+ tcmalloc="yes"
+ LIBS="-ltcmalloc $LIBS"
+ elif compile_prog "" "-l:libtcmalloc_minimal.so.4" "tcmalloc_minimal4"; then
+ tcmalloc="yes"
+ LIBS="-l:libtcmalloc_minimal.so.4 $LIBS"
+ else
+ tcmalloc="no"
+ fi
fi
print_config "TCMalloc support" "$tcmalloc"

diff --git a/memory.c b/memory.c
index 5f0225f7..6cf73333 100644
--- a/memory.c
+++ b/memory.c
@@ -274,7 +274,7 @@ static int alloc_mem_cudamalloc(struct thread_data *td, size_t total_mem)
static void free_mem_cudamalloc(struct thread_data *td)
{
#ifdef CONFIG_CUDA
- if (td->dev_mem_ptr != NULL)
+ if (td->dev_mem_ptr)
cuMemFree(td->dev_mem_ptr);

if (cuCtxDestroy(td->cu_ctx) != CUDA_SUCCESS)
diff --git a/t/jobs/t0011-5d2788d5.fio b/t/jobs/t0011-5d2788d5.fio
index 50daf612..f90cee90 100644
--- a/t/jobs/t0011-5d2788d5.fio
+++ b/t/jobs/t0011-5d2788d5.fio
@@ -7,7 +7,7 @@
bs=4k
ioengine=null
size=100g
-runtime=3
+runtime=10
flow_id=1

[flow1]
diff --git a/t/jobs/t0012.fio b/t/jobs/t0012.fio
index 985eb16b..03fea627 100644
--- a/t/jobs/t0012.fio
+++ b/t/jobs/t0012.fio
@@ -8,7 +8,7 @@
bs=4k
ioengine=null
size=100g
-runtime=3
+runtime=10
flow_id=1
gtod_cpu=1