Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit 7bc12310ac39f8bf694a6785d027e4cee96dbbdf:

  Merge branch 'errdetails' of https://github.com/minwooim/fio (2024-08-16 08:21:20 -0600)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2d416331db2735031e85c8da64ce93382796799b:

  ci: install isal packages for testing (2024-08-21 20:31:46 +0000)

----------------------------------------------------------------
Vincent Fu (3):
      ci: remove arm64 case for tests
      ci: run tests in containers
      ci: install isal packages for testing

 .github/workflows/ci.yml | 32 ++++++++++++++++++++++++
 ci/actions-build.sh      | 14 ++++++++---
 ci/actions-full-test.sh  | 28 ++++++++++++++++++---
 ci/actions-install.sh    | 63 ++++++++++++++++++++++++++++++++++++++++++++++--
 4 files changed, 128 insertions(+), 9 deletions(-)

---

Diff of recent changes:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e53082c3..1601aa9e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,6 +5,38 @@ on:
   pull_request:
 
 jobs:
+  build-containers:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        container:
+        - {os: 'debian', dh: 'debian', ver: 'bookworm', target_arch: 'x86_64'}
+        - {os: 'fedora', dh: 'fedora', ver: '40', target_arch: 'x86_64'}
+        - {os: 'ubuntu', dh: 'ubuntu', ver: 'noble', target_arch: 'i686'}
+        - {os: 'ubuntu', dh: 'ubuntu', ver: 'noble', target_arch: 'x86_64'}
+
+    container:
+      image: ${{ matrix.container.dh }}:${{ matrix.container.ver }}
+      env:
+        CI_TARGET_BUILD: Linux
+        CI_TARGET_ARCH: ${{ matrix.container.target_arch }}
+        CI_TARGET_OS: ${{ matrix.container.os }}
+        CI_TARGET_OS_VER: ${{ matrix.container.ver }}
+
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v4
+    - name: Install dependencies
+      run:  ./ci/actions-install.sh
+    - name: Build
+      run:  ./ci/actions-build.sh
+    - name: Smoke test
+      run:  ./ci/actions-smoke-test.sh
+    - name: Full test
+      run:  ./ci/actions-full-test.sh
+
   build:
     runs-on: ${{ matrix.os }}
     strategy:
diff --git a/ci/actions-build.sh b/ci/actions-build.sh
index 47d4f044..de444a22 100755
--- a/ci/actions-build.sh
+++ b/ci/actions-build.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # This script expects to be invoked from the base fio directory.
 set -eu
 
@@ -29,7 +29,16 @@ main() {
                 return 1
             fi
             ;;
-        */linux)
+        */linux | */ubuntu)
+            case "${CI_TARGET_ARCH}" in
+                "x86_64")
+                    configure_flags+=(
+                        "--enable-cuda"
+                    )
+                    ;;
+	    esac
+	    ;;&
+        */linux | */ubuntu | */debian | */fedora)
             case "${CI_TARGET_ARCH}" in
                 "i686")
                     extra_cflags="${extra_cflags} -m32"
@@ -37,7 +46,6 @@ main() {
                     ;;
                 "x86_64")
                     configure_flags+=(
-                        "--enable-cuda"
                         "--enable-libiscsi"
                         "--enable-libnbd"
                     )
diff --git a/ci/actions-full-test.sh b/ci/actions-full-test.sh
index d2fb4201..23bdd219 100755
--- a/ci/actions-full-test.sh
+++ b/ci/actions-full-test.sh
@@ -10,11 +10,31 @@ main() {
 
     echo "Running long running tests..."
     export PYTHONUNBUFFERED="TRUE"
-    if [[ "${CI_TARGET_ARCH}" == "arm64" ]]; then
-        python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"
-    else
-        python3 t/run-fio-tests.py --skip 6 1007 1008 --debug
+    skip=(
+        6
+	1007
+	1008
+    )
+    args=(
+        --debug
+    )
+    if [ "${GITHUB_JOB}" == "build-containers" ]; then
+        # io_uring is disabled in containers
+        # so skip the io_uring test
+        skip+=(
+            18
+        )
+	# cmd priority does not work in containers
+	# so skip the related latency test cases
+	args+=(
+	    -p
+            "1010:--skip 15 16 17 18 19 20 21 22"
+        )
+
     fi
+
+    echo python3 t/run-fio-tests.py --skip "${skip[@]}" "${args[@]}"
+    python3 t/run-fio-tests.py --skip "${skip[@]}" "${args[@]}"
     make -C doc html
 }
 
diff --git a/ci/actions-install.sh b/ci/actions-install.sh
index 6eb2d795..38d39de9 100755
--- a/ci/actions-install.sh
+++ b/ci/actions-install.sh
@@ -9,6 +9,12 @@ SCRIPT_DIR=$(dirname "$0")
 install_ubuntu() {
     local pkgs
 
+    if [ "${GITHUB_JOB}" == "build-containers" ]; then
+        # containers run as root and do not have sudo
+        apt update
+        apt -y install sudo
+    fi
+
     cat <<DPKGCFG | sudo tee /etc/dpkg/dpkg.cfg.d/dpkg-speedup > /dev/null
 # Skip fsync
 force-unsafe-io
@@ -17,6 +23,7 @@ path-exclude=/usr/share/man/*
 path-exclude=/usr/share/locale/*/LC_MESSAGES/*.mo
 path-exclude=/usr/share/doc/*
 DPKGCFG
+
     # Packages available on i686 and x86_64
     pkgs=(
         libaio-dev
@@ -43,6 +50,7 @@ DPKGCFG
             pkgs+=(
                 libglusterfs-dev
                 libgoogle-perftools-dev
+                libisal-dev
                 libiscsi-dev
                 libnbd-dev
                 libpmem-dev
@@ -50,12 +58,18 @@ DPKGCFG
                 libprotobuf-c-dev
                 librbd-dev
                 libtcmalloc-minimal4
-                nvidia-cuda-dev
                 libibverbs-dev
                 librdmacm-dev
+	        pkg-config
             )
 	    echo "Removing libunwind-14-dev because of conflicts with libunwind-dev"
 	    sudo apt remove -y libunwind-14-dev
+	    if [ "${CI_TARGET_OS}" == "linux" ] || [ "${CI_TARGET_OS}" == "ubuntu" ]; then
+	        # Only for Ubuntu
+		pkgs+=(
+		   nvidia-cuda-dev
+		)
+	    fi
             ;;
     esac
 
@@ -66,6 +80,17 @@ DPKGCFG
 	python3-sphinx
 	python3-statsmodels
     )
+    if [ "${GITHUB_JOB}" == "build-containers" ]; then
+        pkgs+=(
+            bison
+            build-essential
+            cmake
+            flex
+            unzip
+            wget
+            zlib1g-dev
+        )
+    fi
 
     echo "Updating APT..."
     sudo apt-get -qq update
@@ -77,6 +102,40 @@ DPKGCFG
     fi
 }
 
+install_fedora() {
+    dnf install -y \
+        bison-devel \
+        cmake \
+        cunit-devel \
+        flex-devel \
+        isa-l-devel \
+        kernel-devel \
+        libaio-devel \
+        libgfapi-devel \
+        libibverbs-devel \
+        libiscsi-devel \
+        libnbd-devel \
+        libnfs-devel \
+        libpmem-devel \
+        libpmem2-devel \
+        librbd-devel \
+        numactl-devel \
+        protobuf-c-devel \
+        python3-scipy \
+        python3-sphinx \
+        python3-statsmodels \
+        unzip \
+        valgrind-devel \
+        wget \
+
+    # install librpma from sources
+    ci/actions-install-librpma.sh
+}
+
+install_debian() {
+    install_ubuntu
+}
+
 install_linux() {
     install_ubuntu
 }
@@ -86,7 +145,7 @@ install_macos() {
     #echo "Updating homebrew..."
     #brew update >/dev/null 2>&1
     echo "Installing packages..."
-    HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs bash
     pip3 install scipy six statsmodels sphinx
 }