[Buildroot] [PATCH v4] utils/test-pkg: add --jobs option

[Buildroot] [PATCH v4] utils/test-pkg: add --jobs option

[Joseph Kogut]

Builds run by test-pkg are often not CPU limited, so running several
builds concurrently reduces the time needed to test a package across
many toolchains.

Add a -j/--jobs option to run multiple builds in parallel.

Quote "keep" in build_one(), as it is now passed in as an argument
rather than set from a literal.

Testing sdl2 on six toolchains with a 16-core Ryzen 9 9950X reduced wall
clock time from 2m32.979s with -j1 to 0m41.704s with -j0. Testing libpng
across all 35 toolchains reduced wall clock time from 23m34.593s to
2m6.510s, an approximately 11x speedup.

Signed-off-by: Joseph Kogut <joseph.kogut@gmail.com>
---
Changes in v4:
- Rewritten to use GNU parallel
- Changed -C -> -j (more idiomatic)
- Link to v3: https://lore.kernel.org/r/20251028-concurrent-test-pkg-v3-1-ae014d18e5f5@gmail.com

Changes in v3:
- Fix improper removal of '-T' short option during rebase
- Minor tweak to remove newline between prompt and output
- Simplify make job termination
- Link to v2: https://lore.kernel.org/r/20251023-concurrent-test-pkg-v2-1-959ad443d49b@gmail.com

Changes in v2:
- Rebase on origin/master
- Properly restore cursor on interrupt
- Properly terminate running jobs on interrupt
- Add animated spinner for running builds
- Simplify status updates
- Link to v1: https://lore.kernel.org/r/20251022-concurrent-test-pkg-v1-1-1fe96df1102b@gmail.com

To: buildroot@buildroot.org
---
 utils/test-pkg | 84 ++++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 70 insertions(+), 14 deletions(-)

diff --git a/utils/test-pkg b/utils/test-pkg
index cea7ace7cb..887cebb4d2 100755
--- a/utils/test-pkg
+++ b/utils/test-pkg
@@ -17,13 +17,13 @@ do_clean() {
 
 main() {
     local o O opts
-    local cfg dir pkg random toolchains_csv toolchain all number mode prepare_only
-    local ret nb nb_skip nb_fail nb_legal nb_show nb_tc build_dir keep
+    local cfg dir pkg random toolchains_csv toolchain all number mode prepare_only jobs
+    local ret nb nb_skip nb_fail nb_legal nb_show nb_tc keep
     local -a toolchains
     local pkg_br_name
 
-    o='hakc:d:n:p:r:t:T:'
-    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,toolchain-name:'
+    o='hakc:d:j:n:p:r:t:T:'
+    O='help,all,keep,prepare-only,config-snippet:,build-dir:,jobs:,number:,package:,random:,toolchains-csv:,toolchain-name:'
     opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
     eval set -- "${opts}"
 
@@ -33,6 +33,7 @@ main() {
     number=0
     mode=0
     prepare_only=0
+    jobs=1
     toolchains_csv="${TOOLCHAINS_CSV}"
     while [ ${#} -gt 0 ]; do
         case "${1}" in
@@ -54,6 +55,9 @@ main() {
         (-d|--build-dir)
             dir="${2}"; shift 2
             ;;
+        (-j|--jobs)
+            jobs="${2}"; shift 2
+            ;;
         (-n|--number)
             number="${2}"; shift 2
             ;;
@@ -88,6 +92,12 @@ main() {
     if [ ! -e "${cfg}" ]; then
         printf "error: %s: no such file\n" "${cfg}" >&2; exit 1
     fi
+    case "${jobs}" in
+    (''|*[!0-9]*) printf "error: invalid number of jobs: %s\n" "${jobs}" >&2; exit 1;;
+    esac
+    if [ "${jobs}" -eq 0 ]; then
+        jobs="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)"
+    fi
     if [ -z "${dir}" ]; then
         dir="${HOME}/br-test-pkg"
     fi
@@ -137,12 +147,9 @@ main() {
     nb_fail=0
     nb_legal=0
     nb_show=0
-    for toolchainconfig in "${toolchains[@]}"; do
+    while read -r ret toolchain; do
         : $((nb++))
-        toolchain="$(basename "${toolchainconfig}" .config)"
-        build_dir="${dir}/${toolchain}"
         printf "%40s [%*d/%d]: " "${toolchain}" ${#nb_tc} "${nb}" "${nb_tc}"
-        build_one "${build_dir}" "${toolchainconfig}" "${cfg}" "${pkg}" "${prepare_only}" && ret=0 || ret=${?}
         case ${ret} in
         (0) printf "OK\n";;
         (1) : $((nb_skip++)); printf "SKIPPED\n";;
@@ -150,11 +157,15 @@ main() {
         (3) : $((nb_legal++)); printf "FAILED\n";;
         (4) : $((nb_show++)); printf "FAILED\n";;
         esac
+    done < <(run_builds)
 
-        if [ "${abort}" -eq 1 ]; then
-            return 1
-        fi
-    done
+    if [ "${abort}" -eq 1 ]; then
+        return 1
+    fi
+    if [ "${nb}" -ne "${nb_tc}" ]; then
+        printf "error: only %d of the %d builds reported a status\n" "${nb}" "${nb_tc}" >&2
+        return 1
+    fi
 
     printf "%d builds, %d skipped, %d build failed, %d legal-info failed, %d show-info failed\n" \
         "${nb}" "${nb_skip}" "${nb_fail}" "${nb_legal}" "${nb_show}"
@@ -162,6 +173,47 @@ main() {
     return $((nb_fail + nb_legal))
 }
 
+run_builds() {
+    local toolchainconfig
+
+    if [ "${jobs}" -gt 1 ]; then
+        if parallel --version 2>/dev/null | grep -q '^GNU parallel'; then
+            export -f build_one build_one_status
+            # --quote, so that empty and spaced arguments survive.
+            PARALLEL_SHELL=bash parallel --will-cite --quote --jobs "${jobs}" \
+                --ungroup build_one_status {} "${dir}" "${cfg}" "${pkg}" \
+                "${prepare_only}" "${keep}" ::: "${toolchains[@]}" || true
+            return
+        fi
+        printf "warning: GNU parallel not available; running builds serially\n" >&2
+    fi
+    # We run in a sub-shell, which does not inherit the trap set in
+    # main(), so re-arm it to keep stopping after the current build.
+    trap do_abort INT
+    for toolchainconfig in "${toolchains[@]}"; do
+        build_one_status "${toolchainconfig}" "${dir}" "${cfg}" "${pkg}" \
+            "${prepare_only}" "${keep}"
+
+        if [ "${abort}" -eq 1 ]; then
+            break
+        fi
+    done
+}
+
+build_one_status() {
+    local toolchainconfig="${1}"
+    local dir="${2}"
+    local cfg="${3}"
+    local pkg="${4}"
+    local prepare_only="${5}"
+    local keep="${6}"
+    local toolchain ret
+
+    toolchain="$(basename "${toolchainconfig}" .config)"
+    build_one "${dir}/${toolchain}" "${toolchainconfig}" "${cfg}" "${pkg}" "${prepare_only}" && ret=0 || ret=${?}
+    printf "%d %s\n" "${ret}" "${toolchain}"
+}
+
 build_one() {
     local dir="${1}"
     local toolchainconfig="${2}"
@@ -182,7 +234,7 @@ build_one() {
     # done in the same locale.
     comm -23 <(sort "${cfg}") <(sort "${dir}/.config") >"${dir}/missing.config"
     if [ -s "${dir}/missing.config" ]; then
-        if [ ${keep} -ne 1 ]; then
+        if [ "${keep}" -ne 1 ]; then
             # Invalid configuration, drop it
             rm -f "${dir}/.config"
         fi
@@ -229,7 +281,7 @@ build_one() {
 
     # If we get here, the build was successful. Clean up the build/host
     # directories to save disk space, unless 'keep' was set.
-    if [ ${keep} -ne 1 ]; then
+    if [ "${keep}" -ne 1 ]; then
         make O="${dir}" clean >> "${dir}/logfile" 2>&1
     fi
 }
@@ -299,6 +351,10 @@ Options:
         is specified, then a single build will be performed on that
         specific toolchain only.
 
+    -j N, --jobs N
+        Run N builds concurrently using GNU parallel. If N is 0, match the
+        number of logical CPUs. Builds run serially if GNU parallel is missing.
+
     -k, --keep
         Keep the build directories even if the build succeeds.
         Note: the logfile and configuration is always retained, even without

---
base-commit: 6144b0f4b73bea810809f09d23bbe76b4979bc13
change-id: 20250619-concurrent-test-pkg-f3ad6d3c01b4

Best regards,
--  
Joseph Kogut <joseph.kogut@gmail.com>

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot