* [LOCAL-CI v2 1/3] scripts: Add general scripts to aid automated testing
2022-01-04 18:00 [LOCAL-CI v2 0/3] Add support for local automated testing Glenn Washburn
@ 2022-01-04 18:00 ` Glenn Washburn
2022-01-04 18:00 ` [LOCAL-CI v2 2/3] scripts: Add functions for CI stages and default variables to functions.sh Glenn Washburn
2022-01-04 18:00 ` [LOCAL-CI v2 3/3] scripts: Add local-tester.sh script to run local CI tests Glenn Washburn
2 siblings, 0 replies; 4+ messages in thread
From: Glenn Washburn @ 2022-01-04 18:00 UTC (permalink / raw)
To: Daniel Kiper, grub-devel; +Cc: Glenn Washburn
Signed-off-by: Glenn Washburn <development@efficientek.com>
---
scripts/ci/build.sh | 67 ++++++++++++++++++++
scripts/ci/functions.sh | 33 ++++++++++
scripts/ci/make-images.sh | 86 +++++++++++++++++++++++++
scripts/ci/process-tests.sh | 111 +++++++++++++++++++++++++++++++++
scripts/ci/test.sh | 121 ++++++++++++++++++++++++++++++++++++
5 files changed, 418 insertions(+)
create mode 100755 scripts/ci/build.sh
create mode 100644 scripts/ci/functions.sh
create mode 100755 scripts/ci/make-images.sh
create mode 100755 scripts/ci/process-tests.sh
create mode 100755 scripts/ci/test.sh
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
new file mode 100755
index 000000000..723dc292f
--- /dev/null
+++ b/scripts/ci/build.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# JOBS: Number of concurrent jobs while building, defaults to number of
+# processors plus 1, unless number of processors is 1 in which case its 2.
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# INSTALLDIR: Directory to install binaries
+# ARCH: Architecture to build for
+# PLATFORM: Platform to build for
+# CONFIGURE_OPTS: Extra configure options
+# FONT_SOURCE: Path to unicode font named "unifont.<ext>" where ext is one of:
+# pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# DJVU_FONT_SOURCE: Path to DejaVu font named "DejaVuSans.<ext>" where ext is
+# one of: pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+
+[ "x${SHELL_TRACE}" = "xy" ] && set -x
+
+[ -f "$(dirname "$0")/functions.sh" ] &&
+. "$(dirname "$0")/functions.sh"
+
+[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] &&
+. "$(dirname "$0")/functions.$CI_TYPE.sh"
+
+JOBS=${JOBS:-`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 2`}
+[ "$JOBS" = 1 ] || JOBS=$(($JOBS + 1))
+
+TARGET="${ARCH}-${PLATFORM}"
+
+mkdir -pv ${BUILDDIR}
+
+RET=0
+cd ${BUILDDIR}
+
+if test -f "$FONT_SOURCE"; then
+ ln -svf "$(realpath -e "$FONT_SOURCE")" ./
+fi
+
+if test -f "$DJVU_FONT_SOURCE"; then
+ ln -svf "$(realpath -e "$DJVU_FONT_SOURCE")" ./
+fi
+
+start_log -c -n "configure-$TARGET" "Configuring $TARGET"
+[ -x "$SRCDIR/configure" ] && $SRCDIR/configure --help
+$SRCDIR/configure --target=$ARCH --with-platform=$PLATFORM \
+ --prefix=${INSTALLDIR} $CONFIGURE_OPTS || RET=$?
+end_log -n "configure-$TARGET"
+
+if [ "$RET" -ne 0 ]; then
+ start_log -c -n "showlogs-$TARGET" "Configuring $TARGET failed, showing logs"
+ cat ${BUILDDIR}/config.log
+ end_log -n "showlogs-$TARGET"
+ exit $RET
+fi
+
+start_log -c -n "build-$TARGET" "Building $TARGET"
+make ${JOBS:+-j$JOBS} || RET=$?
+end_log -n "build-$TARGET"
+[ "$RET" -ne 0 ] && exit $RET
+
+start_log -c -n "install-$TARGET" "Installing $TARGET"
+make ${JOBS:+-j$JOBS} install || RET=$?
+end_log -n "install-$TARGET"
+exit $RET
diff --git a/scripts/ci/functions.sh b/scripts/ci/functions.sh
new file mode 100644
index 000000000..2f4cecaa1
--- /dev/null
+++ b/scripts/ci/functions.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+TXT_RED="\e[31m"
+TXT_YELLOW="\e[33m"
+TXT_CLEAR="\e[0m"
+TXT_LOG_COLOR="\e[97;100m"
+
+function start_log() {
+ local LOG_COLLAPSE LOG_NAME
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ -c) LOG_COLLAPSE=1; shift;;
+ -n) LOG_NAME="$2"; shift;;
+ *) [ "$#" -eq 1 ] && LOG_MSG="$1"; shift;;
+ esac
+ done
+
+ echo -e "Start:${LOG_NAME} ${LOG_MSG}"
+}
+
+function end_log() {
+ local LOG_NAME
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ -n) LOG_NAME=$2; shift;;
+ *) shift;;
+ esac
+ done
+
+ echo -e "End:${LOG_NAME}"
+}
+
+:;
diff --git a/scripts/ci/make-images.sh b/scripts/ci/make-images.sh
new file mode 100755
index 000000000..d156af46f
--- /dev/null
+++ b/scripts/ci/make-images.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# TARGET: Target to test
+# MAKE_ALL_IMAGE_TARGETS: If set to 'y', then all image targets, even disabled
+# ones, will be run.
+# DISABLED_IMAGES: String of target formats to disable when building grub
+# images delimited by new lines.
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+# NUM_FAILED_LOG_LINES: Set to integer number of log lines to display from the
+# end of a failed log file
+
+if [ "x${SHELL_TRACE}" = "xy" ]; then
+ # If we export SHELL_OPTS, then all shells will be traced, most of which we do not want
+ SHELL_OPTS="-x"
+ set -x
+fi
+
+[ -f "$(dirname "$0")/functions.sh" ] &&
+. "$(dirname "$0")/functions.sh"
+
+[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] &&
+. "$(dirname "$0")/functions.$CI_TYPE.sh"
+
+start_log -c -n "images-$TARGET" "Making images for $TARGET"
+
+if [ ! -x "${BUILDDIR}/grub-mkimage" ]; then
+ echo "Aborting, no grub-mkimage found in builddir"
+ exit 1
+fi
+
+function build_tformat() {
+ TARGET=$1
+ FORMATS=$2
+ for fmt in $FORMATS; do
+ if [ -z "${fmt%~*}" ]; then
+ echo "${TARGET}"
+ else
+ echo "${TARGET}-${fmt}"
+ fi
+ done
+}
+
+tformats="~"
+case "$TARGET" in
+ mipsel-loongson)
+ tformats="mipsel-loongson-elf mipsel-yeeloong-flash mipsel-fuloong2f-flash" ;;
+ *)
+ case "$TARGET" in
+ i386-pc) tformats="~ pxe eltorito" ;;
+ sparc64-ieee1275) tformats="aout cdcore raw" ;;
+ mips-qemu_mips | \
+ mipsel-qemu_mips) tformats="elf flash" ;;
+ arm-coreboot) tformats="vexpress veyron" ;;
+ esac
+ tformats=$(build_tformat "${TARGET}" "$tformats")
+ ;;
+esac
+
+RET=0
+mkdir -pv "${BUILDDIR}/images"
+for tfmt in $tformats; do
+ if [ "x${MAKE_ALL_IMAGE_TARGETS}" != "xy" ] && ( echo "${DISABLED_IMAGES}" | grep -qE "^\s*${tfmt}$" ); then
+ echo "Image build disabled for target format $tfmt"
+ continue
+ fi
+
+ echo "Making image for target format: ${tfmt}"
+ ${BUILDDIR}/grub-mkimage -v -p / -O "${tfmt}" -o "${BUILDDIR}/images/grub-${tfmt}.img" echo reboot normal > "${BUILDDIR}/grub-mkimage-${tfmt}.log" 2>&1 || _RET=$?
+ [ "${_RET:-0}" -ne 0 ] && RET=$_RET
+
+ if [ "$RET" -ne 0 ]; then
+ echo -e "${TXT_RED}Failed to build image for target format ${tfmt}: ${TESTNAME}$TXT_CLEAR"
+ echo -e -n "$TXT_LOG_COLOR"
+ echo "Last ${NUM_FAILED_LOG_LINES} lines of grub-mkimage verbose log"
+ tail -n "${NUM_FAILED_LOG_LINES}" "${BUILDDIR}/grub-mkimage-${tfmt}.log"
+ echo -e -n "$TXT_CLEAR"
+ fi
+done
+
+end_log -n "images-$TARGET"
+exit $RET
diff --git a/scripts/ci/process-tests.sh b/scripts/ci/process-tests.sh
new file mode 100755
index 000000000..9bf763502
--- /dev/null
+++ b/scripts/ci/process-tests.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# TARGET: Target to process
+# TEST_ALL_TARGETS: If set to 'y', then all tests, even disabled ones will be
+# processed.
+# DISABLED_TESTS: String of disabled tests delimited by new lines.
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+# EXPECTED_FAILURES: Multi-line string where each line contains a testname or
+# target:testname indicating an expected failure. Lines whose first non-space
+# character is '#' are ignored.
+# EXPECTED_FUNCTIONAL_FAILURES: Same format as EXPECTED_FAILURES applied to the
+# functional test suite (grub_func_test).
+# FAIL_ON_HARD_ERRORS: Set to 'y' to fail on hard as well as normal errors.
+# IGNORE_TIMEDOUT_TEST_FAILURE: Set to 'y' to ignore test failures due to
+# test timing out.
+# NUM_FAILED_LOG_LINES: Set to integer number of log lines to display from the
+# end of a failed log file
+
+[ "x${SHELL_TRACE}" = "xy" ] && set -x
+
+[ -f "$(dirname "$0")/functions.sh" ] &&
+. "$(dirname "$0")/functions.sh"
+
+[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] &&
+. "$(dirname "$0")/functions.$CI_TYPE.sh"
+
+# TARGET="${ARCH}-${PLATFORM}"
+NUM_FAILED_LOG_LINES=${NUM_FAILED_LOG_LINES:-100}
+
+if [ -f ${BUILDDIR}/test.success ]; then
+ echo "Not processing test logs because make check ran successfully"
+ exit 0
+elif [ "x${TEST_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_TESTS}" | grep -qE "^\s*${TARGET}$" ); then
+ echo "No test output processing because testing was disabled"
+ exit 0
+fi;
+
+start_log -c -n "process-test-$TARGET" "Processing test output of $TARGET"
+: >${BUILDDIR}/test-results.log
+if [ -f ${BUILDDIR}/test-suite.log ]; then
+ ( grep -E "^(FAIL|ERROR|SKIP):" ${BUILDDIR}/test-suite.log || ':' ) |
+ while read STATUS TESTNAME; do
+ STATUS=${STATUS%%:}
+
+ if [ "$STATUS" = "SKIP" ]; then
+ TYPE=skip
+ elif tail -n1 "${BUILDDIR}/${TESTNAME}.log" | grep -qE 'exit status: (124|137)'; then
+ if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" = "xy" ]; then
+ echo -e "${TXT_RED}Ignoring Timed-out test: ${TESTNAME}$TXT_CLEAR"
+ echo -e -n "$TXT_LOG_COLOR"
+ echo "Test failed due to a timeout. This is likely due to"
+ echo "insufficient host resources, and NOT a real failure."
+ echo -e -n "$TXT_CLEAR"
+ fi
+ TYPE=timeout
+ elif [ "$STATUS" = "ERROR" ]; then
+ TYPE=error
+ elif echo "${EXPECTED_FAILURES}" | grep -qE "^\s*(${TESTNAME}|${TARGET}:${TESTNAME})$"; then
+ echo "Expected failed test: $TESTNAME"
+ TYPE=expected
+ # If any unexpected failures in the functional tests, count a failure in
+ # grub_func_test as a true failure.
+ elif [ "$TESTNAME" = "grub_func_test" ]; then
+ grep -E ': FAIL' ${BUILDDIR}/${TESTNAME}.log | sort -u |
+ while read FTESTNAME STATUS; do
+ FTESTNAME=${FTESTNAME%:*}
+ if echo "${EXPECTED_FUNCTIONAL_FAILURES}" | grep -qE "^\s*(${FTESTNAME}|${TARGET}:${FTESTNAME})$"; then
+ echo "Expected failed functional test: $FTESTNAME"
+ TYPE=expected
+ else
+ echo -e "${TXT_RED}Unexpected failed functional test: ${FTESTNAME}$TXT_CLEAR"
+ TYPE=unexpected
+ fi
+ echo "${TARGET}:${TYPE}:${TESTNAME}:${FTESTNAME}" >> ${BUILDDIR}/test-results.log
+ done
+ continue
+ else
+ echo -e "${TXT_RED}Unexpected failed test: ${TESTNAME}$TXT_CLEAR"
+ echo -e -n "$TXT_LOG_COLOR"
+ echo "Last ${NUM_FAILED_LOG_LINES} lines of ${BUILDDIR}/${TESTNAME}.log"
+ tail -n "${NUM_FAILED_LOG_LINES}" "${BUILDDIR}/${TESTNAME}.log"
+ echo -e -n "$TXT_CLEAR"
+ TYPE=unexpected
+ fi
+ echo "${TARGET}:${TYPE}:${TESTNAME}" >> ${BUILDDIR}/test-results.log
+ done
+else
+ echo "No success canary and no test-suite.log, perhaps tests were not run or was killed."
+ RET=2
+fi
+
+# If there are any unexpected errors return exit with error
+FAILCOND="unexpected"
+if [ "x${FAIL_ON_HARD_ERRORS}" = "xy" ]; then
+ FAILCOND="${FAILCOND}|error"
+fi;
+if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" != "xy" ]; then
+ FAILCOND="${FAILCOND}|timeout"
+fi;
+FAILCOND="${TARGET}:(${FAILCOND})"
+
+if grep -E -qs "${FAILCOND}" ${BUILDDIR}/test-results.log; then
+ RET=1
+fi
+end_log -n "process-test-$TARGET"
+exit ${RET:-0}
diff --git a/scripts/ci/test.sh b/scripts/ci/test.sh
new file mode 100755
index 000000000..833912587
--- /dev/null
+++ b/scripts/ci/test.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# TARGET: Target to test
+# TEST_ALL_TARGETS: If set to 'y', then all tests, even disabled ones will be
+# run.
+# DISABLED_TESTS: String of disabled tests delimited by new lines.
+# TESTS_TIMEOUT: Maximum timeout to allow for 'make check'. Set this slightly
+# below total job timeout to allow for cleanup and log packaging code.
+# TEST_VERBOSITY: Number to set verbosity level to
+# TEST_DATA_PREFIX: Filename prefix to use when saving test data. If not set,
+# test data will not be saved.
+# STRACE_TESTS: Set to 'y' to strace each test
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+# JOBS: Number of concurrent jobs while building, defaults to number of
+# processors plus 1, unless number of processors is 1 in which case its 2.
+
+
+if [ "x${SHELL_TRACE}" = "xy" ]; then
+ # If we export SHELL_OPTS, then all shells will be traced, most of which we do not want
+ SHELL_OPTS="-x"
+ set -x
+fi
+
+[ -f "${0%/*}/functions.sh" ] &&
+. "${0%/*}/functions.sh"
+
+[ -f "${0%/*}/functions.$CI_TYPE.sh" ] &&
+. "${0%/*}/functions.$CI_TYPE.sh"
+
+if [ "x${TEST_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_TESTS}" | grep -qE "^\s*${TARGET}$" ); then
+ echo "Tests are disabled for target $TARGET"
+ exit 0
+fi
+
+start_log -c -n "test-$TARGET" "Testing $TARGET"
+
+if [ "${TEST_VERBOSITY:-0}" -gt 0 ]; then
+ export GRUB_SHELL_DEFAULT_DEBUG=${TEST_VERBOSITY}
+ export GRUB_TEST_DEFAULT_DEBUG=${TEST_VERBOSITY}
+fi
+
+JOBS="${JOBS:-`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 2`}"
+TESTTMPDIR="${TESTTMPDIR:-${TMPDIR:-/tmp}/grub-test-$TARGET}"
+COMP=xz
+STRACE_LOG="${BUILDDIR}/xxx.strace.$COMP"
+
+if [ "x${STRACE_TESTS}" = "xy" ]; then
+ STRACE="strace -y -yy -f -v -s4096 -o >($COMP -9 > $STRACE_LOG)"
+fi
+
+cat >${BUILDDIR}/log-tester.sh <<EOF
+#!$SHELL $SHELL_OPTS
+set -e
+
+STRACE_LOG="$STRACE_LOG"
+STRACE="$STRACE"
+TESTNAME="\${1##*/}"
+export SHELL_OPTS="$SHELL_OPTS"
+export TMPDIR=$TESTTMPDIR/\$TESTNAME
+mkdir -p "\$TMPDIR"
+
+# if not a shell script, run normally
+if [ "\$(head -c2 \$1)" = "#!" ]; then
+ BUILD_SHEBANG="\$(head -n1 "${BUILDDIR}/grub-shell" | tail -c+3 | sed -E 's|^\\s*||')"
+ TEST_SHELL=\$(head -n1 "\$1" | tail -c+3 | sed -E 's|^\\s*||')
+ # Only turn on tracing if the shell is the one used by grub-shell
+ if test "\${TEST_SHELL}" = "\${BUILD_SHEBANG}"; then
+ if [ '(' "\${TEST_VERBOSITY:-0}" -gt 0 -o "x\${SHELL_TRACE}" = "xy" ')' \\
+ -a '(' -z "\$SHELL_OPTS" -o -n "\${SHELL_OPTS##*-x*}" ')' ]; then
+ SHELL_OPTS="\$SHELL_OPTS -x"
+ fi
+ fi
+fi
+
+TSTART=\$(date +%s)
+eval \$(echo -n "\$STRACE" | sed "s/xxx/\$TESTNAME/g") "\$TEST_SHELL" "\$@" || RET=\$?
+TEND=\$(date +%s)
+echo "Test duration in seconds: \$((TEND - TSTART))"
+
+if [ "\${RET:-0}" -eq "0" ]; then
+ rm -rf \$(echo -n "\$STRACE_LOG" | sed "s/xxx/\$TESTNAME/g") \$TMPDIR
+elif [ "\${RET:-0}" -eq "124" ] || [ "\${RET:-0}" -eq "137" ]; then
+ # This is these exitcodes are for a timed out process when using timeout
+ # In this case return a hard error, otherwise it will be converted to a
+ # normal error.
+ exit 99
+fi
+
+exit \$RET
+EOF
+export LOG_COMPILER="${BUILDDIR}/log-tester.sh"
+chmod +x ${BUILDDIR}/log-tester.sh
+echo -n -e "${TXT_YELLOW}"
+cat ${BUILDDIR}/log-tester.sh
+echo -n -e "${TXT_CLEAR}"
+
+# Setting debug to greater than 9 turns on the graphical qemu window.
+# But we want to run in headless environments, so make sure debug values
+# do not go above 9.
+[ "${GRUB_SHELL_DEFAULT_DEBUG:-0}" -gt 9 ] && GRUB_SHELL_DEFAULT_DEBUG=9
+[ "${GRUB_TEST_DEFAULT_DEBUG:-0}" -gt 9 ] && GRUB_TEST_DEFAULT_DEBUG=9
+
+TIMEOUT=${TESTS_TIMEOUT}
+
+RET=0
+${TIMEOUT:+timeout $TIMEOUT} make -C ${BUILDDIR} ${JOBS:+-j$JOBS} SUBDIRS= check || RET=$?
+[ "$RET" -eq 124 -o "$RET" -eq 137 ] && echo "ERROR: make check timed out"
+
+end_log -n "test-$TARGET"
+
+[ "$RET" -eq 0 ] && touch ${BUILDDIR}/test.success || :
+if [ -n "$TEST_DATA_PREFIX" ]; then
+ tar -S -cJf "${TEST_DATA_PREFIX}.tar.xz" -C "${TESTTMPDIR%/*}" "${TESTTMPDIR##*/}"
+fi
+
+exit $RET
--
2.27.0
^ permalink raw reply related [flat|nested] 4+ messages in thread* [LOCAL-CI v2 2/3] scripts: Add functions for CI stages and default variables to functions.sh
2022-01-04 18:00 [LOCAL-CI v2 0/3] Add support for local automated testing Glenn Washburn
2022-01-04 18:00 ` [LOCAL-CI v2 1/3] scripts: Add general scripts to aid " Glenn Washburn
@ 2022-01-04 18:00 ` Glenn Washburn
2022-01-04 18:00 ` [LOCAL-CI v2 3/3] scripts: Add local-tester.sh script to run local CI tests Glenn Washburn
2 siblings, 0 replies; 4+ messages in thread
From: Glenn Washburn @ 2022-01-04 18:00 UTC (permalink / raw)
To: Daniel Kiper, grub-devel; +Cc: Glenn Washburn
Signed-off-by: Glenn Washburn <development@efficientek.com>
---
scripts/ci/functions.sh | 923 +++++++++++++++++++++++++++++++++++++++-
1 file changed, 922 insertions(+), 1 deletion(-)
diff --git a/scripts/ci/functions.sh b/scripts/ci/functions.sh
index 2f4cecaa1..734b36d3f 100644
--- a/scripts/ci/functions.sh
+++ b/scripts/ci/functions.sh
@@ -1,5 +1,412 @@
#!/bin/bash
+### Documentation on configuration environment variables
+
+# SDIR [dirname of $0]:
+# Path to directory the caller of this script resides in.
+# ROOT [./grub-tests]:
+# Path to directory where all file operations take place, unless overridden
+# by another environment variable.
+# CI_PROJECT_DIR [$ROOT]:
+# Base directory to use in commands.
+# CI_TYPE [local]:
+# Value used to control sourcing of ${CI_SCRIPT_DIR}/functions.$CI_TYPE.sh
+# which overrides the variables/functions in this file.
+# SCRIPTS_DIR [${SDIR} if ${SDIR}/ci else ${SRCDIR}/scripts]:
+# Path to scripts directory from source.
+# CI_SCRIPTS_DIR [${SCRIPTS_DIR}/ci]:
+# Path to CI scripts directory, normally the ci subdirectory of SCRITS_DIR
+# and where this file resides.
+# SHELL_TRACE [n]:
+# If set, turn on shell tracing of everything. NOTE: TEST_VERBOSITY=3 turns
+# on more targeted shell tracing
+# TMPDIR [$ROOT/testtmp]:
+# Path to directory to use for temporary files.
+# MATRIX [see definition below]:
+# A list of $ARCH:$TARGET[,...]:$CROSS_ARCH, one per line, with lines
+# starting with # being ignored.
+# JOBS:
+# Number of make jobs to run in parallel. Defaulted in build.sh and test.sh
+# to the number of available processors.
+# NUM_FAILED_LOG_LINES [100]:
+# Set to integer number of log lines to display from the end of a failed
+# log file.
+
+### Source checkout variables
+# GIT_REPO:
+# URL to git repo
+# GIT_BRANCH:
+# Branch name in git repo
+# GIT_CLONE_ARGS [--depth=1 --no-single-branch]:
+# Extra args to use when cloning repository
+# SRCDIR [${CI_PROJECT_DIR}/grub]:
+# Path to source checkout
+
+### Distro package variables
+# APT_OPTS [--no-upgrade]:
+# Extra options when running apt.
+# PACKAGES [see definition below]:
+# A list of packages to add to a default list of packages needing to be
+# installed. The default list can only be appended to.
+# PACKAGE_CACHE [${CI_PROJECT_DIR}/packages]:
+# Path to directory where downloaded packages will be cached.
+
+### Ccache variables
+# CCACHE_DIR [${CI_PROJECT_DIR}/ccache]:
+# Directory to use for ccache
+
+### Cross compiler tool chain variables
+# CROSS_DIR [$CI_PROJECT_DIR/cross]:
+# Path to directory containing cross compilers tool chains for each
+# architecture. If it doesn't exist, it will be created and required tool
+# chains will be downloaded/installed there.
+# CROSS_VERSION [10.1.0]:
+# Version of cross-compiler to use (suggested: 10.1.0, 9.3.0, 8.1.0,
+# 7.5.0 [riscv32/64 builds fail before 7.3.0]).
+
+### Configure/Build variables
+# CONFIGURE_OPTS [--enable-boot-time]:
+# Extra options to pass to configure.
+# FONT_SOURCE:
+# Path to unicode font named "unifont.<ext>" where ext is one of:
+# pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# DJVU_FONT_SOURCE:
+# Path to DejaVu font named "DejaVuSans.<ext>" where ext is one of:
+# pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# BUILD_ALL_TARGETS:
+# If set 'y', all targets defined in the build matrix will be
+# built ignoring DISABLED_BUILDS, but can still be disabled by
+# DISABLE_ALL_BUILDS.
+# DISABLED_BUILDS [see definition below]:
+# A list of targets for which builds are disabled, one per line. These
+# default targets are disabled generally because they do not work yet.
+# Lines beginning with '#' may be used as comments and are ignored. Some
+# hints as to why the tests fail are included in comment lines. Patches
+# which get these tests working are very welcome.
+# DISABLE_ALL_BUILDS [n]:
+# If set to 'y', disable completely building (eg. the run_build command).
+
+### Tests variables
+# TESTS_TIMEOUT:
+# Set timeout for completion of all make check tests (see "man timeout"
+# duration argument). This is most useful when the make check job is
+# taking longer than the job timeout configured by the runner. In this
+# case, set this to 5-10 minutes less than runner timeout so that there
+# is time to package up the logs for debugging.
+# IGNORE_TIMEDOUT_TEST_FAILURE [y]:
+# If set to 'y', tests which fail due to a timeout in grub-shell of qemu
+# will not be counted as a failure. These failures are almost always the
+# result of insufficient runner resources to complete the execution of qemu
+# within the timeout period.
+# FAIL_ON_HARD_ERRORS [n]:
+# If set to 'y', hard errors will cause a failure. A hard error indicates
+# that the test was not able to be run (eg. dependencies not found), and
+# as such a hard error does not constitute a true failure of the test.
+# DISABLE_ALL_TESTS [n]:
+# If set to 'y', make check tests will be disabled. Effectively only test
+# whether the building is successful for configured platforms.
+# TEST_ALL_TARGETS:
+# If set 'y', failing targets which have been marked disabled by
+# DISABLED_TESTS will be enabled.
+# TEST_VERBOSITY [3]:
+# Verbosity level when running tests [supported values: '' and 0-9]. Level
+# 9 turns on all debug messages in grub, so tests looking for exact output
+# will likely fail.
+# STRACE_TESTS:
+# If set, strace individual tests. WARNING: This will cause testing to take
+# much longer which can cause some test to fail by timing out or having
+# incorrect timing
+# TEST_DATA_PREFIX:
+# Filename prefix to use when saving test data. If not set, test data
+# will not be saved.
+# GRUB_QEMU_OPTS:
+# Add options to qemu when used in tests (ex: -m size=64M)
+# GRUB_SHELL_DEFAULT_TIMEOUT [600s]:
+# Set grub-shell timeout for qemu instance (see man timeout duration
+# argument). The timeout has been raised from the default of 60 seconds
+# in the grub-shell script to 600 seconds because CI runners may run in
+# a slow virtual machine. This may need to be raised further if grub-shell
+# is failing with a timeout (exit code 124 or 137).
+# EXPECTED_FAILURES [see definition below]:
+# A list of make check tests allowed to fail, one per line. Test may be
+# prefixed with "$ARCHITECTURE-$PLATFORM:" to only apply to certain
+# targets. Lines beginning with '#' may be used as comments and are
+# ignored.
+# EXPECTED_FUNCTIONAL_FAILURES [see definition below]:
+# A list of functional tests from the grub_func_test make check that are
+# allowed to fail, one per line. Lines beginning with '#' may be used as
+# comments and are ignored.
+# DISABLED_TESTS [see definition below]:
+# A list of targets for which make check is not run, one per line. These
+# targets are disabled because tests mostly do not work. Lines beginning
+# with '#' may be used as comments and are ignored. Some hints as to why
+# the tests fail are included in comment lines. Patches which get these
+# tests working are very welcome.
+
+### Make image variables
+# MAKE_ALL_IMAGE_TARGETS:
+# If set 'y', all target image formats will be built, ignoring
+# DISABLED_IMAGES.
+# DISABLED_IMAGES [see definition below]:
+# A list of target formats as accepted by grub-mkimage for which images
+# are not built. These target formats are disabled generally because they
+# do not work. Lines beginning with '#' may be used as comments and are
+# ignored. Some hints as to why the tests fail are included in comment
+# lines. Patches which get these tests working are very welcome.
+
+
+##### CI configuration variables #####
+SDIR=$(realpath -e "${SDIR:-"${0%/*}"}")
+ROOT=$(realpath -m "${ROOT:-"./grub-tests"}")
+TMPDIR=${TMPDIR:-"$ROOT/testtmp"}
+CI_PROJECT_DIR=${CI_PROJECT_DIR:-"$ROOT"}
+CI_TYPE=${CI_TYPE:-"local"}
+SHELL_TRACE=${SHELL_TRACE:-"n"}
+
+GIT_REPO=${GIT_REPO}
+GIT_BRANCH=${GIT_BRANCH}
+GIT_REPO_DEFAULT="https://git.savannah.gnu.org/git/grub.git"
+GIT_BRANCH_DEFAULT="master"
+
+GIT_CLONE_ARGS=${GIT_CLONE_ARGS:-"--depth=1 --no-single-branch"}
+CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+SRCDIR=${SRCDIR:-"${CI_PROJECT_DIR}/grub"}
+
+if [ -d "${SDIR}/ci" ]; then
+ SCRIPTS_DIR=${SCRIPTS_DIR:-"${SDIR}"}
+else
+ SCRIPTS_DIR=${SCRIPTS_DIR:-"${SRCDIR}/scripts"}
+fi
+CI_SCRIPTS_DIR=${CI_SCRIPTS_DIR:-"${SCRIPTS_DIR}/ci"}
+
+CROSS_DIR=${CROSS_DIR:-$CI_PROJECT_DIR/cross}
+CROSS_VERSION=${CROSS_VERSION:-10.1.0}
+
+CONFIGURE_OPTS=${CONFIGURE_OPTS:-"--enable-boot-time"}
+DISABLE_ALL_BUILDS=${DISABLE_ALL_BUILDS:-"n"}
+
+IGNORE_TIMEDOUT_TEST_FAILURE=${IGNORE_TIMEDOUT_TEST_FAILURE:-"y"}
+GRUB_SHELL_DEFAULT_TIMEOUT=${GRUB_SHELL_DEFAULT_TIMEOUT:-"600s"}
+TEST_VERBOSITY=${TEST_VERBOSITY:-3}
+DISABLE_ALL_TESTS=${DISABLE_ALL_TESTS:-"n"}
+
+MATRIX=${MATRIX:-"
+ x86_64:x86_64-efi,x86_64-xen:x86_64-linux
+ i386:i386-coreboot,i386-efi,i386-ieee1275,i386-multiboot,i386-pc,i386-qemu,i386-xen,i386-xen_pvh:i386-linux
+ powerpc:powerpc-ieee1275:powerpc-linux
+ powerpc64:powerpc64-ieee1275:powerpc64-linux
+ sparc:sparc-ieee1275:sparc-linux
+ sparc64:sparc64-ieee1275:sparc64-linux
+ ia64:ia64-efi:ia64-linux
+ mips:mips-arc,mips-qemu_mips,mipsel-arc,mipsel-qemu_mips,mipsel-loongson:mips64-linux
+ arm:arm-coreboot,arm-efi,arm-uboot:arm-linux-gnueabi
+ arm64:arm64-efi:aarch64-linux
+ riscv32:riscv32-efi:riscv32-linux
+ riscv64:riscv64-efi:riscv64-linux
+"}
+
+APT_OPTS=${APT_OPTS:-"--no-upgrade"}
+PACKAGE_CACHE="${CI_PROJECT_DIR}/packages"
+PACKAGES="
+ multitee
+ git
+ ccache
+ ### Install cross compilers
+ tar
+ wget
+ xz-utils
+ ### Bootstrap Setup
+ autoconf
+ automake
+ autopoint
+ gettext
+ git
+ locales
+ patch
+ pkg-config
+ python
+ ### Grub build requirements
+ automake
+ bison
+ flex
+ gettext
+ libdevmapper-dev
+ libfreetype6-dev
+ libfuse-dev
+ liblzma-dev
+ libzfslinux-dev
+ make
+ pkg-config
+ python
+ unifont
+ ### Grub testing requirements
+ g++
+ gawk
+ kmod
+ strace
+ ### Virtualization tools
+ qemu-system
+ # qemu-efi-aarch64
+ # qemu-efi-arm
+ # qemu-efi
+ ovmf
+ # ovmf-ia32
+ openbios-ppc
+ openbios-sparc
+ ### Used by grub-fs-tester
+ wamerican
+ ### needed by grub-mkrescue
+ xorriso
+ ### needed by grub-mkrescue for efi platforms
+ mtools
+ ### Requirements for specific tests
+ parted
+ util-linux
+ ### Archivers
+ cpio
+ squashfs-tools
+ tar
+ ### Compression tools
+ gzip
+ lzop
+ xz-utils
+ ### File system tools
+ btrfs-progs
+ dosfstools
+ e2fsprogs
+ exfat-utils
+ f2fs-tools
+ genromfs
+ hfsprogs
+ jfsutils
+ nilfs-tools
+ ntfs-3g
+ reiserfsprogs
+ udftools
+ xfsprogs
+ zfs-fuse
+ ### Needed by hfs test
+ recode
+ ### Needed for ntfs test
+ attr
+ ### Extra packages specified by user
+ $PACKAGES
+"
+
+DEB_URLS="
+ http://mirrors.edge.kernel.org/ubuntu/pool/universe/e/edk2/ovmf-ia32_2020.11-4_all.deb
+ http://mirrors.edge.kernel.org/ubuntu/pool/main/e/edk2/qemu-efi-arm_2020.11-4_all.deb
+ http://mirrors.edge.kernel.org/ubuntu/pool/main/e/edk2/qemu-efi-aarch64_2020.11-4_all.deb
+"
+
+DISABLED_BUILDS=${DISABLED_BUILDS:-"
+ ### qemu-system-mips64: Could not load MIPS bios 'mips_bios.bin',
+ ### and no -kernel argument was specified
+ mipsel-arc
+ mipsel-qemu_mips
+ mipsel-loongson
+"}
+
+DISABLED_TESTS=${DISABLED_TESTS:-"
+ ### Failing on unreachable target error when invoking grub-mkimage
+ riscv32-efi
+ ### Need to get coreboot ROM and cbfstool
+ i386-coreboot
+ ### qemu-system-mips64: Could not load MIPS bios 'mips_bios.bin',
+ ### and no -kernel argument was specified
+ mips-arc
+ mips-qemu_mips
+ mipsel-arc
+ mipsel-qemu_mips
+ ### Need an i386 built open firmware image, otherwise uses regular bios
+ ### and stalls with 'Booting from Hard Disk...'
+ i386-ieee1275
+ ### Qemu logs 'Boot failed: Could not read from CDROM (code 0004)', which
+ ### according to https://lists.gnu.org/archive/html/qemu-devel/2008-06/msg00476.html
+ ### means 'cd is not eltorito (BRVD)'
+ i386-multiboot
+ ### TODO: These have no support in grub-shell and use the default
+ ### qemu-system-i386, which is clearly not correct
+ arm-coreboot
+ arm-uboot
+ ia64-efi
+ riscv64-efi
+ ### Also not supported in grub-shell
+ i386-xen
+ i386-xen_pvh
+ x86_64-xen
+"}
+
+DISABLED_IMAGES=${DISABLED_IMAGES:-"
+ ### Failing on unreachable target error when invoking grub-mkimage
+ riscv32-efi
+ ### FIXME: There is a bug in grub-mkimage which causes the modules
+ ### directory for these target formats to be not found
+ arm-coreboot-vexpress
+ arm-coreboot-veyron
+"}
+
+EXPECTED_FAILURES=${EXPECTED_FAILURES:-"
+ ### Usually this test passes, but occasionally the virtual machine is
+ ### too slow and the time delta threshold is exceeded. So allow failures.
+ #grub_cmd_sleep
+ ### This can fail on the volume label check because newer Debian versions
+ ### hfsprogs has a bug where mkfs.hfs does not write the volume label.
+ hfs_test
+ ### This fails because grub-fstest ls -la is not outputting file time
+ ### on files, but it is on directories. In linux there are file times.
+ ### Is this a bug in the zfs module? Also the SYMLINK test fails.
+ ### And the LIST test fails, where the error message 'error: unsupported
+ ### embedded BP (type=255)' is outputted.
+ zfs_test
+ ### sparc64-ieee1275 fails these tests, but the rest should succeed
+ ### https://marc.info/?i=20201219002902.4490b844%20()%20crass-HP-ZBook-15-G2
+ sparc64-ieee1275:grub_script_expansion
+ sparc64-ieee1275:gzcompress_test
+ sparc64-ieee1275:grub_func_test
+ sparc64-ieee1275:file_filter_test
+ ### This test is skipped by sparc64-ieee1275 due to OpenBIOS not
+ ### implementing RTC, could this be the same for powerpc-ieee1275?
+ ### This test fails because the returned date is +12h, why??
+ powerpc-ieee1275:grub_cmd_date
+ powerpc64-ieee1275:grub_cmd_date
+ ### This is supposed to sleep for 10 seconds, but the returned date
+ ### before and after is a difference of 2 seconds. RTC issue??
+ powerpc-ieee1275:grub_cmd_sleep
+ powerpc64-ieee1275:grub_cmd_sleep
+"}
+
+EXPECTED_FUNCTIONAL_FAILURES=${EXPECTED_FUNCTIONAL_FAILURES:-"
+ ### These have not worked for a very long time
+ gfxterm_menu
+ cmdline_cat_test
+ ### Sometimes the machine the test are running on are slow causing
+ ### timing tests to fail.
+ sleep_test
+"}
+
+# Exports for build.sh
+export SRCDIR BUILDDIR INSTALLDIR ARCH PLATFORM CONFIGURE_OPTS \
+ FONT_SOURCE DJVU_FONT_SOURCE SHELL_TRACE CI_TYPE
+# Exports for test.sh
+export SRCDIR BUILDDIR TARGET TEST_ALL_TARGETS DISABLED_TESTS TESTS_TIMEOUT \
+ TEST_VERBOSITY TEST_DATA_PREFIX STRACE_TESTS SHELL_TRACE \
+ GRUB_SHELL_DEFAULT_TIMEOUT GRUB_TEST_DEFAULT_DEBUG \
+ IGNORE_TIMEDOUT_TEST_FAILURE CI_TYPE
+# Exports for process-tests.sh
+export SRCDIR BUILDDIR TARGET TEST_ALL_TARGETS DISABLED_TESTS \
+ EXPECTED_FAILURES EXPECTED_FUNCTIONAL_FAILURES FAIL_ON_HARD_ERRORS \
+ IGNORE_TIMEDOUT_TEST_FAILURE NUM_FAILED_LOG_LINES SHELL_TRACE CI_TYPE
+# Exports for make-images.sh
+export MAKE_ALL_IMAGE_TARGETS DISABLED_IMAGES CI_TYPE
+
+
+if [ -z "$EUID" ]; then
+ export EUID=$(id -u)
+fi
+
+
+##### Logging functions #####
TXT_RED="\e[31m"
TXT_YELLOW="\e[33m"
TXT_CLEAR="\e[0m"
@@ -30,4 +437,518 @@ function end_log() {
echo -e "End:${LOG_NAME}"
}
-:;
+
+##### Miscelaneous functions #####
+wait_anypid() {
+ while kill -SIGWINCH "$@" 2>/dev/null; do
+ sleep .25
+ done
+}
+
+helper_process_redirects() {
+ # Process redirects by starting from the last argument and continuing
+ # until the first, stopping when the argument being looked at does not
+ # fit the format of a redirection. Each redirect argument is removed
+ # from the exec arguments because the command is not expecting them.
+ # Each redirect found is saved in REDIRS until right before the exec
+ # so that they can be done all at once and not be potentially cluttered
+ # with output from this function.
+ local REDIRS= i="$#"
+ while [ "$i" -gt 0 ]; do
+ local LASTARG=$(eval echo "\${$i}")
+ if echo "$LASTARG" | grep -qE '^[0-9]*[<>].*'; then
+ REDIRS="$LASTARG $REDIRS"
+ i=$((i-1))
+ continue
+ fi
+ break
+ done
+
+ eval $(echo -n "set -- \$"; seq -s ' ' 1 $i | sed 's/ / $/g')
+ (
+ set +x
+ eval exec "$REDIRS"
+ exec "$@"
+ )
+}
+
+# multipipe is a less featureful version of the shell's native pipe, except
+# that stderr is piped into the next command in the pipe as fd 3.
+multipipe() {
+ local ADELIM=$'\x01'
+ local ERROR_ANY RET_PID RET_PID_NUM IFS_OLD CMD
+ local PIPEFILEOUT PIPEFILEERR PIPEFILEOUTPREV PIPEFILEERRPREV
+
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ -e) ERROR_ANY=1; shift;;
+ -r) RET_PID_NUM="$2"; shift 2;;
+ *) break;;
+ esac
+ done
+
+ # For each command from last to first, start command with input and output
+ # as file descriptors to named pipes.
+ local i=0 max=$(echo "$*" | sed 's/|/\n/g' | wc -l) PID=
+ while read CMD; do
+ IFSOLD=$IFS
+ IFS=$ADELIM
+ set -- $(echo -n "$CMD")
+ if [ "$i" -eq 0 ]; then
+ PIPEFILEOUT=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEOUT
+ PIPEFILEERR=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEERR
+ helper_process_redirects "$@" <$PIPEFILEOUT 3<$PIPEFILEERR &
+ PID=$!
+ elif [ "$i" -eq "$((max-1))" ]; then
+ helper_process_redirects "$@" >$PIPEFILEOUT 2>$PIPEFILEERR &
+ PID=$!
+ rm -f $PIPEFILEOUT $PIPEFILEERR
+ else
+ PIPEFILEOUTPREV=$PIPEFILEOUT
+ PIPEFILEERRPREV=$PIPEFILEERR
+ PIPEFILEOUT=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEOUT
+ PIPEFILEERR=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEERR
+ (
+ # Need to run in subshell so we aren't overwriting file descriptors in
+ # the parent. Also we need to split up the redirects. First set stdout
+ # and stderr to the write ends of the pipes for the next command. We
+ # can not do this when calling helper_process_redirects as with the
+ # other redirects because helper_process_redirects is backgrounded
+ # causing a race with the following rm of those pipes, and the rm tends
+ # to win. We also can not open the read ends of the pipes for the
+ # previous command because the write end is not opened. So the exec
+ # which is not backgrounded would block, not allowing us to get to the
+ # point where the write ends of those pipes would be opened by the
+ # previous command in the pipeline.
+ exec >$PIPEFILEOUTPREV 2>$PIPEFILEERRPREV
+ helper_process_redirects "$@" <$PIPEFILEOUT 3<$PIPEFILEERR &
+ exit $!
+ )
+ PID=$?
+ rm -f $PIPEFILEOUTPREV $PIPEFILEERRPREV
+ fi
+
+ if [ "$ERROR_ANY" = "1" ]; then
+ RET_PID="$RET_PID $PID"
+ elif [ -z "$RET_PID_NUM" ] || [ "$RET_PID_NUM" -eq "$((max-$i))" ]; then
+ RET_PID=$PID
+ elif [ "$RET_PID_NUM" -eq "$((max-$i))" ]; then
+ RET_PID=$PID
+ fi
+
+ i=$((i+1))
+ IFS=$IFSOLD
+ done < <(IFS=$ADELIM; { echo -n "$*" | sed "s/${ADELIM}|${ADELIM}/\n/g"; echo; } | tac)
+
+ for PID in $RET_PID; do
+ wait -f $PID || return $?
+ done
+}
+
+multipipe_test() {
+ exec 2>/tmp/multipipe.trace.log
+ multipipe bash -x -c 'echo XXX $1' cmd arg1 '|' multitee 0:1,5 3:2,5 "5>/tmp/multipipe.file.log" ">/tmp/multipipe.stdout.log" "2>/tmp/multipipe.stderr.log"
+ multipipe bash -x -c 'echo XXX $1; echo FD3 >&4' cmd arg1 "4>/tmp/multipipe.bash.stderr.log" \
+ '|' multitee 0:1 3:2 5:4 "4>/tmp/multipipe.1.multitee.file.log" "5</etc/hosts" \
+ '|' multitee 0:1,5 3:2,5 "5>/tmp/multipipe.file.log" ">/tmp/multipipe.stdout.log" "2>/tmp/multipipe.stderr.log"
+}
+
+
+##### Functions for CI stages #####
+setup_distro() {
+ if [ ! -f /etc/os-release ]; then
+ echo "Could not determine OS version"
+ return
+ fi
+ . /etc/os-release
+ if [ "$ID" == "debian" ]; then
+ setup_debian
+ elif [ "$ID" == "ubuntu" ]; then
+ :
+ fi
+}
+
+setup_debian() {
+ if [ "$EUID" = 0 ]; then
+ sed -i 's/ main$/ main contrib non-free/' /etc/apt/sources.list
+ fi
+}
+
+setup_packages() {
+ start_log -c -n "setup_packages" "Installing required packages"
+ if ! which apt >/dev/null 2>&1; then
+ echo "No apt binary found, skipping package installation..."
+ return
+ fi
+ if [ "$EUID" != 0 ]; then
+ echo "Not root, continuing assuming that all needed packages are installed..."
+ return
+ fi
+ apt $APT_OPTS update -yqq &&
+ apt $APT_OPTS install -y $(grep -vE "^\s*#" <<<"$PACKAGES" | sort -u)
+ end_log -n "setup_packages"
+}
+
+setup_repo() {
+ start_log -c -n "setup_repo" "Setting up source repository"
+ # First find out if we are being run from a checked out branch
+ if [ -n "$(cd ${SDIR%/*}; git log -n1 "${SDIR##*/}" 2>/dev/null)" ]; then
+ # The script dir is in a git repo, assume we want to use that repo's
+ # working dir, unless a specific branch and repo url were given
+ GIT_REPO_DEFAULT="${SDIR%/*}"
+ while [ ! -d "${GIT_REPO_DEFAULT}/.git" ]; do
+ GIT_REPO_DEFAULT="${GIT_REPO_DEFAULT%/*}"
+ done
+ GIT_REPO_DEFAULT="file://${GIT_REPO_DEFAULT}"
+ fi
+
+ if [ -d "$SRCDIR" ]; then
+ # If the source directory already exists, assume that we should be using it.
+ # Make sure its up to date. Note, this can cause problems if the checked out
+ # repo is not in a clean state.
+ git -C "$SRCDIR" pull
+ if [ -n "${GIT_BRANCH}" ]; then
+ git -C "$SRCDIR" checkout "${GIT_BRANCH}"
+ fi
+ elif [ -n "$GIT_REPO" ]; then
+ # If the source dir does not exist, and a remote repo was specified, then
+ # clone that repo
+ git clone $GIT_CLONE_ARGS "$GIT_REPO" -b "${GIT_BRANCH:-$GIT_BRANCH_DEFAULT}" "$SRCDIR"
+ elif [ -d "$SRCDIR".external ]; then
+ # If a directory that is the source dir postfixed with .external exists
+ # then use that as the repo to clone from and use its gnulib dir.
+ git clone $GIT_CLONE_ARGS file://"${SRCDIR}.external" -b "${GIT_BRANCH:-$GIT_BRANCH_DEFAULT}" "$SRCDIR"
+ [ -d "${SRCDIR}.external/gnulib" ] &&
+ git clone --depth=1 file://"${SRCDIR}.external/gnulib" "$SRCDIR"/gnulib
+ else
+ # If all else fails use the default, which is official master
+ git clone $GIT_CLONE_ARGS "$GIT_REPO_DEFAULT" -b "${GIT_BRANCH:-$GIT_BRANCH_DEFAULT}" "$SRCDIR"
+ fi
+
+ export CI_COMMIT_SHORT_SHA=$(git -C "$SRCDIR" log -n1 --oneline | cut -d' ' -f1)
+
+ [ -f "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh" ] &&
+ . "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh"
+
+ end_log -n "setup_repo"
+}
+
+setup_build_scripts() {
+ if [ ! -d "${CI_SCRIPTS_DIR}" ]; then (
+ echo "No CI helper scripts found, retrieving from alternate repository"
+ cd "$SRCDIR"
+ git remote add ci "$GIT_CI_REPO"
+ # git ls-remote "$GIT_CI_REPO" "$GIT_CI_BRANCH"
+ git fetch ci "$GIT_CI_BRANCH"
+ git checkout "ci/$GIT_CI_BRANCH" -- scripts/ci
+ ) fi
+}
+
+setup_fonts() {
+ FONT_SOURCE=${FONT_SOURCE:-"/usr/share/fonts/X11/misc/unifont.pcf.gz"}
+ DJVU_FONT_SOURCE=${DJVU_FONT_SOURCE:-"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"}
+ export FONT_SOURCE DJVU_FONT_SOURCE
+}
+
+setup_loopback() {
+ if [ "$EUID" != 0 ]; then
+ echo "Insufficient privileges to create device files, skipping loopback setup..."
+ return
+ fi
+ if [ ! -e /dev/loop-control ]; then
+ mknod -m 0660 /dev/loop-control c 10 237 || :
+ fi
+ for i in $(seq 0 64); do
+ [ -e "/dev/loop$i" ] ||
+ mknod -m 0660 "/dev/loop$i" b 7 "$i" || :
+ done
+}
+
+setup_cross_compilers() {
+ local RET= CROSS_TARGET="$1"
+ start_log -c -n "setup_cross_compilers" "Downloading/Installing Cross Compilers for ${CROSS_TARGET}"
+ export HOST_ARCH=$(uname -m)
+ # These give us binaries like $CROSS_DIR/gcc-8.1.0-nolibc/ia64-linux/bin/ia64-linux-gcc
+ export CROSS_BIN_DIR="${CROSS_DIR}/gcc-${CROSS_VERSION}-nolibc/${CROSS_TARGET}/bin"
+ [ -d "${CROSS_BIN_DIR}" ] || {
+ # If not writable, fail immediately. This is stricter than we need to be.
+ test -w "$CROSS_DIR" &&
+ mkdir -pv "$CROSS_DIR" &&
+ wget -nv -t 3 -O - "https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/$HOST_ARCH/$CROSS_VERSION/$HOST_ARCH-gcc-$CROSS_VERSION-nolibc-$CROSS_TARGET.tar.xz" | \
+ tar xJ -C "$CROSS_DIR" &&
+ test -d "$CROSS_BIN_DIR" || RET=$?
+ }
+ export PATH="${CROSS_BIN_DIR}:${PATH}"
+ end_log -n "setup_cross_compilers"
+ return ${RET:-0}
+}
+
+setup_ccache() {
+ if ! which ccache >/dev/null 2>&1; then
+ echo "No ccache binary found, skipping ccache setup..."
+ return
+ fi
+ # CCache Config
+ export CCACHE_DIR="${1:-${CCACHE_DIR:-${CI_PROJECT_DIR}/ccache}}"
+ export CCACHE_CONFIGPATH=${CCACHE_DIR}/ccache.conf
+ export CCACHE_HARDLINK=true
+ export CCACHE_TEMPDIR="${XDG_RUNTIME_DIR:-/tmp}"/.ccache.$UID
+ export CCACHE_LOGFILE="${CCACHE_DIR}/ccache.log"
+ export CC="ccache gcc"
+ mkdir -p "${CCACHE_DIR}/bin" &&
+ ccache --show-stats
+}
+
+setup_qemu() {
+ for DEB_URL in $DEB_URLS; do
+ local DEB_FILENAME="${DEB_URL##*/}"
+ local DEB_PATH="$PACKAGE_CACHE/archives/${DEB_FILENAME}"
+ if [ ! -f "$DEB_PATH" ]; then
+ wget -nv -P "$PACKAGE_CACHE/archives" "$DEB_URL"
+ fi
+ [ "$EUID" = 0 ] && ! dpkg -l "${DEB_FILENAME%%_*}" >/dev/null 2>&1 &&
+ dpkg -i "$DEB_PATH"
+ done
+ # TODO: i386-efi: Unfortunately the non-Ubuntu debian package
+ # ovmf-ia32_2020.11-2_all.deb installs OVMF images that are not usable by
+ # QEMU's -bios option. This is the most official looking build found. Its
+ # probably possible to modify grub-shell to use successfully the files
+ # installed by ovmf-ia32 by using the QEMU option -drive if=pflash,...[1].
+ # I've submitted a bug report to debian to proposing that the file acceptable
+ # via -bios is added to the package.
+ # [1] https://wiki.debian.org/SecureBoot/VirtualMachine
+ [ -f "${SRCDIR}/OVMF-ia32.fd" ] ||
+ wget -nv -t 3 -O "${SRCDIR}/OVMF-ia32.fd" \
+ https://github.com/retrage/edk2-nightly/raw/master/bin/RELEASEIa32_OVMF.fd ||:
+ cp -vau /usr/share/qemu-efi-aarch64/QEMU_EFI.fd "${SRCDIR}/OVMF-aarch64.fd" ||:
+ cp -vau /usr/share/AAVMF/AAVMF32_CODE.fd "${SRCDIR}/OVMF-arm.fd" ||:
+}
+
+setup_bootstrap() {
+ local RET
+ start_log -c -n "bootstrap" "Bootstrap log"
+ # We cache the bootstrap, so if the bootstrap canary exists, don't run the bootstrap
+ if [ ! -f "$SRCDIR"/.bootstrap.finished ]; then
+ ( cd "$SRCDIR" && ./bootstrap ) &&
+ touch "$SRCDIR"/.bootstrap.finished || RET=1
+ fi
+ end_log -n "bootstrap"
+ return ${RET:-0}
+}
+
+run_matrix() {
+ local RET=0 MATRIX_ROW
+ for MATRIX_ROW in $(echo "${MATRIX}" | grep -vE '^\s*#'); do
+ run_matrix_row "$MATRIX_ROW" || RET=$?
+ done
+ return ${RET:-0}
+}
+
+run_matrix_row() {
+ local RET=0 MATRIX_ROW="$1"
+ if [ -n "$MATRIX_ROW" ]; then
+ export ARCH="${MATRIX_ROW%%:*}"
+ export GRUB_TARGETS=$(IFS=:; echo "${MATRIX_ROW}" | (read A G C; echo "$G"))
+ export CROSS_TARGET="${MATRIX_ROW##*:}"
+ fi
+
+ setup_cross_compilers "${CROSS_TARGET}" || return $?
+
+ for TARGET in $(echo "$GRUB_TARGETS" | tr , ' '); do
+ run_target "$TARGET" || RET=$?
+ if [ ${RET:-0} -ne 0 ]; then
+ echo -e "${TXT_RED}${TARGET} failed, continuing to next target.${TXT_CLEAR}"
+ fi
+ done
+ return ${RET:-0}
+}
+
+run_target() {
+ local RET=0
+ setup_env &&
+ (
+ run_build &&
+ run_test
+ ) || RET=$?
+
+ # Do processing of testing output
+ "${CI_SCRIPTS_DIR}/process-tests.sh" || RET=$?
+ # Make images for all formats
+ "${CI_SCRIPTS_DIR}/make-images.sh" || RET=$?
+
+ return ${RET:-0}
+}
+
+setup_env() {
+ local TARGET="${1:-${TARGET}}"
+ export BUILDDIR=$CI_PROJECT_DIR/build/obj-$TARGET
+ export INSTALLDIR=$CI_PROJECT_DIR/install/grub-$TARGET
+ export TEST_DATA_PREFIX=$CI_PROJECT_DIR/build/obj-${TARGET}/test-data
+
+ mkdir -pv "${BUILDDIR}" "${INSTALLDIR}"
+
+ export ARCH=${TARGET%-*}
+ export PLATFORM=${TARGET#*-}
+ if [ "x${BUILD_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_BUILDS}" | grep -qE "^\s*${TARGET}$" ); then
+ echo -e "${TXT_CLEAR}${TXT_YELLOW}Building $TARGET has been disabled, skipping.${TXT_CLEAR}"
+ return
+ fi
+}
+
+run_build() {
+ local RET=0
+ if [ "x${DISABLE_ALL_BUILDS}" = "xy" ]; then
+ echo "All builds are disabled"
+ return
+ fi
+
+ case "$ARCH" in
+ arm64) ARCH=aarch64-linux;;
+ arm) ARCH=arm-linux-gnueabi;;
+ mips) ARCH=mips64-linux;;
+ ia64|powerpc*|riscv32|riscv64|sparc*) ARCH=$ARCH-linux;;
+ i386|x86_64) ;;
+ *) echo "Unknown architecture":" $ARCH"; return 1;;
+ esac
+ case "$PLATFORM" in
+ coreboot)
+ export GRUB_CBFSTOOL="/where/is/cbfstool"
+ export GRUB_COREBOOT_ROM="/where/is/coreboot/rom";;
+ esac
+
+ cat >${BUILDDIR}/shell-wrapper.sh <<EOF
+#!$SHELL
+if [ "x\${SHELL_TRACE}" = "xy" ]; then
+ set -x
+ if [ -z "\$SHELL_OPTS" ] || [ -n "\${SHELL_OPTS##* -x*}" ]; then
+ SHELL_OPTS="\$SHELL_OPTS -x"
+ fi
+fi
+REAL_SHELL=\${REAL_SHELL:-\$(readlink /proc/\$\$/exe)}
+exec \$REAL_SHELL \$SHELL_OPTS "\$@"
+EOF
+ chmod +x ${BUILDDIR}/shell-wrapper.sh
+ export CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh
+ export CONFIGURE_OPTS="${CONFIGURE_OPTS} CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh"
+
+ if type multipipe >/dev/null; then
+ multipipe -e "${CI_SCRIPTS_DIR}/build.sh" \
+ '|' multitee 0:1,5 3:2,5 "5>${BUILDDIR}/build.log" || RET=$?
+ else
+ { { "${CI_SCRIPTS_DIR}/build.sh" || RET=$?; } |
+ tee >(exec cat >&3) >"${BUILDDIR}/build.log" 2>&1; } 3>&1 || :
+ fi
+ return $RET
+}
+
+run_test() {
+ local RET=0
+ if [ "x${DISABLE_ALL_TESTS}" = "xy" ]; then
+ echo "All tests are disabled"
+ return
+ fi
+
+ export TESTTMPDIR="${TMPDIR:-/tmp}/grub-test-$TARGET-${CI_COMMIT_SHORT_SHA}"
+ if type multipipe >/dev/null; then
+ multipipe -e "${CI_SCRIPTS_DIR}/test.sh" \
+ '|' multitee 0:1,5 3:2,5 "5>${BUILDDIR}/test.log" || RET=$?
+ else
+ { { "${CI_SCRIPTS_DIR}/test.sh" || RET=$?; } |
+ tee >(exec cat >&3) >"${BUILDDIR}/test.log" 2>&1; } 3>&1 || :
+ fi
+ return $RET
+}
+
+post_processing() {
+ local RET=0 SUMLOG="${CI_PROJECT_DIR}/build/test-summary.log"
+ : >"${SUMLOG}"
+ for F in "${CI_PROJECT_DIR}"/build/obj-*/test-results.log; do
+ [ ! -e "$F" ] && continue
+ local TARGETDIR="${F%/*}"
+ local TARGET="${TARGETDIR##*/}"
+ TARGET="${TARGET#*-}"
+ if ! echo "${MATRIX}" | grep -vE "^\s*#" | grep -qE "${TARGET}"; then
+ # Skip processing if the TARGET is not one of the ones in the MATRIX
+ continue
+ fi
+ echo "Ran test suite for target: ${TARGET}"
+ start_log -n "test-$TARGET" "Ran test suite for target: ${TARGET}" >>${SUMLOG}
+ grep "^${TARGET}:" "${TARGETDIR}/test-results.log" | sort |
+ ( IFS=:;
+ while read TARGET FTYPE TESTNAME; do
+ case "$FTYPE" in
+ skip)
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Skipped test: ${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ error)
+ if [ "x${FAIL_ON_HARD_ERRORS}" = "xy" ]; then
+ RET=1
+ fi
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Hard error in test: ${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ expected)
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Expected failed test: ${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ timeout)
+ if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" != "xy" ]; then
+ RET=1
+ TEST_IGNORED=" ignored"
+ fi
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Timed-out test${TEST_IGNORED}: ${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ unexpected)
+ RET=1
+ echo -e "${TXT_CLEAR}${TXT_RED} Failed test: ${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ *)
+ RET=1
+ echo -e "${TXT_CLEAR}${TXT_RED} Unhandled test type for ${TESTNAME} (type=$FTYPE)${TXT_CLEAR}" >>${SUMLOG};;
+ esac
+ done
+ exit ${RET:-0}
+ ) || RET=$?
+ echo -e -n "${TXT_CLEAR}" >>${SUMLOG}
+ end_log -n "test-$TARGET" >>${SUMLOG}
+ done
+ if [ -f "${SUMLOG}" ]; then
+ cat "${SUMLOG}"
+ fi
+ return ${RET:-0}
+}
+
+setup_all() {
+ setup_distro &&
+ setup_packages &&
+ setup_repo &&
+ setup_build_scripts &&
+ setup_fonts &&
+ setup_loopback &&
+ setup_ccache &&
+ setup_qemu &&
+ setup_bootstrap
+}
+
+all() {
+ setup_all && {
+ run_matrix ||:
+ post_processing
+ }
+}
+
+main() {
+ local CMD
+ mkdir -p "$ROOT" "$TMPDIR"
+ cd "$ROOT"
+
+ if [ -z "$*" ]; then
+ all
+ fi
+
+ for CMD in "$@"; do
+ $CMD
+ done
+}
+
+# If found source helper functions based on CI type. Do this here allows the
+# CI type functions to override these general default ones.
+[ -f "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh" ] &&
+. "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh"
--
2.27.0
^ permalink raw reply related [flat|nested] 4+ messages in thread