From mboxrd@z Thu Jan 1 00:00:00 1970 From: Romain Naour Date: Sun, 27 Jun 2021 16:47:42 +0200 Subject: [Buildroot] [PATCH 2/2 v4] utils/test-pkg: add gitlab-ci support In-Reply-To: References: Message-ID: <1f4d9886-8c1b-e804-bbca-edb62bab4086@gmail.com> List-Id: MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit To: buildroot@busybox.net Le 27/06/2021 ? 16:09, Yann E. MORIN a ?crit?: > From: Romain Naour > > The gitlab-ci support in test-pkg allows to parallelize the test-pkg > work into several gitlab jobs. It's much faster than local serialized > testing. > > We allow that with two main changes: > > - the test-pkg script is enhanced to be able to only list the builds > that are not skipped (i.e. for which the configuration fragment > yield an actual, valid .config) > > We add a new option, --list-only, which returns that list to stdout, > and still outputs the previous traces, but to stderr (for post- > mortem analysis, and for local testing to have quick feedback) > > - the script that generates the pipline, uses that list to create one pipeline > job for each test. > > This is triggered when the last commit log of the series (HEAD), > contains the 'test-pkg config:' directive all alone on its own line, > followed by a configuration fragment snippet to be used as input to > test-pkg. > > If the user provides an empty fragment, this is considered an error: > indeed, without a fragment (and the package name), there is noway to > know what to test. > > Furtheremore, if that fragment yields an empty list of tests, then Furthermore > there is nothing to test either, so that is also considered an > error. > > As far as Gitlab-CI itslef is concerned, we need to propagate a bit of itself > information from the parent pipeline to the chile pipeline. Indeed, all child > the per-test config files are generated in the parent pipeline, but used > in the child. > > So we export the parent pipeline ID, and use that to retrieve all the > per-test .config files, as per the gitlab-CI documentation [0] (we can't > save the whole of the "br-test/" directory, because that will hit the > 5-MiB limit in the public Gitlab-CI instance). > > [0] https://docs.gitlab.com/ee/ci/yaml/README.html#artifact-downloads-to-child-pipelines > > Signed-off-by: Romain Naour > Cc: Arnout Vandecappelle (Essensium/Mind) > [yann.morin.1998 at free.fr: > - do not inject yml-knowledge in test-pkg, make it totally agnostic to > the testing infra: just list matching builds > - generate the actual yml snippet in support/scripts/generate-gitlab-ci-yml, > using the list emitted by test-pkg > - some code-style-candies... > ] Thanks for the final rework! > Signed-off-by: Yann E. MORIN > > --- > v4: reworked by Yann: > tst-pkg only lists, does not generate the yml code > The yml code is generated by support/scripts/generate-gitlab-ci-yml > Empty fragment is an error > Fragment that yierlds no test is also an error > > v3: Implement Arnout's review: http://lists.busybox.net/pipermail/buildroot/2021-May/310656.html > Enable artifacts download from child-pipeline > > v2: Rework this patch following Arnout review > use CI_COMMIT_DESCRIPTION > remove .config from artifacts but keep images directory since > it can be useful for further issue investigation > use the "br-test-pkg" prefix for test-pkg jobs > --- > .gitlab-ci.yml | 3 ++ > support/misc/gitlab-ci.yml.in | 22 +++++++++++++ > support/scripts/generate-gitlab-ci-yml | 30 +++++++++++++++++- > utils/test-pkg | 44 ++++++++++++++++++++------ > 4 files changed, 88 insertions(+), 11 deletions(-) > > diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml > index e85ac32033..31eb495859 100644 > --- a/.gitlab-ci.yml > +++ b/.gitlab-ci.yml > @@ -13,6 +13,7 @@ generate-gitlab-ci-yml: > artifacts: > paths: > - generated-gitlab-ci.yml > + - br-test-pkg/*/.config > > buildroot-pipeline: > stage: build > @@ -21,3 +22,5 @@ buildroot-pipeline: > - artifact: generated-gitlab-ci.yml > job: generate-gitlab-ci-yml > strategy: depend > + variables: > + PARENT_PIPELINE_ID: $CI_PIPELINE_ID > diff --git a/support/misc/gitlab-ci.yml.in b/support/misc/gitlab-ci.yml.in > index 1ee3772154..be7951b3d2 100644 > --- a/support/misc/gitlab-ci.yml.in > +++ b/support/misc/gitlab-ci.yml.in > @@ -80,3 +80,25 @@ > - test-output/*/.config > - test-output/*/images/* > > +.test_pkg: > + stage: build > + before_script: > + - OUTPUT_DIR=${CI_JOB_NAME} > + script: > + - echo "Configure Buildroot for ${OUTPUT_DIR}" > + - make O=${OUTPUT_DIR} syncconfig > + - make O=${OUTPUT_DIR} savedefconfig > + - echo 'Build buildroot' > + - *run_make > + needs: > + - pipeline: $PARENT_PIPELINE_ID > + job: generate-gitlab-ci-yml > + artifacts: > + when: always > + expire_in: 2 weeks > + paths: > + - build.log > + - br-test-pkg/*/.config > + - br-test-pkg/*/defconfig > + - br-test-pkg/*/build/build-time.log > + - br-test-pkg/*/build/packages-file-list*.txt > diff --git a/support/scripts/generate-gitlab-ci-yml b/support/scripts/generate-gitlab-ci-yml > index 3f498e08fd..fc0f8cedf7 100755 > --- a/support/scripts/generate-gitlab-ci-yml > +++ b/support/scripts/generate-gitlab-ci-yml > @@ -23,7 +23,7 @@ _EOF_ > > gen_tests() { > local -a basics defconfigs runtimes > - local do_basics do_defconfigs do_runtime > + local do_basics do_defconfigs do_runtime do_testpkg > local defconfigs_ext cfg tst > > basics=( DEVELOPERS flake8 package ) > @@ -77,9 +77,33 @@ gen_tests() { > esac > fi > > + # Retrieve defconfig for test-pkg from the git commit message (if any) > + echo "$CI_COMMIT_DESCRIPTION" \ > + | sed -n '/^test-pkg config:$/,/^$/p' \ > + > defconfig.frag > + > + if [ -s defconfig.frag ]; then > + sed -i 1d defconfig.frag > + if [ ! -s defconfig.frag ]; then > + printf "Empty configuration fragment.\n" >&2; exit 1 > + fi > + # Use --all since we expect the user having already pre-tested the new package > + # with the default subset of toolchains. > + do_testpkg=( $( ./utils/test-pkg \ > + --all --list-only \ > + --config-snippet defconfig.frag \ > + --build-dir br-test-pkg \ > + ) > + ) > + if [ "${#do_testpkg[@]}" -eq 0 ]; then > + printf "Configuration fragment enables no test.\n" >&2; exit 1 > + fi > + fi > + > # If nothing else, at least do the basics to generate a valid pipeline > if [ -z "${do_defconfigs}" \ > -a -z "${do_runtime}" \ > + -a -z "${do_testpkg}" \ > ] > then > do_basics=true > @@ -101,6 +125,10 @@ gen_tests() { > if ${do_runtime:-false}; then > printf '%s: { extends: .runtime_test_base }\n' "${runtimes[@]}" > fi > + > + if [ -n "${do_testpkg}" ]; then > + printf '%s: { extends: .test_pkg }\n' "${do_test_pkg[@]}" s/do_test_pkg/do_testpkg/ With that fixed: https://gitlab.com/kubu93/buildroot/-/pipelines/327870496 Best regards, Romain > + fi > } > > main "${@}" > diff --git a/utils/test-pkg b/utils/test-pkg > index a317d8c17a..2349674069 100755 > --- a/utils/test-pkg > +++ b/utils/test-pkg > @@ -10,15 +10,19 @@ do_clean() { > fi > } > > +trace() { > + printf "${@}" > +} > + > main() { > local o O opts > - local cfg dir pkg random toolchains_csv toolchain all number mode > + local cfg dir pkg random toolchains_csv toolchain all number mode list_only > local ret nb nb_skip nb_fail nb_legal nb_tc build_dir keep > local -a toolchains > local pkg_br_name > > - o='hakc:d:n:p:r:t:' > - O='help,all,keep,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:' > + o='hakgc:d:n:p:r:t:' > + O='help,all,keep,config-snippet:,build-dir:,list-only,number:,package:,random:,toolchains-csv:' > opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")" > eval set -- "${opts}" > > @@ -27,6 +31,7 @@ main() { > keep=0 > number=0 > mode=0 > + list_only=0 > toolchains_csv="${TOOLCHAINS_CSV}" > while [ ${#} -gt 0 ]; do > case "${1}" in > @@ -39,6 +44,10 @@ main() { > (-k|--keep) > keep=1; shift 1 > ;; > + (-l|--list-only) > + list_only=1; shift 1 > + trace() { printf "${@}" >&2; } > + ;; > (-c|--config-snippet) > cfg="${2}"; shift 2 > ;; > @@ -118,6 +127,11 @@ main() { > printf "error: no toolchain found (networking issue?)\n" >&2; exit 1 > fi > > + if [ -n "${list_file}" ]; then > + # Running in list-only implies keeping the build directories. > + keep=1 > + fi > + > nb=0 > nb_skip=0 > nb_fail=0 > @@ -126,17 +140,21 @@ main() { > : $((nb++)) > toolchain="$(basename "${toolchainconfig}" .config)" > build_dir="${dir}/${toolchain}" > - printf "%40s [%*d/%d]: " "${toolchain}" ${#nb_tc} ${nb} ${nb_tc} > - build_one "${build_dir}" "${toolchainconfig}" "${cfg}" "${pkg}" && ret=0 || ret=${?} > + trace "%40s [%*d/%d]: " "${toolchain}" ${#nb_tc} ${nb} ${nb_tc} > + build_one "${build_dir}" "${toolchainconfig}" "${cfg}" "${pkg}" "${list_only}" && ret=0 || ret=${?} > case ${ret} in > - (0) printf "OK\n";; > - (1) : $((nb_skip++)); printf "SKIPPED\n";; > - (2) : $((nb_fail++)); printf "FAILED\n";; > - (3) : $((nb_legal++)); printf "FAILED\n";; > + (0) trace "OK\n" > + if [ ${list_only} -eq 1 ]; then > + printf '%s\n' "$build_dir" > + fi > + ;; > + (1) : $((nb_skip++)); trace "SKIPPED\n";; > + (2) : $((nb_fail++)); trace "FAILED\n";; > + (3) : $((nb_legal++)); trace "FAILED\n";; > esac > done > > - printf "%d builds, %d skipped, %d build failed, %d legal-info failed\n" \ > + trace "%d builds, %d skipped, %d build failed, %d legal-info failed\n" \ > ${nb} ${nb_skip} ${nb_fail} ${nb_legal} > > return $((nb_fail + nb_legal)) > @@ -147,6 +165,7 @@ build_one() { > local toolchainconfig="${2}" > local cfg="${3}" > local pkg="${4}" > + local defer="${5}" > > mkdir -p "${dir}" > > @@ -166,6 +185,11 @@ build_one() { > # Remove file, it's empty anyway. > rm -f "${dir}/missing.config" > > + # Defer building the job to the caller (e.g. a gitlab pipeline) > + if [ ${defer} -eq 1 ]; then > + return 0 > + fi > + > if [ -n "${pkg}" ]; then > if ! make O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then > return 2 >