[PATCH v2 0/4] Few CI improvements

[PATCH v2 0/4] Few CI improvements

[Marek Marczykowski-Górecki]

- Add some more test jobs
- Allow selecting individual jobs, without editing yaml files

I don't think it needs to be included in 4.20, but may be considered later for
backporting.

Marek Marczykowski-Górecki (4):
  automation: skip building domU if there is no test defined for it
  automation: add jobs running tests from tools/tests/*
  automation: allow selecting individual jobs via CI variables
  automation: add tools/tests jobs on the AMD Zen3+ runner too

 automation/gitlab-ci/build.yaml    |  6 ++-
 automation/gitlab-ci/test.yaml     | 60 ++++++++++++++++++++++++-
 automation/scripts/build           |  1 +-
 automation/scripts/qubes-x86-64.sh | 78 +++++++++++++++++++++++--------
 automation/scripts/run-tools-tests | 47 +++++++++++++++++++-
 5 files changed, 173 insertions(+), 19 deletions(-)
 create mode 100755 automation/scripts/run-tools-tests

base-commit: 819c3cb186a86ef3e04fb5af4d9f9f6de032c3ee
-- 
git-series 0.9.1

[PATCH v2 1/4] automation: skip building domU if there is no test defined for it

[Marek Marczykowski-Górecki]

This will be useful for later tests not using generic domU (unit tests,
xtf etc).

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
Reviewed-by: Stefano Stabellini <stefano.stabellini@amd.com>
---
 automation/scripts/qubes-x86-64.sh | 50 +++++++++++++++++++------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/automation/scripts/qubes-x86-64.sh b/automation/scripts/qubes-x86-64.sh
index 8a0b7bfbc0d0..7eb3ce1bf703 100755
--- a/automation/scripts/qubes-x86-64.sh
+++ b/automation/scripts/qubes-x86-64.sh
@@ -144,26 +144,28 @@ disk = [ ]
 ${domU_extra_config}
 "
 
-# DomU
-mkdir -p rootfs
-cd rootfs
-# fakeroot is needed to preserve device nodes in rootless podman container
-fakeroot -s ../fakeroot-save tar xzf ../binaries/initrd.tar.gz
-mkdir proc
-mkdir run
-mkdir srv
-mkdir sys
-rm var/run
-echo "#!/bin/sh
+if [ -n "$domU_check" ]; then
+    # DomU
+    mkdir -p rootfs
+    cd rootfs
+    # fakeroot is needed to preserve device nodes in rootless podman container
+    fakeroot -s ../fakeroot-save tar xzf ../binaries/initrd.tar.gz
+    mkdir proc
+    mkdir run
+    mkdir srv
+    mkdir sys
+    rm var/run
+    echo "#!/bin/sh
 
 ${domU_check}
 " > etc/local.d/xen.start
-chmod +x etc/local.d/xen.start
-echo "rc_verbose=yes" >> etc/rc.conf
-sed -i -e 's/^Welcome/domU \0/' etc/issue
-find . | fakeroot -i ../fakeroot-save cpio -H newc -o | gzip > ../binaries/domU-rootfs.cpio.gz
-cd ..
-rm -rf rootfs
+    chmod +x etc/local.d/xen.start
+    echo "rc_verbose=yes" >> etc/rc.conf
+    sed -i -e 's/^Welcome/domU \0/' etc/issue
+    find . | fakeroot -i ../fakeroot-save cpio -H newc -o | gzip > ../binaries/domU-rootfs.cpio.gz
+    cd ..
+    rm -rf rootfs
+fi
 
 # DOM0 rootfs
 mkdir -p rootfs
@@ -188,11 +190,19 @@ ifconfig eth0 up
 ifconfig xenbr0 up
 ifconfig xenbr0 192.168.0.1
 
+" > etc/local.d/xen.start
+
+if [ -n "$domU_check" ]; then
+    echo "
 # get domU console content into test log
 tail -F /var/log/xen/console/guest-domU.log 2>/dev/null | sed -e \"s/^/(domU) /\" &
 xl create /etc/xen/domU.cfg
 ${dom0_check}
-" > etc/local.d/xen.start
+" >> etc/local.d/xen.start
+else
+    echo "${dom0_check}" >> etc/local.d/xen.start
+fi
+
 chmod +x etc/local.d/xen.start
 echo "$domU_config" > etc/xen/domU.cfg
 
@@ -201,7 +211,9 @@ echo "XENCONSOLED_TRACE=all" >> etc/default/xencommons
 echo "QEMU_XEN=/bin/false" >> etc/default/xencommons
 mkdir -p var/log/xen/console
 cp ../binaries/bzImage boot/vmlinuz
-cp ../binaries/domU-rootfs.cpio.gz boot/initrd-domU
+if [ -n "$domU_check" ]; then
+    cp ../binaries/domU-rootfs.cpio.gz boot/initrd-domU
+fi
 find . | fakeroot -i ../fakeroot-save cpio -H newc -o | gzip > ../binaries/dom0-rootfs.cpio.gz
 cd ..
 
-- 
git-series 0.9.1

[PATCH v2 2/4] automation: add jobs running tests from tools/tests/*

[Marek Marczykowski-Górecki]

There are a bunch of tests in tools/tests/, let them run in CI.
For each subdirectory expect "make run" will run the test, and observe
its exit code. This way, adding new tests is easy, and they will be
automatically picked up.

For better visibility, log test output to junit xml format, and let
gitlab ingest it. Set SUT_ADDR variable with name/address of the system
under test, so a network can be used to extract the file. The actual
address is set using DHCP. And for the test internal network, still add
the 192.168.0.1 IP (but don't replace the DHCP-provided one).

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
Changes in v2:
 - use bash shebang
 - clarify skipped message
 - cleanup extra printf params
 - limit calling DHCP in dom0 to only tests that need it
---
 automation/gitlab-ci/test.yaml     | 23 +++++++++++++++-
 automation/scripts/build           |  1 +-
 automation/scripts/qubes-x86-64.sh | 28 ++++++++++++++++++-
 automation/scripts/run-tools-tests | 47 +++++++++++++++++++++++++++++++-
 4 files changed, 99 insertions(+)
 create mode 100755 automation/scripts/run-tools-tests

diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 1822e3ea5fd7..c21a37933881 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -130,6 +130,7 @@
     PCIDEV: "03:00.0"
     PCIDEV_INTR: "MSI-X"
     CONSOLE_OPTS: "console=com1 com1=115200,8n1"
+    SUT_ADDR: test-2.testnet
   artifacts:
     paths:
       - smoke.serial
@@ -263,6 +264,28 @@ adl-pvshim-x86-64-gcc-debug:
     - *x86-64-test-needs
     - alpine-3.18-gcc-debug
 
+adl-tools-tests-pv-x86-64-gcc-debug:
+  extends: .adl-x86-64
+  script:
+    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
+  artifacts:
+    reports:
+      junit: tests-junit.xml
+  needs:
+    - *x86-64-test-needs
+    - alpine-3.18-gcc-debug
+
+adl-tools-tests-pvh-x86-64-gcc-debug:
+  extends: .adl-x86-64
+  script:
+    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
+  artifacts:
+    reports:
+      junit: tests-junit.xml
+  needs:
+    - *x86-64-test-needs
+    - alpine-3.18-gcc-debug
+
 zen3p-smoke-x86-64-gcc-debug:
   extends: .zen3p-x86-64
   script:
diff --git a/automation/scripts/build b/automation/scripts/build
index 952599cc25c2..522efe774ef3 100755
--- a/automation/scripts/build
+++ b/automation/scripts/build
@@ -109,5 +109,6 @@ else
     # even though dist/ contains everything, while some containers don't even
     # build Xen
     cp -r dist binaries/
+    cp -r tools/tests binaries/
     collect_xen_artefacts
 fi
diff --git a/automation/scripts/qubes-x86-64.sh b/automation/scripts/qubes-x86-64.sh
index 7eb3ce1bf703..7c80e0c23318 100755
--- a/automation/scripts/qubes-x86-64.sh
+++ b/automation/scripts/qubes-x86-64.sh
@@ -10,6 +10,8 @@ set -ex
 #  - pci-pv         PV dom0,  PV domU + PCI Passthrough
 #  - pvshim         PV dom0,  PVSHIM domU
 #  - s3             PV dom0,  S3 suspend/resume
+#  - tools-tests-pv PV dom0, run tests from tools/tests/*
+#  - tools-tests-pvh PVH dom0, run tests from tools/tests/*
 test_variant=$1
 
 ### defaults
@@ -19,6 +21,7 @@ timeout=120
 domU_type="pvh"
 domU_vif="'bridge=xenbr0',"
 domU_extra_config=
+retrieve_xml=
 
 case "${test_variant}" in
     ### test: smoke test & smoke test PVH & smoke test HVM & smoke test PVSHIM
@@ -126,6 +129,21 @@ done
 "
         ;;
 
+    ### tests: tools-tests-pv, tools-tests-pvh
+    "tools-tests-pv"|"tools-tests-pvh")
+        retrieve_xml=1
+        passed="test passed"
+        domU_check=""
+        dom0_check="
+/tests/run-tools-tests /tests /tmp/tests-junit.xml && echo \"${passed}\"
+nc -l -p 8080 < /tmp/tests-junit.xml >/dev/null &
+"
+        if [ "${test_variant}" = "tools-tests-pvh" ]; then
+            extra_xen_opts="dom0=pvh"
+        fi
+
+        ;;
+
     *)
         echo "Unrecognised test_variant '${test_variant}'" >&2
         exit 1
@@ -178,6 +196,8 @@ mkdir srv
 mkdir sys
 rm var/run
 cp -ar ../binaries/dist/install/* .
+cp -ar ../binaries/tests .
+cp -a ../automation/scripts/run-tools-tests tests/
 
 echo "#!/bin/bash
 
@@ -192,6 +212,10 @@ ifconfig xenbr0 192.168.0.1
 
 " > etc/local.d/xen.start
 
+if [ -n "$retrieve_xml" ]; then
+    echo "timeout 30s udhcpc -i xenbr0" >> etc/local.d/xen.start
+fi
+
 if [ -n "$domU_check" ]; then
     echo "
 # get domU console content into test log
@@ -272,6 +296,10 @@ if [ $timeout -le 0 ]; then
     exit 1
 fi
 
+if [ -n "$retrieve_xml" ]; then
+    nc -w 10 "$SUT_ADDR" 8080 > tests-junit.xml </dev/null
+fi
+
 sleep 1
 
 (grep -q "^Welcome to Alpine Linux" smoke.serial && grep -q "${passed}" smoke.serial) || exit 1
diff --git a/automation/scripts/run-tools-tests b/automation/scripts/run-tools-tests
new file mode 100755
index 000000000000..770e97c3e943
--- /dev/null
+++ b/automation/scripts/run-tools-tests
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 tests-dir xml-out"
+}
+
+xml_out=$2
+if [ -z "$xml_out" ]; then
+  xml_out=/dev/null
+fi
+printf '<?xml version="1.0" encoding="UTF-8"?>\n' > "$xml_out"
+printf '<testsuites name="tools.tests">\n' >> "$xml_out"
+printf ' <testsuite name="tools.tests">\n' >> "$xml_out"
+failed=
+for dir in "$1"/*; do
+    [ -d "$dir" ] || continue
+    echo "Running test in $dir"
+    printf '  <testcase name="%s">\n' "$dir" >> "$xml_out"
+    ret=
+    for f in "$dir"/*; do
+        [ -f "$f" ] || continue
+        [ -x "$f" ] || continue
+        "$f" 2>&1 | tee /tmp/out
+        ret=$?
+        if [ "$ret" -ne 0 ]; then
+            echo "FAILED: $ret"
+            failed+=" $dir"
+            printf '   <failure type="failure" message="binary %s exited with code %d">\n' "$f" "$ret" >> "$xml_out"
+            # TODO: could use xml escaping... but current tests seems to
+            # produce sane output
+            cat /tmp/out >> "$xml_out"
+            printf '   </failure>\n' >> "$xml_out"
+        else
+            echo "PASSED"
+        fi
+    done
+    if [ -z "$ret" ]; then
+        printf '   <skipped type="skipped" message="no executable test found in %s"/>\n' "$dir" >> "$xml_out"
+    fi
+    printf '  </testcase>\n' >> "$xml_out"
+done
+printf ' </testsuite>\n' >> "$xml_out"
+printf '</testsuites>\n' >> "$xml_out"
+
+if [ -n "$failed" ]; then
+    exit 1
+fi
-- 
git-series 0.9.1

[PATCH v2 3/4] automation: allow selecting individual jobs via CI variables

[Marek Marczykowski-Górecki]

Debugging sometimes involves running specific jobs on different
versions. It's useful to easily avoid running all of the not interesting
ones (for given case) to save both time and CI resources. Doing so used
to require changing the yaml files, usually in several places.
Ease this step by adding SELECTED_JOBS_ONLY variable that takes a regex.
Note that one needs to satisfy job dependencies on their own (for
example if a test job needs a build job, that specific build job
needs to be included too).

The variable can be specified via Gitlab web UI when scheduling a
pipeline, but it can be also set when doing git push directly:

    git push -o ci.variable=SELECTED_JOBS_ONLY="/job1|job2/"

More details at https://docs.gitlab.co.jp/ee/user/project/push_options.html

The variable needs to include regex for selecting jobs, including
enclosing slashes.
A coma/space separated list of jobs to select would be friendlier UX,
but unfortunately that is not supported:
https://gitlab.com/gitlab-org/gitlab/-/issues/209904 (note the proposed
workaround doesn't work for job-level CI_JOB_NAME).
On the other hand, the regex is more flexible (one can select for
example all arm32 jobs).

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
This probably wants documenting beyond this commit message. I don't
think we have any CI-related docs anywhere, do we? Some new file in
docs/misc?

And also, it's possible to extend web ui for starting pipelines to
include pre-defined variables. I use it in qubes here if you want to
see:
https://gitlab.com/QubesOS/qubes-continuous-integration/-/pipelines/new
Does it make sense to include SELECTED_JOBS_ONLY this way too?
Personally, I'll probably use it via cmdline push only anyway, but I
don't know what workflows other people have.
---
 automation/gitlab-ci/build.yaml |  6 ++++++
 automation/gitlab-ci/test.yaml  | 14 ++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
index 35e224366f62..f12de00a164a 100644
--- a/automation/gitlab-ci/build.yaml
+++ b/automation/gitlab-ci/build.yaml
@@ -12,6 +12,12 @@
       - '*/*.log'
     when: always
   needs: []
+  rules:
+  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
+    when: always
+  - if: $SELECTED_JOBS_ONLY
+    when: never
+  - when: on_success
 
 .gcc-tmpl:
   variables: &gcc
diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index c21a37933881..93632f1f9204 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -1,6 +1,11 @@
 .test-jobs-common:
   stage: test
   image: ${XEN_REGISTRY}/${CONTAINER}
+  rules:
+  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
+  - if: $SELECTED_JOBS_ONLY
+    when: never
+  - when: on_success
 
 .arm64-test-needs: &arm64-test-needs
   - alpine-3.18-arm64-rootfs-export
@@ -99,6 +104,9 @@
       - '*.dtb'
     when: always
   rules:
+    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
+    - if: $SELECTED_JOBS_ONLY
+      when: never
     - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
   tags:
     - xilinx
@@ -117,6 +125,9 @@
       - '*.log'
     when: always
   rules:
+    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
+    - if: $SELECTED_JOBS_ONLY
+      when: never
     - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
   tags:
     - xilinx
@@ -137,6 +148,9 @@
       - '*.log'
     when: always
   rules:
+    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
+    - if: $SELECTED_JOBS_ONLY
+      when: never
     - if: $QUBES_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
   tags:
     - qubes-hw2
-- 
git-series 0.9.1

[PATCH v2 4/4] automation: add tools/tests jobs on the AMD Zen3+ runner too

[Marek Marczykowski-Górecki]

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
new in v2
If those tests are sensitive to underlying hardware, I guess it makes
sense to run them on other runners too. Are they?
Similarly - does it matter if dom0 is PV or PVH for those tests? If not,
probably better to put just one of those jobs (PV?) to save CI
time.
---
 automation/gitlab-ci/test.yaml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 93632f1f9204..fc7663e3367a 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -162,6 +162,7 @@
     PCIDEV: "01:00.0"
     PCIDEV_INTR: "MSI-X"
     CONSOLE_OPTS: "console=com1 com1=115200,8n1,pci,msi"
+    SUT_ADDR: test-11.testnet
   tags:
     - qubes-hw11
 
@@ -340,6 +341,28 @@ zen3p-pvshim-x86-64-gcc-debug:
     - *x86-64-test-needs
     - alpine-3.18-gcc-debug
 
+zen3p-tools-tests-pv-x86-64-gcc-debug:
+  extends: .zen3p-x86-64
+  script:
+    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
+  artifacts:
+    reports:
+      junit: tests-junit.xml
+  needs:
+    - *x86-64-test-needs
+    - alpine-3.18-gcc-debug
+
+zen3p-tools-tests-pvh-x86-64-gcc-debug:
+  extends: .zen3p-x86-64
+  script:
+    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
+  artifacts:
+    reports:
+      junit: tests-junit.xml
+  needs:
+    - *x86-64-test-needs
+    - alpine-3.18-gcc-debug
+
 qemu-smoke-dom0-arm64-gcc:
   extends: .qemu-arm64
   script:
-- 
git-series 0.9.1

Re: [PATCH v2 2/4] automation: add jobs running tests from tools/tests/*

[Stefano Stabellini]

[-- Attachment #1: Type: text/plain, Size: 6893 bytes --]

On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> There are a bunch of tests in tools/tests/, let them run in CI.
> For each subdirectory expect "make run" will run the test, and observe
> its exit code. This way, adding new tests is easy, and they will be
> automatically picked up.
> 
> For better visibility, log test output to junit xml format, and let
> gitlab ingest it. Set SUT_ADDR variable with name/address of the system
> under test, so a network can be used to extract the file. The actual
> address is set using DHCP. And for the test internal network, still add
> the 192.168.0.1 IP (but don't replace the DHCP-provided one).
> 
> Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>

Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>


> ---
> Changes in v2:
>  - use bash shebang
>  - clarify skipped message
>  - cleanup extra printf params
>  - limit calling DHCP in dom0 to only tests that need it
> ---
>  automation/gitlab-ci/test.yaml     | 23 +++++++++++++++-
>  automation/scripts/build           |  1 +-
>  automation/scripts/qubes-x86-64.sh | 28 ++++++++++++++++++-
>  automation/scripts/run-tools-tests | 47 +++++++++++++++++++++++++++++++-
>  4 files changed, 99 insertions(+)
>  create mode 100755 automation/scripts/run-tools-tests
> 
> diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> index 1822e3ea5fd7..c21a37933881 100644
> --- a/automation/gitlab-ci/test.yaml
> +++ b/automation/gitlab-ci/test.yaml
> @@ -130,6 +130,7 @@
>      PCIDEV: "03:00.0"
>      PCIDEV_INTR: "MSI-X"
>      CONSOLE_OPTS: "console=com1 com1=115200,8n1"
> +    SUT_ADDR: test-2.testnet
>    artifacts:
>      paths:
>        - smoke.serial
> @@ -263,6 +264,28 @@ adl-pvshim-x86-64-gcc-debug:
>      - *x86-64-test-needs
>      - alpine-3.18-gcc-debug
>  
> +adl-tools-tests-pv-x86-64-gcc-debug:
> +  extends: .adl-x86-64
> +  script:
> +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
> +  artifacts:
> +    reports:
> +      junit: tests-junit.xml
> +  needs:
> +    - *x86-64-test-needs
> +    - alpine-3.18-gcc-debug
> +
> +adl-tools-tests-pvh-x86-64-gcc-debug:
> +  extends: .adl-x86-64
> +  script:
> +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
> +  artifacts:
> +    reports:
> +      junit: tests-junit.xml
> +  needs:
> +    - *x86-64-test-needs
> +    - alpine-3.18-gcc-debug
> +
>  zen3p-smoke-x86-64-gcc-debug:
>    extends: .zen3p-x86-64
>    script:
> diff --git a/automation/scripts/build b/automation/scripts/build
> index 952599cc25c2..522efe774ef3 100755
> --- a/automation/scripts/build
> +++ b/automation/scripts/build
> @@ -109,5 +109,6 @@ else
>      # even though dist/ contains everything, while some containers don't even
>      # build Xen
>      cp -r dist binaries/
> +    cp -r tools/tests binaries/
>      collect_xen_artefacts
>  fi
> diff --git a/automation/scripts/qubes-x86-64.sh b/automation/scripts/qubes-x86-64.sh
> index 7eb3ce1bf703..7c80e0c23318 100755
> --- a/automation/scripts/qubes-x86-64.sh
> +++ b/automation/scripts/qubes-x86-64.sh
> @@ -10,6 +10,8 @@ set -ex
>  #  - pci-pv         PV dom0,  PV domU + PCI Passthrough
>  #  - pvshim         PV dom0,  PVSHIM domU
>  #  - s3             PV dom0,  S3 suspend/resume
> +#  - tools-tests-pv PV dom0, run tests from tools/tests/*
> +#  - tools-tests-pvh PVH dom0, run tests from tools/tests/*
>  test_variant=$1
>  
>  ### defaults
> @@ -19,6 +21,7 @@ timeout=120
>  domU_type="pvh"
>  domU_vif="'bridge=xenbr0',"
>  domU_extra_config=
> +retrieve_xml=
>  
>  case "${test_variant}" in
>      ### test: smoke test & smoke test PVH & smoke test HVM & smoke test PVSHIM
> @@ -126,6 +129,21 @@ done
>  "
>          ;;
>  
> +    ### tests: tools-tests-pv, tools-tests-pvh
> +    "tools-tests-pv"|"tools-tests-pvh")
> +        retrieve_xml=1
> +        passed="test passed"
> +        domU_check=""
> +        dom0_check="
> +/tests/run-tools-tests /tests /tmp/tests-junit.xml && echo \"${passed}\"
> +nc -l -p 8080 < /tmp/tests-junit.xml >/dev/null &
> +"
> +        if [ "${test_variant}" = "tools-tests-pvh" ]; then
> +            extra_xen_opts="dom0=pvh"
> +        fi
> +
> +        ;;
> +
>      *)
>          echo "Unrecognised test_variant '${test_variant}'" >&2
>          exit 1
> @@ -178,6 +196,8 @@ mkdir srv
>  mkdir sys
>  rm var/run
>  cp -ar ../binaries/dist/install/* .
> +cp -ar ../binaries/tests .
> +cp -a ../automation/scripts/run-tools-tests tests/
>  
>  echo "#!/bin/bash
>  
> @@ -192,6 +212,10 @@ ifconfig xenbr0 192.168.0.1
>  
>  " > etc/local.d/xen.start
>  
> +if [ -n "$retrieve_xml" ]; then
> +    echo "timeout 30s udhcpc -i xenbr0" >> etc/local.d/xen.start
> +fi
> +
>  if [ -n "$domU_check" ]; then
>      echo "
>  # get domU console content into test log
> @@ -272,6 +296,10 @@ if [ $timeout -le 0 ]; then
>      exit 1
>  fi
>  
> +if [ -n "$retrieve_xml" ]; then
> +    nc -w 10 "$SUT_ADDR" 8080 > tests-junit.xml </dev/null
> +fi
> +
>  sleep 1
>  
>  (grep -q "^Welcome to Alpine Linux" smoke.serial && grep -q "${passed}" smoke.serial) || exit 1
> diff --git a/automation/scripts/run-tools-tests b/automation/scripts/run-tools-tests
> new file mode 100755
> index 000000000000..770e97c3e943
> --- /dev/null
> +++ b/automation/scripts/run-tools-tests
> @@ -0,0 +1,47 @@
> +#!/bin/bash
> +
> +usage() {
> +    echo "Usage: $0 tests-dir xml-out"
> +}
> +
> +xml_out=$2
> +if [ -z "$xml_out" ]; then
> +  xml_out=/dev/null
> +fi
> +printf '<?xml version="1.0" encoding="UTF-8"?>\n' > "$xml_out"
> +printf '<testsuites name="tools.tests">\n' >> "$xml_out"
> +printf ' <testsuite name="tools.tests">\n' >> "$xml_out"
> +failed=
> +for dir in "$1"/*; do
> +    [ -d "$dir" ] || continue
> +    echo "Running test in $dir"
> +    printf '  <testcase name="%s">\n' "$dir" >> "$xml_out"
> +    ret=
> +    for f in "$dir"/*; do
> +        [ -f "$f" ] || continue
> +        [ -x "$f" ] || continue
> +        "$f" 2>&1 | tee /tmp/out
> +        ret=$?
> +        if [ "$ret" -ne 0 ]; then
> +            echo "FAILED: $ret"
> +            failed+=" $dir"
> +            printf '   <failure type="failure" message="binary %s exited with code %d">\n' "$f" "$ret" >> "$xml_out"
> +            # TODO: could use xml escaping... but current tests seems to
> +            # produce sane output
> +            cat /tmp/out >> "$xml_out"
> +            printf '   </failure>\n' >> "$xml_out"
> +        else
> +            echo "PASSED"
> +        fi
> +    done
> +    if [ -z "$ret" ]; then
> +        printf '   <skipped type="skipped" message="no executable test found in %s"/>\n' "$dir" >> "$xml_out"
> +    fi
> +    printf '  </testcase>\n' >> "$xml_out"
> +done
> +printf ' </testsuite>\n' >> "$xml_out"
> +printf '</testsuites>\n' >> "$xml_out"
> +
> +if [ -n "$failed" ]; then
> +    exit 1
> +fi
> -- 
> git-series 0.9.1
> 

Re: [PATCH v2 3/4] automation: allow selecting individual jobs via CI variables

[Stefano Stabellini]

[-- Attachment #1: Type: text/plain, Size: 4492 bytes --]

On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> Debugging sometimes involves running specific jobs on different
> versions. It's useful to easily avoid running all of the not interesting
> ones (for given case) to save both time and CI resources. Doing so used
> to require changing the yaml files, usually in several places.
> Ease this step by adding SELECTED_JOBS_ONLY variable that takes a regex.
> Note that one needs to satisfy job dependencies on their own (for
> example if a test job needs a build job, that specific build job
> needs to be included too).
> 
> The variable can be specified via Gitlab web UI when scheduling a
> pipeline, but it can be also set when doing git push directly:
> 
>     git push -o ci.variable=SELECTED_JOBS_ONLY="/job1|job2/"
> 
> More details at https://docs.gitlab.co.jp/ee/user/project/push_options.html
> 
> The variable needs to include regex for selecting jobs, including
> enclosing slashes.

Does it work with a single job like this?

git push -o ci.variable=SELECTED_JOBS_ONLY="job1"

If it does, is there any way we could use to automatically whitelist its
dependencies too? Because that would be so much easier to use...


> A coma/space separated list of jobs to select would be friendlier UX,
> but unfortunately that is not supported:
> https://gitlab.com/gitlab-org/gitlab/-/issues/209904 (note the proposed
> workaround doesn't work for job-level CI_JOB_NAME).
> On the other hand, the regex is more flexible (one can select for
> example all arm32 jobs).
> 
> Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> ---
> This probably wants documenting beyond this commit message. I don't
> think we have any CI-related docs anywhere, do we? Some new file in
> docs/misc?

Please add documentation for it. Other than that, I think it could be
committed. I would prefer you also added documentation about alternative
methods such as removing all the jobs except the ones you care about.


> And also, it's possible to extend web ui for starting pipelines to
> include pre-defined variables. I use it in qubes here if you want to
> see:
> https://gitlab.com/QubesOS/qubes-continuous-integration/-/pipelines/new
> Does it make sense to include SELECTED_JOBS_ONLY this way too?
> Personally, I'll probably use it via cmdline push only anyway, but I
> don't know what workflows other people have.
> ---
>  automation/gitlab-ci/build.yaml |  6 ++++++
>  automation/gitlab-ci/test.yaml  | 14 ++++++++++++++
>  2 files changed, 20 insertions(+)
> 
> diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
> index 35e224366f62..f12de00a164a 100644
> --- a/automation/gitlab-ci/build.yaml
> +++ b/automation/gitlab-ci/build.yaml
> @@ -12,6 +12,12 @@
>        - '*/*.log'
>      when: always
>    needs: []
> +  rules:
> +  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> +    when: always
> +  - if: $SELECTED_JOBS_ONLY
> +    when: never
> +  - when: on_success
>  
>  .gcc-tmpl:
>    variables: &gcc
> diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> index c21a37933881..93632f1f9204 100644
> --- a/automation/gitlab-ci/test.yaml
> +++ b/automation/gitlab-ci/test.yaml
> @@ -1,6 +1,11 @@
>  .test-jobs-common:
>    stage: test
>    image: ${XEN_REGISTRY}/${CONTAINER}
> +  rules:
> +  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> +  - if: $SELECTED_JOBS_ONLY
> +    when: never
> +  - when: on_success
>  
>  .arm64-test-needs: &arm64-test-needs
>    - alpine-3.18-arm64-rootfs-export
> @@ -99,6 +104,9 @@
>        - '*.dtb'
>      when: always
>    rules:
> +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> +    - if: $SELECTED_JOBS_ONLY
> +      when: never
>      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
>    tags:
>      - xilinx
> @@ -117,6 +125,9 @@
>        - '*.log'
>      when: always
>    rules:
> +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> +    - if: $SELECTED_JOBS_ONLY
> +      when: never
>      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
>    tags:
>      - xilinx
> @@ -137,6 +148,9 @@
>        - '*.log'
>      when: always
>    rules:
> +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> +    - if: $SELECTED_JOBS_ONLY
> +      when: never
>      - if: $QUBES_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
>    tags:
>      - qubes-hw2
> -- 
> git-series 0.9.1
> 

Re: [PATCH v2 4/4] automation: add tools/tests jobs on the AMD Zen3+ runner too

[Stefano Stabellini]

[-- Attachment #1: Type: text/plain, Size: 1982 bytes --]

On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> ---
> new in v2
> If those tests are sensitive to underlying hardware, I guess it makes
> sense to run them on other runners too. Are they?
> Similarly - does it matter if dom0 is PV or PVH for those tests? If not,
> probably better to put just one of those jobs (PV?) to save CI
> time.

It should make a difference for the vpci test probably. I think we
should keep both a PV and a PVH test of it. I think it is less important
to run them on multiple runners, but it cannot hurt.

> ---
>  automation/gitlab-ci/test.yaml | 23 +++++++++++++++++++++++
>  1 file changed, 23 insertions(+)
> 
> diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> index 93632f1f9204..fc7663e3367a 100644
> --- a/automation/gitlab-ci/test.yaml
> +++ b/automation/gitlab-ci/test.yaml
> @@ -162,6 +162,7 @@
>      PCIDEV: "01:00.0"
>      PCIDEV_INTR: "MSI-X"
>      CONSOLE_OPTS: "console=com1 com1=115200,8n1,pci,msi"
> +    SUT_ADDR: test-11.testnet
>    tags:
>      - qubes-hw11

Is this a spurious change?



> @@ -340,6 +341,28 @@ zen3p-pvshim-x86-64-gcc-debug:
>      - *x86-64-test-needs
>      - alpine-3.18-gcc-debug
>  
> +zen3p-tools-tests-pv-x86-64-gcc-debug:
> +  extends: .zen3p-x86-64
> +  script:
> +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
> +  artifacts:
> +    reports:
> +      junit: tests-junit.xml
> +  needs:
> +    - *x86-64-test-needs
> +    - alpine-3.18-gcc-debug
> +
> +zen3p-tools-tests-pvh-x86-64-gcc-debug:
> +  extends: .zen3p-x86-64
> +  script:
> +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
> +  artifacts:
> +    reports:
> +      junit: tests-junit.xml
> +  needs:
> +    - *x86-64-test-needs
> +    - alpine-3.18-gcc-debug
> +
>  qemu-smoke-dom0-arm64-gcc:
>    extends: .qemu-arm64
>    script:
> -- 
> git-series 0.9.1
> 

Re: [PATCH v2 4/4] automation: add tools/tests jobs on the AMD Zen3+ runner too

[Marek Marczykowski-Górecki]

[-- Attachment #1: Type: text/plain, Size: 2415 bytes --]

On Fri, Feb 14, 2025 at 04:33:07PM -0800, Stefano Stabellini wrote:
> On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> > Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> > ---
> > new in v2
> > If those tests are sensitive to underlying hardware, I guess it makes
> > sense to run them on other runners too. Are they?
> > Similarly - does it matter if dom0 is PV or PVH for those tests? If not,
> > probably better to put just one of those jobs (PV?) to save CI
> > time.
> 
> It should make a difference for the vpci test probably. I think we
> should keep both a PV and a PVH test of it. I think it is less important
> to run them on multiple runners, but it cannot hurt.
> 
> > ---
> >  automation/gitlab-ci/test.yaml | 23 +++++++++++++++++++++++
> >  1 file changed, 23 insertions(+)
> > 
> > diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> > index 93632f1f9204..fc7663e3367a 100644
> > --- a/automation/gitlab-ci/test.yaml
> > +++ b/automation/gitlab-ci/test.yaml
> > @@ -162,6 +162,7 @@
> >      PCIDEV: "01:00.0"
> >      PCIDEV_INTR: "MSI-X"
> >      CONSOLE_OPTS: "console=com1 com1=115200,8n1,pci,msi"
> > +    SUT_ADDR: test-11.testnet
> >    tags:
> >      - qubes-hw11
> 
> Is this a spurious change?

No, the SUT_ADDR variable is used by the test script to extract the
junit xml file via network.

> > @@ -340,6 +341,28 @@ zen3p-pvshim-x86-64-gcc-debug:
> >      - *x86-64-test-needs
> >      - alpine-3.18-gcc-debug
> >  
> > +zen3p-tools-tests-pv-x86-64-gcc-debug:
> > +  extends: .zen3p-x86-64
> > +  script:
> > +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
> > +  artifacts:
> > +    reports:
> > +      junit: tests-junit.xml
> > +  needs:
> > +    - *x86-64-test-needs
> > +    - alpine-3.18-gcc-debug
> > +
> > +zen3p-tools-tests-pvh-x86-64-gcc-debug:
> > +  extends: .zen3p-x86-64
> > +  script:
> > +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
> > +  artifacts:
> > +    reports:
> > +      junit: tests-junit.xml
> > +  needs:
> > +    - *x86-64-test-needs
> > +    - alpine-3.18-gcc-debug
> > +
> >  qemu-smoke-dom0-arm64-gcc:
> >    extends: .qemu-arm64
> >    script:
> > -- 
> > git-series 0.9.1
> > 


-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH v2 3/4] automation: allow selecting individual jobs via CI variables

[Marek Marczykowski-Górecki]

[-- Attachment #1: Type: text/plain, Size: 6089 bytes --]

On Fri, Feb 14, 2025 at 04:29:17PM -0800, Stefano Stabellini wrote:
> On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> > Debugging sometimes involves running specific jobs on different
> > versions. It's useful to easily avoid running all of the not interesting
> > ones (for given case) to save both time and CI resources. Doing so used
> > to require changing the yaml files, usually in several places.
> > Ease this step by adding SELECTED_JOBS_ONLY variable that takes a regex.
> > Note that one needs to satisfy job dependencies on their own (for
> > example if a test job needs a build job, that specific build job
> > needs to be included too).
> > 
> > The variable can be specified via Gitlab web UI when scheduling a
> > pipeline, but it can be also set when doing git push directly:
> > 
> >     git push -o ci.variable=SELECTED_JOBS_ONLY="/job1|job2/"
> > 
> > More details at https://docs.gitlab.co.jp/ee/user/project/push_options.html
> > 
> > The variable needs to include regex for selecting jobs, including
> > enclosing slashes.
> 
> Does it work with a single job like this?
> 
> git push -o ci.variable=SELECTED_JOBS_ONLY="job1"

No, it works with:

    git push -o ci.variable=SELECTED_JOBS_ONLY="/job1/"

or rather:

    git push -o ci.variable=SELECTED_JOBS_ONLY="/^job1$/"

> If it does, is there any way we could use to automatically whitelist its
> dependencies too? Because that would be so much easier to use...

I guess it should be possible to add some extra condition for
dependencies, like extending rules for alpine-3.18-gcc-debug
specifically with

  - if: $SELECTED_JOBS_ONLY && "adl-smoke-x86-64-gcc-debug" =~ $SELECTED_JOBS_ONLY
    when: always

(and repeated for other tests depending on this build job)

But that means dependencies need to be kept in sync manually, in two
places. The absolute lack of any variables processing (even a simple
string concatenation...) at this stage of gitlab yaml processing makes
it challenging to propose any even semi-reasonable solution...

On the other hand, if you care about specific test, you can easily get
its dependencies by either looking at test.yaml, or clicking "show
depdendencies" in gitlab ui and hovering over the job you want.

> > A coma/space separated list of jobs to select would be friendlier UX,
> > but unfortunately that is not supported:
> > https://gitlab.com/gitlab-org/gitlab/-/issues/209904 (note the proposed
> > workaround doesn't work for job-level CI_JOB_NAME).
> > On the other hand, the regex is more flexible (one can select for
> > example all arm32 jobs).
> > 
> > Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> > ---
> > This probably wants documenting beyond this commit message. I don't
> > think we have any CI-related docs anywhere, do we? Some new file in
> > docs/misc?
> 
> Please add documentation for it. Other than that, I think it could be
> committed. I would prefer you also added documentation about alternative
> methods such as removing all the jobs except the ones you care about.

Do we really want to recommend removing unnecessary jobs, given a better
option exists now?
I know https://xkcd.com/1171/, but still...

> > And also, it's possible to extend web ui for starting pipelines to
> > include pre-defined variables. I use it in qubes here if you want to
> > see:
> > https://gitlab.com/QubesOS/qubes-continuous-integration/-/pipelines/new
> > Does it make sense to include SELECTED_JOBS_ONLY this way too?
> > Personally, I'll probably use it via cmdline push only anyway, but I
> > don't know what workflows other people have.
> > ---
> >  automation/gitlab-ci/build.yaml |  6 ++++++
> >  automation/gitlab-ci/test.yaml  | 14 ++++++++++++++
> >  2 files changed, 20 insertions(+)
> > 
> > diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
> > index 35e224366f62..f12de00a164a 100644
> > --- a/automation/gitlab-ci/build.yaml
> > +++ b/automation/gitlab-ci/build.yaml
> > @@ -12,6 +12,12 @@
> >        - '*/*.log'
> >      when: always
> >    needs: []
> > +  rules:
> > +  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > +    when: always
> > +  - if: $SELECTED_JOBS_ONLY
> > +    when: never
> > +  - when: on_success
> >  
> >  .gcc-tmpl:
> >    variables: &gcc
> > diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> > index c21a37933881..93632f1f9204 100644
> > --- a/automation/gitlab-ci/test.yaml
> > +++ b/automation/gitlab-ci/test.yaml
> > @@ -1,6 +1,11 @@
> >  .test-jobs-common:
> >    stage: test
> >    image: ${XEN_REGISTRY}/${CONTAINER}
> > +  rules:
> > +  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > +  - if: $SELECTED_JOBS_ONLY
> > +    when: never
> > +  - when: on_success
> >  
> >  .arm64-test-needs: &arm64-test-needs
> >    - alpine-3.18-arm64-rootfs-export
> > @@ -99,6 +104,9 @@
> >        - '*.dtb'
> >      when: always
> >    rules:
> > +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > +    - if: $SELECTED_JOBS_ONLY
> > +      when: never
> >      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
> >    tags:
> >      - xilinx
> > @@ -117,6 +125,9 @@
> >        - '*.log'
> >      when: always
> >    rules:
> > +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > +    - if: $SELECTED_JOBS_ONLY
> > +      when: never
> >      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
> >    tags:
> >      - xilinx
> > @@ -137,6 +148,9 @@
> >        - '*.log'
> >      when: always
> >    rules:
> > +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > +    - if: $SELECTED_JOBS_ONLY
> > +      when: never
> >      - if: $QUBES_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
> >    tags:
> >      - qubes-hw2
> > -- 
> > git-series 0.9.1
> > 


-- 
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [PATCH v2 4/4] automation: add tools/tests jobs on the AMD Zen3+ runner too

[Stefano Stabellini]

[-- Attachment #1: Type: text/plain, Size: 1621 bytes --]

On Sat, 15 Feb 2025, Marek Marczykowski-Górecki wrote:
> On Fri, Feb 14, 2025 at 04:33:07PM -0800, Stefano Stabellini wrote:
> > On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> > > Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> > > ---
> > > new in v2
> > > If those tests are sensitive to underlying hardware, I guess it makes
> > > sense to run them on other runners too. Are they?
> > > Similarly - does it matter if dom0 is PV or PVH for those tests? If not,
> > > probably better to put just one of those jobs (PV?) to save CI
> > > time.
> > 
> > It should make a difference for the vpci test probably. I think we
> > should keep both a PV and a PVH test of it. I think it is less important
> > to run them on multiple runners, but it cannot hurt.
> > 
> > > ---
> > >  automation/gitlab-ci/test.yaml | 23 +++++++++++++++++++++++
> > >  1 file changed, 23 insertions(+)
> > > 
> > > diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> > > index 93632f1f9204..fc7663e3367a 100644
> > > --- a/automation/gitlab-ci/test.yaml
> > > +++ b/automation/gitlab-ci/test.yaml
> > > @@ -162,6 +162,7 @@
> > >      PCIDEV: "01:00.0"
> > >      PCIDEV_INTR: "MSI-X"
> > >      CONSOLE_OPTS: "console=com1 com1=115200,8n1,pci,msi"
> > > +    SUT_ADDR: test-11.testnet
> > >    tags:
> > >      - qubes-hw11
> > 
> > Is this a spurious change?
> 
> No, the SUT_ADDR variable is used by the test script to extract the
> junit xml file via network.

Ah yes, I only looked at the patch without the context.

Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>

Re: [PATCH v2 3/4] automation: allow selecting individual jobs via CI variables

[Stefano Stabellini]

[-- Attachment #1: Type: text/plain, Size: 6926 bytes --]

On Sat, 15 Feb 2025, Marek Marczykowski-Górecki wrote:
> On Fri, Feb 14, 2025 at 04:29:17PM -0800, Stefano Stabellini wrote:
> > On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> > > Debugging sometimes involves running specific jobs on different
> > > versions. It's useful to easily avoid running all of the not interesting
> > > ones (for given case) to save both time and CI resources. Doing so used
> > > to require changing the yaml files, usually in several places.
> > > Ease this step by adding SELECTED_JOBS_ONLY variable that takes a regex.
> > > Note that one needs to satisfy job dependencies on their own (for
> > > example if a test job needs a build job, that specific build job
> > > needs to be included too).
> > > 
> > > The variable can be specified via Gitlab web UI when scheduling a
> > > pipeline, but it can be also set when doing git push directly:
> > > 
> > >     git push -o ci.variable=SELECTED_JOBS_ONLY="/job1|job2/"
> > > 
> > > More details at https://docs.gitlab.co.jp/ee/user/project/push_options.html
> > > 
> > > The variable needs to include regex for selecting jobs, including
> > > enclosing slashes.
> > 
> > Does it work with a single job like this?
> > 
> > git push -o ci.variable=SELECTED_JOBS_ONLY="job1"
> 
> No, it works with:
> 
>     git push -o ci.variable=SELECTED_JOBS_ONLY="/job1/"
> 
> or rather:
> 
>     git push -o ci.variable=SELECTED_JOBS_ONLY="/^job1$/"
> 
> > If it does, is there any way we could use to automatically whitelist its
> > dependencies too? Because that would be so much easier to use...
> 
> I guess it should be possible to add some extra condition for
> dependencies, like extending rules for alpine-3.18-gcc-debug
> specifically with
> 
>   - if: $SELECTED_JOBS_ONLY && "adl-smoke-x86-64-gcc-debug" =~ $SELECTED_JOBS_ONLY
>     when: always
> 
> (and repeated for other tests depending on this build job)
> 
> But that means dependencies need to be kept in sync manually, in two
> places. The absolute lack of any variables processing (even a simple
> string concatenation...) at this stage of gitlab yaml processing makes
> it challenging to propose any even semi-reasonable solution...

Yeah I agree with you


> On the other hand, if you care about specific test, you can easily get
> its dependencies by either looking at test.yaml, or clicking "show
> depdendencies" in gitlab ui and hovering over the job you want.

I am less sure about this. It becomes a matter of personal taste but if
I need to figure out the dependencies, I prefer to do it with vim, and
if I do that, I can very quickly edit the yaml file deleting all the
unneeded jobs. I can see someone else might instead prefer to write down
the list and setup the regex for SELECTED_JOBS_ONLY.

In an ideal world, we wouldn't need to do that because there would be a
way to do this automatically. Unfortunately it is not the case.

I am OK with this patch, I am sure someone will find it useful. It is
just that its usefulness is limited due to the restrictions we have with
Gitlab.



> > > A coma/space separated list of jobs to select would be friendlier UX,
> > > but unfortunately that is not supported:
> > > https://gitlab.com/gitlab-org/gitlab/-/issues/209904 (note the proposed
> > > workaround doesn't work for job-level CI_JOB_NAME).
> > > On the other hand, the regex is more flexible (one can select for
> > > example all arm32 jobs).
> > > 
> > > Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> > > ---
> > > This probably wants documenting beyond this commit message. I don't
> > > think we have any CI-related docs anywhere, do we? Some new file in
> > > docs/misc?
> > 
> > Please add documentation for it. Other than that, I think it could be
> > committed. I would prefer you also added documentation about alternative
> > methods such as removing all the jobs except the ones you care about.
> 
> Do we really want to recommend removing unnecessary jobs, given a better
> option exists now?
> I know https://xkcd.com/1171/, but still...

Not recommend! Only document as a way of explanation.


> > > And also, it's possible to extend web ui for starting pipelines to
> > > include pre-defined variables. I use it in qubes here if you want to
> > > see:
> > > https://gitlab.com/QubesOS/qubes-continuous-integration/-/pipelines/new
> > > Does it make sense to include SELECTED_JOBS_ONLY this way too?
> > > Personally, I'll probably use it via cmdline push only anyway, but I
> > > don't know what workflows other people have.
> > > ---
> > >  automation/gitlab-ci/build.yaml |  6 ++++++
> > >  automation/gitlab-ci/test.yaml  | 14 ++++++++++++++
> > >  2 files changed, 20 insertions(+)
> > > 
> > > diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
> > > index 35e224366f62..f12de00a164a 100644
> > > --- a/automation/gitlab-ci/build.yaml
> > > +++ b/automation/gitlab-ci/build.yaml
> > > @@ -12,6 +12,12 @@
> > >        - '*/*.log'
> > >      when: always
> > >    needs: []
> > > +  rules:
> > > +  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > > +    when: always
> > > +  - if: $SELECTED_JOBS_ONLY
> > > +    when: never
> > > +  - when: on_success
> > >  
> > >  .gcc-tmpl:
> > >    variables: &gcc
> > > diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> > > index c21a37933881..93632f1f9204 100644
> > > --- a/automation/gitlab-ci/test.yaml
> > > +++ b/automation/gitlab-ci/test.yaml
> > > @@ -1,6 +1,11 @@
> > >  .test-jobs-common:
> > >    stage: test
> > >    image: ${XEN_REGISTRY}/${CONTAINER}
> > > +  rules:
> > > +  - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > > +  - if: $SELECTED_JOBS_ONLY
> > > +    when: never
> > > +  - when: on_success
> > >  
> > >  .arm64-test-needs: &arm64-test-needs
> > >    - alpine-3.18-arm64-rootfs-export
> > > @@ -99,6 +104,9 @@
> > >        - '*.dtb'
> > >      when: always
> > >    rules:
> > > +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > > +    - if: $SELECTED_JOBS_ONLY
> > > +      when: never
> > >      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
> > >    tags:
> > >      - xilinx
> > > @@ -117,6 +125,9 @@
> > >        - '*.log'
> > >      when: always
> > >    rules:
> > > +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > > +    - if: $SELECTED_JOBS_ONLY
> > > +      when: never
> > >      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
> > >    tags:
> > >      - xilinx
> > > @@ -137,6 +148,9 @@
> > >        - '*.log'
> > >      when: always
> > >    rules:
> > > +    - if: $SELECTED_JOBS_ONLY && $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY
> > > +    - if: $SELECTED_JOBS_ONLY
> > > +      when: never
> > >      - if: $QUBES_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
> > >    tags:
> > >      - qubes-hw2
> > > -- 
> > > git-series 0.9.1
> > >