[PATCH drm-test] drm-test: run the tests with libtest

[PATCH drm-test] drm-test: run the tests with libtest

[Edwin Peer]

Replace the hand rolled test runner with libtest, the harness behind the
Cargo #[test] attribute. Tests are now discovered from their annotations
rather than registered by hand - adding one no longer means implementing
the Test trait.

Switching to libtest brings a number of benefits:

  - Tests are independent. Each one runs with panics caught, so a failed
    assertion reports just that test as failed instead of aborting the
    process and leaving the remaining tests unreported. Pass --fail-fast
    (nightly, with -Z unstable-options) to stop at the first failure
    instead.

  - Results are reported per test, with a summary and a non-zero exit
    status when any test fails, which is easier to consume from CI than
    a single verdict for the whole suite.

  - Tests are selected by name, which replaces the driver dispatch that
    keyed off argv[1]. Tests are namespaced by their module path, so the
    previous invocation still selects the nova tests:

        drm-test nova

    while individual tests can now be run on their own:

        drm-test nova::test::test_gem_size

    and --list enumerates the tests without touching the hardware. See
    --exact and --skip for the remaining filtering options.

  - Test output is captured and printed only for failing tests, so that
    passing runs stay quiet without losing diagnostics when they are
    needed. Pass --show-output to print the output of successful tests
    too, or --no-capture to print it as it happens, as before.

Note that libtest runs tests in parallel; pass --test-threads=1 to run
them one at a time.

libtest is linked into the test binary, so cargo is only required to
build the tests, not to run them. The binary is built from the --tests
target, which cargo keeps in an internal location that differs between
releases, so add build.sh to place it in the target directory, where the
binary was found before.

Signed-off-by: Edwin Peer <epeer@nvidia.com>
---
This patch is for drm-test, the userspace test suite for the nova uAPI. It
lives outside the kernel tree, at:

  https://gitlab.freedesktop.org/dakr/drm-test

It is posted here for wider review, and is also open as a merge request
against that repository:

  https://gitlab.freedesktop.org/dakr/drm-test/-/merge_requests/3

The motivation is running these tests from CI. The current runner reports a
single verdict for the whole suite, and a failed assertion aborts the process,
so the tests after it go unreported. libtest reports a result per test and,
with --format json, machine readable output.

Tested on Turing (TU104), Ampere (GA104) and Blackwell (GB102), against
drm-rust-next.

 build.sh         |  49 ++++++++++++++
 src/drm/mod.rs   |   5 --
 src/lib.rs       |   2 +
 src/main.rs      |  18 ------
 src/nova/mod.rs  |   4 +-
 src/nova/test.rs | 163 +++++++++++++++++++++--------------------------
 6 files changed, 124 insertions(+), 117 deletions(-)
 create mode 100755 build.sh
 delete mode 100644 src/main.rs

diff --git a/build.sh b/build.sh
new file mode 100755
index 0000000..0134846
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Build drm-test and place the binary in the target directory, where a plain
+# `cargo build` would have left it, e.g. target/debug/drm-test.
+#
+# The tests are run by libtest, so the executable is built from the --tests
+# target rather than from a plain `cargo build`. Cargo keeps test binaries in
+# an internal location that differs between releases, so ask it where the
+# binary was written and copy it next to the profile directory.
+#
+# Arguments are passed through to cargo, for example:
+#
+#     ./build.sh --release
+#     ./build.sh --target x86_64-unknown-linux-musl
+#
+set -eu
+
+cd "$(dirname "$0")"
+
+# Only the test target reports a non-null "executable". Diagnostics are
+# rendered to stderr as usual, leaving just artifact messages on stdout.
+bin=$(cargo build --tests --message-format=json-render-diagnostics "$@" \
+	| grep -o '"executable":"[^"]*"' \
+	| head -n1 \
+	| sed 's/^"executable":"//; s/"$//')
+
+if [ -z "${bin}" ]; then
+	echo "error: cargo produced no test binary" >&2
+	exit 1
+fi
+
+# Which directory cargo stages the binary in depends on the release: stable
+# uses <profile>/deps, while nightly uses <profile>/build/<pkg>/<hash>/out.
+# Climb out of whichever it used to reach the profile directory.
+dir="${bin%/*}"
+while :; do
+	case "${dir}" in
+	*/deps | */build) break ;;
+	*/*) dir="${dir%/*}" ;;
+	*)
+		echo "error: cannot locate the cargo profile directory" >&2
+		exit 1
+		;;
+	esac
+done
+
+dest="${dir%/*}/drm-test"
+cp -f "${bin}" "${dest}"
+echo "built: ${dest}"
diff --git a/src/drm/mod.rs b/src/drm/mod.rs
index c767dbe..5d6f280 100644
--- a/src/drm/mod.rs
+++ b/src/drm/mod.rs
@@ -85,8 +85,3 @@ impl Drm {
         Ok(())
     }
 }
-
-/// Represents the test functions all drivers must provide.
-pub trait Test {
-    fn all(&self);
-}
diff --git a/src/lib.rs b/src/lib.rs
index 663f484..5f73c55 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,6 +2,7 @@ mod bindings;
 pub mod drm;
 pub mod nova;
 
+#[cfg(test)]
 macro_rules! assert_ge {
     ($left:expr, $right:expr $(,)?) => {{
         match (&$left, &$right) {
@@ -16,4 +17,5 @@ macro_rules! assert_ge {
         }
     }};
 }
+#[cfg(test)]
 pub(crate) use assert_ge;
diff --git a/src/main.rs b/src/main.rs
deleted file mode 100644
index 2034285..0000000
--- a/src/main.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-use drm_test::{drm, nova};
-use std::{env, process::exit};
-
-fn main() {
-    let args: Vec<String> = env::args().collect();
-
-    if args.len() < 2 {
-        println!("Usage: {} {{ nova }}", args[0]);
-        exit(1);
-    }
-
-    let test: Box<dyn drm::Test> = match args[1].as_str() {
-        "nova" => Box::new(nova::Test::open()),
-        _ => todo!(),
-    };
-
-    test.all();
-}
diff --git a/src/nova/mod.rs b/src/nova/mod.rs
index 7b53758..1d03335 100644
--- a/src/nova/mod.rs
+++ b/src/nova/mod.rs
@@ -1,8 +1,8 @@
 pub mod gpu;
 pub mod nova;
-pub mod test;
+#[cfg(test)]
+mod test;
 
 pub use self::nova::Nova;
-pub use self::test::Test;
 
 pub use self::gpu::{Architecture, Chipset};
diff --git a/src/nova/test.rs b/src/nova/test.rs
index 75c92d8..a02e449 100644
--- a/src/nova/test.rs
+++ b/src/nova/test.rs
@@ -1,106 +1,85 @@
 use std::{ffi::CStr, io, mem::size_of};
 
 use crate::{
-    assert_ge, bindings, drm,
+    assert_ge, bindings,
     nova::{Architecture, Chipset, Nova},
 };
 
-pub struct Test(Nova);
+/// Test whether GPU information is reported correctly.
+#[test]
+fn test_gpu_info() {
+    let nova = Nova::open().unwrap();
+    let info = nova.gpu_info().unwrap();
+    let gpu_name =
+        CStr::from_bytes_until_nul(&info.gpu_name).expect("GPU name must be NUL-terminated");
+    let gpu_short_name = CStr::from_bytes_until_nul(&info.gpu_short_name)
+        .expect("GPU short name must be NUL-terminated");
+    let architecture = Architecture::from_repr(info.architecture)
+        .map(|arch| arch.to_string())
+        .unwrap_or_else(|| "unknown".into());
+    let chipset = Chipset::from_repr(info.chipid)
+        .map(|chip| chip.to_string())
+        .unwrap_or_else(|| "unknown".into());
 
-impl Test {
-    /// Create a new `Test` instance.
-    pub fn open() -> Self {
-        let nova = Nova::open().unwrap();
-
-        Self(nova)
-    }
-
-    /// Test whether GPU information is reported correctly.
-    pub fn test_gpu_info(&self) {
-        let info = self.0.gpu_info().unwrap();
-        let gpu_name =
-            CStr::from_bytes_until_nul(&info.gpu_name).expect("GPU name must be NUL-terminated");
-        let gpu_short_name = CStr::from_bytes_until_nul(&info.gpu_short_name)
-            .expect("GPU short name must be NUL-terminated");
-        let architecture = Architecture::from_repr(info.architecture)
-            .map(|arch| arch.to_string())
-            .unwrap_or_else(|| "unknown".into());
-        let chipset = Chipset::from_repr(info.chipid)
-            .map(|chip| chip.to_string())
-            .unwrap_or_else(|| "unknown".into());
-
-        assert_ne!(info.architecture, 0);
-        println!("NOVA GPU information:");
-        println!(
-            "  Architecture: {} ({:#x})",
-            architecture, info.architecture
-        );
-        println!("  Chipset: {} ({:#x})", chipset, info.chipid);
-        println!(
-            "  VRAM size: {} bytes ({:.2} GiB)",
-            info.vram_size,
-            info.vram_size as f64 / (1u64 << 30) as f64
-        );
-        println!(
-            "  VRAM BAR size: {} bytes ({:.2} GiB)",
-            info.vram_bar_size,
-            info.vram_bar_size as f64 / (1u64 << 30) as f64
-        );
-        println!("  GPU name: {}", gpu_name.to_string_lossy());
-        println!("  GPU short name: {}", gpu_short_name.to_string_lossy());
-        println!(
-            "  GPU GID: {}",
-            [
-                &info.gpu_gid[..4],
-                &info.gpu_gid[4..6],
-                &info.gpu_gid[6..8],
-                &info.gpu_gid[8..10],
-                &info.gpu_gid[10..],
-            ]
-            .into_iter()
-            .map(|chunk| chunk
-                .iter()
-                .map(|byte| format!("{byte:02x}"))
-                .collect::<String>())
-            .collect::<Vec<_>>()
-            .join("-")
-        );
-    }
-
-    /// Test whether a NULL data pointer reports the required GPU information size.
-    pub fn test_gpu_info_size(&self) {
-        let size = self.0.gpu_info_size().unwrap();
-
-        assert_ge!(size as usize, size_of::<bindings::drm_nova_info_gpu>());
-    }
+    assert_ne!(info.architecture, 0);
+    println!("NOVA GPU information:");
+    println!(
+        "  Architecture: {} ({:#x})",
+        architecture, info.architecture
+    );
+    println!("  Chipset: {} ({:#x})", chipset, info.chipid);
+    println!(
+        "  VRAM size: {} bytes ({:.2} GiB)",
+        info.vram_size,
+        info.vram_size as f64 / (1u64 << 30) as f64
+    );
+    println!(
+        "  VRAM BAR size: {} bytes ({:.2} GiB)",
+        info.vram_bar_size,
+        info.vram_bar_size as f64 / (1u64 << 30) as f64
+    );
+    println!("  GPU name: {}", gpu_name.to_string_lossy());
+    println!("  GPU short name: {}", gpu_short_name.to_string_lossy());
+    println!(
+        "  GPU GID: {}",
+        [
+            &info.gpu_gid[..4],
+            &info.gpu_gid[4..6],
+            &info.gpu_gid[6..8],
+            &info.gpu_gid[8..10],
+            &info.gpu_gid[10..],
+        ]
+        .into_iter()
+        .map(|chunk| chunk
+            .iter()
+            .map(|byte| format!("{byte:02x}"))
+            .collect::<String>())
+        .collect::<Vec<_>>()
+        .join("-")
+    );
+}
 
-    fn gem_create(&self, size: u64) -> u32 {
-        self.0.gem_create(size).unwrap()
-    }
+/// Test whether a NULL data pointer reports the required GPU information size.
+#[test]
+fn test_gpu_info_size() {
+    let nova = Nova::open().unwrap();
+    let size = nova.gpu_info_size().unwrap();
 
-    fn gem_size(&self, handle: u32) -> u64 {
-        self.0.gem_size(handle).unwrap()
-    }
+    assert_ge!(size as usize, size_of::<bindings::drm_nova_info_gpu>());
+}
 
-    /// Test whether GEM objects are created with the expected size.
-    pub fn test_gem_size(&self) {
-        let ret = self.0.gem_create(0x0);
-        let err = ret.expect_err("GEM object must not be zero-sized!");
-        assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
+/// Test whether GEM objects are created with the expected size.
+#[test]
+fn test_gem_size() {
+    let nova = Nova::open().unwrap();
 
-        let handle = self.gem_create(0x1000 + 0x1);
-        assert_eq!(self.gem_size(handle), 0x2000);
+    let ret = nova.gem_create(0x0);
+    let err = ret.expect_err("GEM object must not be zero-sized!");
+    assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
 
-        let handle = self.gem_create(0x2000);
-        assert_eq!(self.gem_size(handle), 0x2000);
-    }
-}
+    let handle = nova.gem_create(0x1000 + 0x1).unwrap();
+    assert_eq!(nova.gem_size(handle).unwrap(), 0x2000);
 
-impl drm::Test for Test {
-    // Run all tests.
-    fn all(&self) {
-        self.test_gpu_info();
-        self.test_gpu_info_size();
-        self.test_gem_size();
-    }
+    let handle = nova.gem_create(0x2000).unwrap();
+    assert_eq!(nova.gem_size(handle).unwrap(), 0x2000);
 }
-- 
2.54.0 (Apple Git-157)