Use `libtest-mimic` to drive some `harness = false` tests (#8236)

This provides a bit of a nicer experience than the default "build your
own test harness" experience by providing things like filters and
parallel execution by default. This helps speed up the `disas` test
suite, for example, which previously had no parallelism.

Cargo.lock

@@ -1098,6 +1098,15 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "escape8259"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba4f4911e3666fcd7826997b4745c8224295a6f3072f1418c3067b97a67557ee"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "example-component-wasm"
 version = "0.0.0"
@@ -1736,6 +1745,18 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
 
+[[package]]
+name = "libtest-mimic"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f0f4c6f44ecfd52e8b443f2ad18f2b996540135771561283c2352ce56a1c70b"
+dependencies = [
+ "clap",
+ "escape8259",
+ "termcolor",
+ "threadpool",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.12"
@@ -2774,6 +2795,15 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "threadpool"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
+dependencies = [
+ "num_cpus",
+]
+
 [[package]]
 name = "tinytemplate"
 version = "1.2.1"
@@ -3486,6 +3516,7 @@ dependencies = [
  "humantime",
  "hyper",
  "libc",
+ "libtest-mimic",
  "listenfd",
  "log",
  "memchr",

Cargo.toml

@@ -96,6 +96,7 @@ cranelift-codegen = { workspace = true }
 cranelift-reader = { workspace = true }
 toml = { workspace = true }
 similar = { workspace = true }
+libtest-mimic = "0.7.0"
 
 [target.'cfg(windows)'.dev-dependencies]
 windows-sys = { workspace = true, features = ["Win32_System_Memory"] }

supply-chain/audits.toml

@@ -1215,6 +1215,12 @@ criteria = "safe-to-deploy"
 version = "0.1.2"
 notes = "This should be portable to any POSIX system and seems like it should be part of the libc crate, but at any rate it's safe as is."
 
+[[audits.escape8259]]
+who = "Alex Crichton <alex@alexcrichton.com>"
+criteria = "safe-to-run"
+version = "0.5.2"
+notes = "No unsafe code, crate does what it says on the tin, small crate."
+
 [[audits.fallible-iterator]]
 who = "Alex Crichton <alex@alexcrichton.com>"
 criteria = "safe-to-deploy"
@@ -1713,6 +1719,12 @@ This is a minor update which has some testing affordances as well as some
 updated math algorithms.
 """
 
+[[audits.libtest-mimic]]
+who = "Alex Crichton <alex@alexcrichton.com>"
+criteria = "safe-to-run"
+delta = "0.6.1 -> 0.7.0"
+notes = "Only minor changes with new flags and such, no major updates to `unsafe` or anything outside of a test framework."
+
 [[audits.linux-raw-sys]]
 who = "Dan Gohman <dev@sunfishcode.online>"
 criteria = "safe-to-deploy"

supply-chain/config.toml

@@ -109,9 +109,6 @@ audit-as-crates-io = true
 [policy.wasmtime-cranelift]
 audit-as-crates-io = true
 
-[policy.wasmtime-cranelift-shared]
-audit-as-crates-io = true
-
 [policy.wasmtime-environ]
 audit-as-crates-io = true
 

supply-chain/imports.lock

@@ -954,12 +954,6 @@ when = "2024-02-28"
 user-id = 73222
 user-login = "wasmtime-publish"
 
-[[publisher.wasmtime-cranelift-shared]]
-version = "18.0.2"
-when = "2024-02-28"
-user-id = 73222
-user-login = "wasmtime-publish"
-
 [[publisher.wasmtime-environ]]
 version = "18.0.2"
 when = "2024-02-28"
@@ -1422,6 +1416,19 @@ criteria = "safe-to-run"
 version = "0.4.4"
 aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/main/cargo-vet/audits.toml?format=TEXT"
 
+[[audits.google.audits.libtest-mimic]]
+who = "Dennis Kempin <denniskempin@google.com>"
+criteria = "safe-to-run"
+version = "0.6.0"
+notes = "Used in tests only"
+aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/main/cargo-vet/audits.toml?format=TEXT"
+
+[[audits.google.audits.libtest-mimic]]
+who = "George Burgess IV <gbiv@google.com>"
+criteria = "safe-to-run"
+delta = "0.6.0 -> 0.6.1"
+aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/main/cargo-vet/audits.toml?format=TEXT"
+
 [[audits.google.audits.pin-project-lite]]
 who = "David Koloski <dkoloski@google.com>"
 criteria = "safe-to-deploy"
@@ -1429,6 +1436,12 @@ version = "0.2.9"
 notes = "Reviewed on https://fxrev.dev/824504"
 aggregated-from = "https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/third_party/rust_crates/supply-chain/audits.toml?format=TEXT"
 
+[[audits.google.audits.threadpool]]
+who = "Dennis Kempin <denniskempin@google.com>"
+criteria = "safe-to-run"
+version = "1.8.1"
+aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/main/cargo-vet/audits.toml?format=TEXT"
+
 [[audits.google.audits.version_check]]
 who = "George Burgess IV <gbiv@google.com>"
 criteria = "safe-to-deploy"

tests/disas.rs

@@ -44,6 +44,7 @@ use clap::Parser;
 use cranelift_codegen::ir::{Function, UserExternalName, UserFuncName};
 use cranelift_codegen::isa::{lookup_by_name, TargetIsa};
 use cranelift_codegen::settings::{Configurable, Flags, SetError};
+use libtest_mimic::{Arguments, Trial};
 use serde::de::DeserializeOwned;
 use serde_derive::Deserialize;
 use similar::TextDiff;
@@ -59,16 +60,24 @@ fn main() -> Result<()> {
         return Ok(());
     }
 
-    // First discover all tests ...
     let mut tests = Vec::new();
     find_tests("./tests/disas".as_ref(), &mut tests)?;
 
-    // ... then run all tests!
-    for test in tests.iter() {
-        run_test(&test).with_context(|| format!("failed to run tests {test:?}"))?;
+    let mut trials = Vec::new();
+    for test in tests {
+        trials.push(Trial::test(test.to_str().unwrap().to_string(), move || {
+            run_test(&test)
+                .with_context(|| format!("failed to run tests {test:?}"))
+                .map_err(|e| format!("{e:?}").into())
+        }))
     }
 
-    Ok(())
+    // These tests have some long names so use the "quiet" output by default.
+    let mut arguments = Arguments::parse();
+    if arguments.format.is_none() {
+        arguments.quiet = true;
+    }
+    libtest_mimic::run(&arguments, trials).exit()
 }
 
 fn find_tests(path: &Path, dst: &mut Vec<PathBuf>) -> Result<()> {

tests/host_segfault.rs

@@ -9,6 +9,7 @@
 // then we also make sure that stderr is empty to confirm that no weird panics
 // happened or anything like that.
 
+use libtest_mimic::{Arguments, Trial};
 use std::env;
 use std::future::Future;
 use std::io::{self, Write};
@@ -190,10 +191,14 @@ fn main() {
             test();
         }
         Err(_) => {
+            let mut trials = Vec::new();
             for (name, _test, stack_overflow) in tests {
-                println!("running {name}");
-                run_test(name, *stack_overflow);
+                trials.push(Trial::test(name.to_string(), || {
+                    run_test(name, *stack_overflow);
+                    Ok(())
+                }));
             }
+            libtest_mimic::run(&Arguments::from_args(), trials).exit()
         }
     }
 }