Use an incremental cache when testing WASI (#8354)

Currently we've got a good number of WASI tests and they're all
relatively large. We also can run a single test in up to three
configurations:

* As-is with a module
* As a component in "sync" mode
* As a component in "async" mode

In debug mode compilation of all these modules can take a significant
chunk of time (20-30s in total for test suites) This commit updates
these test suites to use an in-memory per-process incremental cache
backed by a simple `Mutex<HashMap>`. This gives some good speedups in
debug mode, locally the wasi-common, wasmtime-wasi, and
wasmtime-wasi-http test suites were reduced from 32 to 17 seconds. I'd
expect larger speedups on less-parallel machines such as our CI.

Cargo.lock

@@ -2755,6 +2755,7 @@ version = "0.0.0"
 dependencies = [
  "cargo_metadata",
  "heck",
+ "wasmtime",
  "wit-component 0.203.0",
 ]
 

crates/test-programs/artifacts/Cargo.toml

@@ -9,6 +9,9 @@ license = "Apache-2.0 WITH LLVM-exception"
 [lints]
 workspace = true
 
+[dependencies]
+wasmtime = { workspace = true, features = ['incremental-cache', 'cranelift'] }
+
 [build-dependencies]
 heck = { workspace = true }
 wit-component = { workspace = true }

crates/test-programs/artifacts/src/lib.rs

@@ -1,6 +1,10 @@
 include!(concat!(env!("OUT_DIR"), "/gen.rs"));
 
+use std::borrow::Cow;
+use std::collections::HashMap;
 use std::io::IsTerminal;
+use std::sync::{Arc, Mutex};
+use wasmtime::{CacheStore, Config, Engine};
 
 /// The wasi-tests binaries use these environment variables to determine their
 /// expected behavior.
@@ -32,3 +36,44 @@ pub fn stdio_is_terminal() -> bool {
         && std::io::stdout().is_terminal()
         && std::io::stderr().is_terminal()
 }
+
+// Simple incremental cache used during tests to help improve test runtime.
+//
+// Many tests take a similar module (e.g. a component, a preview1 thing, sync,
+// async, etc) and run it in different contexts and this improve cache hit rates
+// across usages by sharing one incremental cache across tests.
+fn cache_store() -> Arc<dyn CacheStore> {
+    #[derive(Debug)]
+    struct MyCache;
+
+    static CACHE: Mutex<Option<HashMap<Vec<u8>, Vec<u8>>>> = Mutex::new(None);
+
+    impl CacheStore for MyCache {
+        fn get(&self, key: &[u8]) -> Option<Cow<[u8]>> {
+            let mut cache = CACHE.lock().unwrap();
+            let cache = cache.get_or_insert_with(HashMap::new);
+            cache.get(key).map(|s| s.to_vec().into())
+        }
+
+        fn insert(&self, key: &[u8], value: Vec<u8>) -> bool {
+            let mut cache = CACHE.lock().unwrap();
+            let cache = cache.get_or_insert_with(HashMap::new);
+            cache.insert(key.to_vec(), value);
+            true
+        }
+    }
+
+    Arc::new(MyCache)
+}
+
+/// Helper to create an `Engine` with a pre-configured `Config` that uses a
+/// cache for faster building of modules.
+pub fn engine(configure: impl FnOnce(&mut Config)) -> Engine {
+    let mut config = Config::new();
+    config.wasm_component_model(true);
+    config
+        .enable_incremental_compilation(cache_store())
+        .unwrap();
+    configure(&mut config);
+    Engine::new(&config).unwrap()
+}

crates/wasi-common/tests/all/async_.rs

@@ -17,9 +17,9 @@ async fn run(path: &str, inherit_stdio: bool) -> Result<()> {
     let stdout = WritePipe::new_in_memory();
     let stderr = WritePipe::new_in_memory();
     let r = {
-        let mut config = Config::new();
-        config.async_support(true);
-        let engine = Engine::new(&config)?;
+        let engine = test_programs_artifacts::engine(|config| {
+            config.async_support(true);
+        });
         let mut linker = Linker::new(&engine);
         add_to_linker(&mut linker, |cx| cx)?;
 

crates/wasi-common/tests/all/main.rs

@@ -2,7 +2,7 @@ use anyhow::Result;
 use std::path::Path;
 use tempfile::TempDir;
 use wasi_common::pipe::WritePipe;
-use wasmtime::{Config, Engine, Linker, Module, Store};
+use wasmtime::{Linker, Module, Store};
 
 pub fn prepare_workspace(exe_name: &str) -> Result<TempDir> {
     let prefix = format!("wasi_common_{}_", exe_name);

crates/wasi-common/tests/all/sync.rs

@@ -11,7 +11,7 @@ fn run(path: &str, inherit_stdio: bool) -> Result<()> {
     let stdout = WritePipe::new_in_memory();
     let stderr = WritePipe::new_in_memory();
     let r = {
-        let engine = Engine::default();
+        let engine = test_programs_artifacts::engine(|_| {});
         let mut linker = Linker::new(&engine);
         add_to_linker(&mut linker, |cx| cx)?;
 

crates/wasi-http/tests/all/async_.rs

@@ -5,11 +5,10 @@ use wasmtime_wasi::bindings::Command;
 foreach_http!(assert_test_exists);
 
 async fn run(path: &str, server: &Server) -> Result<()> {
-    let mut config = Config::new();
-    config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
-    config.wasm_component_model(true);
-    config.async_support(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|config| {
+        config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
+        config.async_support(true);
+    });
     let component = Component::from_file(&engine, path)?;
     let mut store = store(&engine, server);
     let mut linker = Linker::new(&engine);

crates/wasi-http/tests/all/sync.rs

@@ -5,10 +5,9 @@ use wasmtime_wasi::bindings::sync::Command;
 foreach_http!(assert_test_exists);
 
 fn run(path: &str, server: &Server) -> Result<()> {
-    let mut config = Config::new();
-    config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
-    config.wasm_component_model(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|config| {
+        config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
+    });
     let component = Component::from_file(&engine, path)?;
     let mut store = store(&engine, server);
     let mut linker = Linker::new(&engine);

crates/wasi/Cargo.toml

@@ -42,7 +42,7 @@ test-log = { workspace = true }
 tracing-subscriber = { workspace = true }
 test-programs-artifacts = { workspace = true }
 tempfile = { workspace = true }
-wasmtime = { workspace = true, features = ['cranelift'] }
+wasmtime = { workspace = true, features = ['cranelift', 'incremental-cache'] }
 
 [target.'cfg(unix)'.dependencies]
 rustix = { workspace = true, features = ["event", "fs", "net"] }

crates/wasi/tests/all/api.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 use std::sync::Mutex;
 use std::time::Duration;
 use wasmtime::component::{Component, Linker, ResourceTable};
-use wasmtime::{Config, Engine, Store};
+use wasmtime::Store;
 use wasmtime_wasi::bindings::Command;
 use wasmtime_wasi::{
     add_to_linker_async,
@@ -30,9 +30,9 @@ use test_programs_artifacts::*;
 foreach_api!(assert_test_exists);
 
 async fn instantiate(path: &str, ctx: CommandCtx) -> Result<(Store<CommandCtx>, Command)> {
-    let mut config = Config::new();
-    config.async_support(true).wasm_component_model(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|config| {
+        config.async_support(true);
+    });
     let mut linker = Linker::new(&engine);
     add_to_linker_async(&mut linker)?;
 
@@ -133,10 +133,9 @@ wasmtime::component::bindgen!({
 async fn api_reactor() -> Result<()> {
     let table = ResourceTable::new();
     let wasi = WasiCtxBuilder::new().env("GOOD_DOG", "gussie").build();
-
-    let mut config = Config::new();
-    config.async_support(true).wasm_component_model(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|config| {
+        config.async_support(true);
+    });
     let mut linker = Linker::new(&engine);
     add_to_linker_async(&mut linker)?;
 

crates/wasi/tests/all/async_.rs

@@ -7,9 +7,9 @@ use wasmtime_wasi::bindings::Command;
 async fn run(path: &str, inherit_stdio: bool) -> Result<()> {
     let path = Path::new(path);
     let name = path.file_stem().unwrap().to_str().unwrap();
-    let mut config = Config::new();
-    config.async_support(true).wasm_component_model(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|config| {
+        config.async_support(true);
+    });
     let mut linker = Linker::new(&engine);
     add_to_linker_async(&mut linker)?;
 

crates/wasi/tests/all/main.rs

@@ -2,7 +2,7 @@ use anyhow::Result;
 use tempfile::TempDir;
 use wasmtime::{
     component::{Component, Linker, ResourceTable},
-    Config, Engine, Store,
+    Engine, Store,
 };
 use wasmtime_wasi::preview1::WasiP1Ctx;
 use wasmtime_wasi::{

crates/wasi/tests/all/preview1.rs

@@ -7,9 +7,9 @@ use wasmtime_wasi::preview1::add_to_linker_async;
 async fn run(path: &str, inherit_stdio: bool) -> Result<()> {
     let path = Path::new(path);
     let name = path.file_stem().unwrap().to_str().unwrap();
-    let mut config = Config::new();
-    config.async_support(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|config| {
+        config.async_support(true);
+    });
     let mut linker = Linker::<Ctx>::new(&engine);
     add_to_linker_async(&mut linker, |t| &mut t.wasi)?;
 

crates/wasi/tests/all/sync.rs

@@ -7,9 +7,7 @@ use wasmtime_wasi::bindings::sync::Command;
 fn run(path: &str, inherit_stdio: bool) -> Result<()> {
     let path = Path::new(path);
     let name = path.file_stem().unwrap().to_str().unwrap();
-    let mut config = Config::new();
-    config.wasm_component_model(true);
-    let engine = Engine::new(&config)?;
+    let engine = test_programs_artifacts::engine(|_| {});
     let mut linker = Linker::new(&engine);
     add_to_linker_sync(&mut linker)?;