Run all `*.wast` tests in fuzzing (#8121)

* Run all `*.wast` tests in fuzzing

Currently we have a `spectest` fuzzer which uses fuzz input to generate
an arbitrary configuration for Wasmtime and then executes the spec test.
This ensures that no matter the configuration Wasmtime can pass spec
tests. This commit expands this testing to include all `*.wast` tests we
have in this repository. While we don't have a ton we still have some
significant ones like in #8118 which will only reproduce when turning
knobs on CPU features.

* Fix CLI build

* Fix wast testing

crates/fuzzing/build.rs

@@ -1,4 +1,4 @@
-// A small build script to include the contents of the spec test suite into the
+// A small build script to include the contents of the wast test suite into the
 // final fuzzing binary so the fuzzing binary can be run elsewhere and doesn't
 // rely on the original source tree.
 
@@ -9,22 +9,34 @@ fn main() {
     println!("cargo:rerun-if-changed=build.rs");
 
     let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
-    let dir = env::current_dir()
-        .unwrap()
-        .join("../../tests/spec_testsuite");
+
+    let dirs = [
+        "tests/spec_testsuite",
+        "tests/misc_testsuite",
+        "tests/misc_testsuite/multi-memory",
+        "tests/misc_testsuite/simd",
+        "tests/misc_testsuite/threads",
+    ];
+    let mut root = env::current_dir().unwrap();
+    root.pop(); // chop off 'fuzzing'
+    root.pop(); // chop off 'crates'
     let mut code = format!("static FILES: &[(&str, &str)] = &[\n");
-    let mut entries = dir
-        .read_dir()
-        .unwrap()
-        .map(|p| p.unwrap().path().display().to_string())
-        .collect::<Vec<_>>();
+
+    let mut entries = Vec::new();
+    for dir in dirs {
+        for entry in root.join(dir).read_dir().unwrap() {
+            let entry = entry.unwrap();
+            let path = entry.path();
+            if path.extension().and_then(|s| s.to_str()) == Some("wast") {
+                entries.push(path);
+            }
+        }
+    }
     entries.sort();
     for path in entries {
-        if !path.ends_with(".wast") {
-            continue;
-        }
+        let path = path.to_str().expect("path is not valid utf-8");
         code.push_str(&format!("({:?}, include_str!({0:?})),\n", path));
     }
     code.push_str("];\n");
-    std::fs::write(out_dir.join("spectests.rs"), code).unwrap();
+    std::fs::write(out_dir.join("wasttests.rs"), code).unwrap();
 }

crates/fuzzing/src/generators.rs

@@ -17,10 +17,10 @@ mod memory;
 mod module;
 mod pooling_config;
 mod single_inst_module;
-mod spec_test;
 mod stacks;
 pub mod table_ops;
 mod value;
+mod wast_test;
 
 pub use codegen_settings::CodegenSettings;
 pub use config::CompilerStrategy;
@@ -30,6 +30,6 @@ pub use memory::{MemoryConfig, NormalMemoryConfig, UnalignedMemory, UnalignedMem
 pub use module::ModuleConfig;
 pub use pooling_config::PoolingAllocationConfig;
 pub use single_inst_module::SingleInstModule;
-pub use spec_test::SpecTest;
 pub use stacks::Stacks;
 pub use value::{DiffValue, DiffValueType};
+pub use wast_test::WastTest;

crates/fuzzing/src/generators/config.rs

@@ -100,12 +100,12 @@ impl Config {
         self.module_config.generate(input, default_fuel)
     }
 
-    /// Tests whether this configuration is capable of running all spec tests.
-    pub fn is_spectest_compliant(&self) -> bool {
+    /// Tests whether this configuration is capable of running all wast tests.
+    pub fn is_wast_test_compliant(&self) -> bool {
         let config = &self.module_config.config;
 
         // Check for wasm features that must be disabled to run spec tests
-        if config.memory64_enabled || config.threads_enabled {
+        if config.memory64_enabled {
             return false;
         }
 
@@ -114,6 +114,8 @@ impl Config {
             || !config.reference_types_enabled
             || !config.multi_value_enabled
             || !config.simd_enabled
+            || !config.threads_enabled
+            || config.max_memories <= 1
         {
             return false;
         }
@@ -412,7 +414,7 @@ pub struct WasmtimeConfig {
     canonicalize_nans: bool,
     interruptable: bool,
     pub(crate) consume_fuel: bool,
-    epoch_interruption: bool,
+    pub(crate) epoch_interruption: bool,
     /// The Wasmtime memory configuration to use.
     pub memory_config: MemoryConfig,
     force_jump_veneers: bool,

crates/fuzzing/src/generators/spec_test.rs → crates/fuzzing/src/generators/wast_test.rs

@@ -3,24 +3,23 @@
 use arbitrary::{Arbitrary, Unstructured};
 
 // See `build.rs` for how the `FILES` array is generated.
-include!(concat!(env!("OUT_DIR"), "/spectests.rs"));
+include!(concat!(env!("OUT_DIR"), "/wasttests.rs"));
 
-/// A spec test from the upstream wast testsuite, arbitrarily chosen from the
-/// list of known spec tests.
+/// A wast test from this repository.
 #[derive(Debug)]
-pub struct SpecTest {
+pub struct WastTest {
     /// The filename of the spec test
     pub file: &'static str,
     /// The `*.wast` contents of the spec test
     pub contents: &'static str,
 }
 
-impl<'a> Arbitrary<'a> for SpecTest {
+impl<'a> Arbitrary<'a> for WastTest {
     fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
         // NB: this does get a uniform value in the provided range.
         let i = u.int_in_range(0..=FILES.len() - 1)?;
         let (file, contents) = FILES[i];
-        Ok(SpecTest { file, contents })
+        Ok(WastTest { file, contents })
     }
 
     fn size_hint(_depth: usize) -> (usize, Option<usize>) {

crates/fuzzing/src/oracles.rs

@@ -574,17 +574,31 @@ pub fn make_api_calls(api: generators::api::ApiCalls) {
     }
 }
 
-/// Executes the wast `test` spectest with the `config` specified.
+/// Executes the wast `test` with the `config` specified.
 ///
-/// Ensures that spec tests pass regardless of the `Config`.
-pub fn spectest(fuzz_config: generators::Config, test: generators::SpecTest) {
+/// Ensures that wast tests pass regardless of the `Config`.
+pub fn wast_test(fuzz_config: generators::Config, test: generators::WastTest) {
     crate::init_fuzzing();
-    if !fuzz_config.is_spectest_compliant() {
+    if !fuzz_config.is_wast_test_compliant() {
         return;
     }
+
+    // Fuel and epochs don't play well with threads right now, so exclude any
+    // thread-spawning test if it looks like threads are spawned in that case.
+    if fuzz_config.wasmtime.consume_fuel || fuzz_config.wasmtime.epoch_interruption {
+        if test.contents.contains("(thread") {
+            return;
+        }
+    }
+
     log::debug!("running {:?}", test.file);
     let mut wast_context = WastContext::new(fuzz_config.to_store());
-    wast_context.register_spectest(false).unwrap();
+    wast_context
+        .register_spectest(&wasmtime_wast::SpectestConfig {
+            use_shared_memory: false,
+            suppress_prints: true,
+        })
+        .unwrap();
     wast_context
         .run_buffer(test.file, test.contents.as_bytes())
         .unwrap();

crates/wast/src/lib.rs

@@ -8,7 +8,7 @@ mod core;
 mod spectest;
 mod wast;
 
-pub use crate::spectest::link_spectest;
+pub use crate::spectest::{link_spectest, SpectestConfig};
 pub use crate::wast::WastContext;
 
 /// Version number of this crate.

crates/wast/src/spectest.rs

@@ -1,24 +1,53 @@
 use wasmtime::*;
 
+/// Configuration of how spectest primitives work.
+pub struct SpectestConfig {
+    /// Whether or not to have a `shared_memory` definition.
+    pub use_shared_memory: bool,
+    /// Whether or not spectest functions that print things actually print things.
+    pub suppress_prints: bool,
+}
+
 /// Return an instance implementing the "spectest" interface used in the
 /// spec testsuite.
 pub fn link_spectest<T>(
     linker: &mut Linker<T>,
     store: &mut Store<T>,
-    use_shared_memory: bool,
+    config: &SpectestConfig,
 ) -> Result<()> {
+    let suppress = config.suppress_prints;
     linker.func_wrap("spectest", "print", || {})?;
-    linker.func_wrap("spectest", "print_i32", |val: i32| println!("{}: i32", val))?;
-    linker.func_wrap("spectest", "print_i64", |val: i64| println!("{}: i64", val))?;
-    linker.func_wrap("spectest", "print_f32", |val: f32| println!("{}: f32", val))?;
-    linker.func_wrap("spectest", "print_f64", |val: f64| println!("{}: f64", val))?;
-    linker.func_wrap("spectest", "print_i32_f32", |i: i32, f: f32| {
-        println!("{}: i32", i);
-        println!("{}: f32", f);
+    linker.func_wrap("spectest", "print_i32", move |val: i32| {
+        if !suppress {
+            println!("{}: i32", val)
+        }
+    })?;
+    linker.func_wrap("spectest", "print_i64", move |val: i64| {
+        if !suppress {
+            println!("{}: i64", val)
+        }
+    })?;
+    linker.func_wrap("spectest", "print_f32", move |val: f32| {
+        if !suppress {
+            println!("{}: f32", val)
+        }
+    })?;
+    linker.func_wrap("spectest", "print_f64", move |val: f64| {
+        if !suppress {
+            println!("{}: f64", val)
+        }
     })?;
-    linker.func_wrap("spectest", "print_f64_f64", |f1: f64, f2: f64| {
-        println!("{}: f64", f1);
-        println!("{}: f64", f2);
+    linker.func_wrap("spectest", "print_i32_f32", move |i: i32, f: f32| {
+        if !suppress {
+            println!("{}: i32", i);
+            println!("{}: f32", f);
+        }
+    })?;
+    linker.func_wrap("spectest", "print_f64_f64", move |f1: f64, f2: f64| {
+        if !suppress {
+            println!("{}: f64", f1);
+            println!("{}: f64", f2);
+        }
     })?;
 
     let ty = GlobalType::new(ValType::I32, Mutability::Const);
@@ -45,7 +74,7 @@ pub fn link_spectest<T>(
     let memory = Memory::new(&mut *store, ty)?;
     linker.define(&mut *store, "spectest", "memory", memory)?;
 
-    if use_shared_memory {
+    if config.use_shared_memory {
         let ty = MemoryType::shared(1, 1);
         let memory = Memory::new(&mut *store, ty)?;
         linker.define(&mut *store, "spectest", "shared_memory", memory)?;

crates/wast/src/wast.rs

@@ -137,8 +137,8 @@ where
     }
 
     /// Register "spectest" which is used by the spec testsuite.
-    pub fn register_spectest(&mut self, use_shared_memory: bool) -> Result<()> {
-        link_spectest(&mut self.core_linker, &mut self.store, use_shared_memory)?;
+    pub fn register_spectest(&mut self, config: &SpectestConfig) -> Result<()> {
+        link_spectest(&mut self.core_linker, &mut self.store, config)?;
         #[cfg(feature = "component-model")]
         link_component_spectest(&mut self.component_linker)?;
         Ok(())

fuzz/Cargo.toml

@@ -68,8 +68,8 @@ test = false
 doc = false
 
 [[bin]]
-name = "spectests"
-path = "fuzz_targets/spectests.rs"
+name = "wast_tests"
+path = "fuzz_targets/wast_tests.rs"
 test = false
 doc = false
 

fuzz/fuzz_targets/spectests.rs

@@ -1,9 +0,0 @@
-#![no_main]
-
-use libfuzzer_sys::fuzz_target;
-use wasmtime_fuzzing::generators::{Config, SpecTest};
-
-fuzz_target!(|pair: (Config, SpecTest)| {
-    let (config, test) = pair;
-    wasmtime_fuzzing::oracles::spectest(config, test);
-});

fuzz/fuzz_targets/wast_tests.rs

@@ -0,0 +1,9 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use wasmtime_fuzzing::generators::{Config, WastTest};
+
+fuzz_target!(|pair: (Config, WastTest)| {
+    let (config, test) = pair;
+    wasmtime_fuzzing::oracles::wast_test(config, test);
+});

src/commands/wast.rs

@@ -5,7 +5,7 @@ use clap::Parser;
 use std::path::PathBuf;
 use wasmtime::{Engine, Store};
 use wasmtime_cli_flags::CommonOptions;
-use wasmtime_wast::WastContext;
+use wasmtime_wast::{SpectestConfig, WastContext};
 
 /// Runs a WebAssembly test script file
 #[derive(Parser, PartialEq)]
@@ -28,7 +28,10 @@ impl WastCommand {
         let mut wast_context = WastContext::new(store);
 
         wast_context
-            .register_spectest(true)
+            .register_spectest(&SpectestConfig {
+                use_shared_memory: true,
+                suppress_prints: false,
+            })
             .expect("error instantiating \"spectest\"");
 
         for script in self.scripts.iter() {

tests/all/wast.rs

@@ -8,7 +8,7 @@ use wasmtime::{
 };
 use wasmtime_environ::WASM_PAGE_SIZE;
 use wasmtime_runtime::MpkEnabled;
-use wasmtime_wast::WastContext;
+use wasmtime_wast::{SpectestConfig, WastContext};
 
 include!(concat!(env!("OUT_DIR"), "/wast_testsuite_tests.rs"));
 
@@ -162,7 +162,10 @@ fn run_wast(wast: &str, strategy: Strategy, pooling: bool) -> anyhow::Result<()>
     for (engine, desc) in engines {
         let store = Store::new(&engine, ());
         let mut wast_context = WastContext::new(store);
-        wast_context.register_spectest(use_shared_memory)?;
+        wast_context.register_spectest(&SpectestConfig {
+            use_shared_memory,
+            suppress_prints: false,
+        })?;
         wast_context
             .run_buffer(wast.to_str().unwrap(), &wast_bytes)
             .with_context(|| format!("failed to run spec test with {desc} engine"))?;