Split our existing fuzz targets into separate generators and oracles

Part of #611

crates/fuzzing/Cargo.toml

@@ -9,3 +9,9 @@ version = "0.1.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+arbitrary = "0.2.0"
+binaryen = "0.8.1"
+cranelift-codegen = "0.50.0"
+cranelift-native = "0.50.0"
+wasmparser = "0.42.1"
+wasmtime-jit = { path = "../jit" }

crates/fuzzing/src/generators.rs

@@ -0,0 +1,29 @@
+//! Test case generators.
+//!
+//! Test case generators take raw, unstructured input from a fuzzer
+//! (e.g. libFuzzer) and translate that into a structured test case (e.g. a
+//! valid Wasm binary).
+//!
+//! These are generally implementations of the `Arbitrary` trait, or some
+//! wrapper over an external tool, such that the wrapper implements the
+//! `Arbitrary` trait for the wrapped external tool.
+
+use arbitrary::{Arbitrary, Unstructured};
+
+/// A Wasm test case generator that is powered by Binaryen's `wasm-opt -ttf`.
+pub struct WasmOptTtf {
+    /// The raw, encoded Wasm bytes.
+    pub wasm: Vec<u8>,
+}
+
+impl Arbitrary for WasmOptTtf {
+    fn arbitrary<U>(input: &mut U) -> Result<Self, U::Error>
+    where
+        U: Unstructured + ?Sized,
+    {
+        let seed: Vec<u8> = Arbitrary::arbitrary(input)?;
+        let module = binaryen::tools::translate_to_fuzz_mvp(&seed);
+        let wasm = module.write();
+        Ok(WasmOptTtf { wasm })
+    }
+}

crates/fuzzing/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod generators;
+pub mod oracles;

crates/fuzzing/src/oracles.rs

@@ -0,0 +1,66 @@
+//! Oracles.
+//!
+//! Oracles take a test case and determine whether we have a bug. For example,
+//! one of the simplest oracles is to take a Wasm binary as our input test case,
+//! validate and instantiate it, and (implicitly) check that no assertions
+//! failed or segfaults happened. A more complicated oracle might compare the
+//! result of executing a Wasm file with and without optimizations enabled, and
+//! make sure that the two executions are observably identical.
+//!
+//! When an oracle finds a bug, it should report it to the fuzzing engine by
+//! panicking.
+
+use cranelift_codegen::settings;
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::rc::Rc;
+use wasmtime_jit::{CompilationStrategy, CompiledModule, Compiler, NullResolver};
+
+fn host_isa() -> Box<dyn cranelift_codegen::isa::TargetIsa> {
+    let flag_builder = settings::builder();
+    let isa_builder = cranelift_native::builder().expect("host machine is not a supported target");
+    isa_builder.finish(settings::Flags::new(flag_builder))
+}
+
+/// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected
+/// panic or segfault or anything else that can be detected "passively".
+///
+/// Performs initial validation, and returns early if the Wasm is invalid.
+///
+/// You can control which compiler is used via passing a `CompilationStrategy`.
+pub fn instantiate(wasm: &[u8], compilation_strategy: CompilationStrategy) {
+    if wasmparser::validate(wasm, None).is_err() {
+        return;
+    }
+
+    let isa = host_isa();
+    let mut compiler = Compiler::new(isa, compilation_strategy);
+    let mut imports_resolver = NullResolver {};
+
+    wasmtime_jit::instantiate(
+        &mut compiler,
+        wasm,
+        &mut imports_resolver,
+        Default::default(),
+        true,
+    )
+    .expect("failed to instantiate valid Wasm!");
+}
+
+/// Compile the Wasm buffer, and implicitly fail if we have an unexpected
+/// panic or segfault or anything else that can be detected "passively".
+///
+/// Performs initial validation, and returns early if the Wasm is invalid.
+///
+/// You can control which compiler is used via passing a `CompilationStrategy`.
+pub fn compile(wasm: &[u8], compilation_strategy: CompilationStrategy) {
+    if wasmparser::validate(wasm, None).is_err() {
+        return;
+    }
+
+    let isa = host_isa();
+    let mut compiler = Compiler::new(isa, compilation_strategy);
+    let mut resolver = NullResolver {};
+    let global_exports = Rc::new(RefCell::new(HashMap::new()));
+    let _ = CompiledModule::new(&mut compiler, wasm, &mut resolver, global_exports, false);
+}

fuzz/Cargo.toml

@@ -9,14 +9,10 @@ publish = false
 cargo-fuzz = true
 
 [dependencies]
-wasmtime-environ = { path = "../crates/environ" }
+arbitrary = "0.2.0"
+wasmtime-fuzzing = { path = "../crates/fuzzing" }
 wasmtime-jit = { path = "../crates/jit" }
-cranelift-codegen = "0.50"
-cranelift-wasm = "0.50"
-cranelift-native = "0.50"
 libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git" }
-wasmparser = { version = "0.39.2", default-features = false, features = ["core"] }
-binaryen = "0.8.1"
 
 # Prevent this from interfering with workspaces
 [workspace]

fuzz/fuzz_targets/compile.rs

@@ -2,49 +2,15 @@
 
 extern crate libfuzzer_sys;
 
-use cranelift_codegen::settings;
 use libfuzzer_sys::fuzz_target;
-use std::cell::RefCell;
-use std::collections::HashMap;
-use std::rc::Rc;
-use wasmparser::validate;
-use wasmtime_jit::{CompilationStrategy, CompiledModule, Compiler, NullResolver};
+use wasmtime_fuzzing::oracles;
+use wasmtime_jit::CompilationStrategy;
 
 fuzz_target!(|data: &[u8]| {
-    if validate(data, None).is_err() {
-        return;
-    }
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Cranelift);
-    let mut resolver = NullResolver {};
-    let global_exports = Rc::new(RefCell::new(HashMap::new()));
-    let _compiled =
-        match CompiledModule::new(&mut compiler, data, &mut resolver, global_exports, false) {
-            Ok(x) => x,
-            Err(_) => return,
-        };
+    oracles::compile(data, CompilationStrategy::Cranelift);
 });
 
 #[cfg(feature = "lightbeam")]
 fuzz_target!(|data: &[u8]| {
-    if validate(data, None).is_err() {
-        return;
-    }
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Lightbeam);
-    let mut resolver = NullResolver {};
-    let global_exports = Rc::new(RefCell::new(HashMap::new()));
-    let _compiled =
-        match CompiledModule::new(&mut compiler, data, &mut resolver, global_exports, false) {
-            Ok(x) => x,
-            Err(_) => return,
-        };
+    oracles::compile(data, CompilationStrategy::Lightbeam);
 });

fuzz/fuzz_targets/instantiate.rs

@@ -2,28 +2,10 @@
 
 extern crate libfuzzer_sys;
 
-use cranelift_codegen::settings;
 use libfuzzer_sys::fuzz_target;
-use wasmparser::validate;
-use wasmtime_jit::{instantiate, CompilationStrategy, Compiler, NullResolver};
+use wasmtime_fuzzing::oracles;
+use wasmtime_jit::{CompilationStrategy};
 
 fuzz_target!(|data: &[u8]| {
-    if validate(data, None).is_err() {
-        return;
-    }
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Auto);
-    let mut imports_resolver = NullResolver {};
-    let _instance = instantiate(
-        &mut compiler,
-        data,
-        &mut imports_resolver,
-        Default::default(),
-        true,
-    )
-    .unwrap();
+    oracles::instantiate(data, CompilationStrategy::Auto);
 });

fuzz/fuzz_targets/instantiate_translated.rs

@@ -2,26 +2,10 @@
 
 extern crate libfuzzer_sys;
 
-use cranelift_codegen::settings;
 use libfuzzer_sys::fuzz_target;
-use wasmtime_jit::{instantiate, CompilationStrategy, Compiler, NullResolver};
+use wasmtime_fuzzing::{generators, oracles};
+use wasmtime_jit::CompilationStrategy;
 
-fuzz_target!(|data: &[u8]| {
-    let binaryen_module = binaryen::tools::translate_to_fuzz_mvp(data);
-    let wasm = binaryen_module.write();
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Auto);
-    let mut imports_resolver = NullResolver {};
-    let _instance = instantiate(
-        &mut compiler,
-        &wasm,
-        &mut imports_resolver,
-        Default::default(),
-        true,
-    )
-    .unwrap();
+fuzz_target!(|data: generators::WasmOptTtf| {
+    oracles::instantiate(&data.wasm, CompilationStrategy::Auto);
 });