From 1bf8de35f352237f58370363b6ca1ce5221d263d Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 16 Jan 2020 16:39:28 -0800 Subject: [PATCH 1/3] Add initial differential fuzzing Part of #611 --- crates/fuzzing/src/generators.rs | 38 ++++++ crates/fuzzing/src/oracles.rs | 185 ++++++++++++++++++++++++++-- crates/fuzzing/src/oracles/dummy.rs | 5 + fuzz/Cargo.toml | 4 + fuzz/fuzz_targets/differential.rs | 13 ++ 5 files changed, 237 insertions(+), 8 deletions(-) create mode 100755 fuzz/fuzz_targets/differential.rs diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index 62ed219aef..c3813a1b6e 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -45,3 +45,41 @@ impl Arbitrary for WasmOptTtf { Ok(WasmOptTtf { wasm }) } } + +/// A description of configuration options that we should do differential +/// testing between. +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)] +pub struct DifferentialConfig { + strategy: DifferentialStrategy, + opt_level: DifferentialOptLevel, +} + +impl DifferentialConfig { + /// Convert this differential fuzzing config into a `wasmtime::Config`. + pub fn to_wasmtime_config(&self) -> anyhow::Result { + let mut config = wasmtime::Config::new(); + config.strategy(match self.strategy { + DifferentialStrategy::Cranelift => wasmtime::Strategy::Cranelift, + DifferentialStrategy::Lightbeam => wasmtime::Strategy::Lightbeam, + })?; + config.cranelift_opt_level(match self.opt_level { + DifferentialOptLevel::None => wasmtime::OptLevel::None, + DifferentialOptLevel::Speed => wasmtime::OptLevel::Speed, + DifferentialOptLevel::SpeedAndSize => wasmtime::OptLevel::SpeedAndSize, + }); + Ok(config) + } +} + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)] +enum DifferentialStrategy { + Cranelift, + Lightbeam, +} + +#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)] +enum DifferentialOptLevel { + None, + Speed, + SpeedAndSize, +} diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 09d7afca8c..c7d9ecfb10 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -12,8 +12,8 @@ pub mod dummy; -use dummy::{dummy_imports, dummy_value}; -use std::collections::HashMap; +use dummy::{dummy_imports, dummy_values}; +use std::collections::{HashMap, HashSet}; use wasmtime::*; /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected @@ -67,6 +67,180 @@ pub fn compile(wasm: &[u8], strategy: Strategy) { let _ = Module::new(&store, wasm); } +/// Instantiate the given Wasm module with each `Config` and call all of its +/// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are +/// or aren't enabled for different configs, we should get the same results when +/// we call the exported functions for all of our different configs. +pub fn differential_execution( + ttf: &crate::generators::WasmOptTtf, + configs: &[crate::generators::DifferentialConfig], +) { + // We need at least two configs. + if configs.len() < 2 + // And all the configs should be unique. + || configs.iter().collect::>().len() != configs.len() + { + return; + } + + let configs: Vec<_> = match configs.iter().map(|c| c.to_wasmtime_config()).collect() { + Ok(cs) => cs, + // If the config is trying to use something that was turned off at + // compile time, eg lightbeam, just continue to the next fuzz input. + Err(_) => return, + }; + + let mut export_func_results: HashMap, Trap>> = Default::default(); + + for config in &configs { + let engine = Engine::new(config); + let store = Store::new(&engine); + + let module = match Module::new(&store, &ttf.wasm) { + Ok(module) => module, + // The module might rely on some feature that our config didn't + // enable or something like that. + Err(e) => { + eprintln!("Warning: failed to compile `wasm-opt -ttf` module: {}", e); + continue; + } + }; + + // TODO: we should implement tracing versions of these dummy imports + // that record a trace of the order that imported functions were called + // in and with what values. Like the results of exported functions, + // calls to imports should also yield the same values for each + // configuration, and we should assert that. + let imports = match dummy_imports(&store, module.imports()) { + Ok(imps) => imps, + Err(e) => { + // There are some value types that we can't synthesize a + // dummy value for (e.g. anyrefs) and for modules that + // import things of these types we skip instantiation. + eprintln!("Warning: failed to synthesize dummy imports: {}", e); + continue; + } + }; + + // Don't unwrap this: there can be instantiation-/link-time errors that + // aren't caught during validation or compilation. For example, an imported + // table might not have room for an element segment that we want to + // initialize into it. + let instance = match Instance::new(&module, &imports) { + Ok(instance) => instance, + Err(e) => { + eprintln!( + "Warning: failed to instantiate `wasm-opt -ttf` module: {}", + e + ); + continue; + } + }; + + let funcs = module + .exports() + .iter() + .filter_map(|e| { + if let ExternType::Func(_) = e.ty() { + Some(e.name()) + } else { + None + } + }) + .collect::>(); + + for name in funcs { + // Always call the hang limit initializer first, so that we don't + // infinite loop when calling another export. + init_hang_limit(&instance); + + let f = match instance + .get_export(&name) + .expect("instance should have export from module") + { + Extern::Func(f) => f.clone(), + _ => panic!("export should be a function"), + }; + + let ty = f.ty(); + let params = match dummy_values(ty.params()) { + Ok(p) => p, + Err(_) => continue, + }; + let this_result = f.call(¶ms); + + let existing_result = export_func_results + .entry(name.to_string()) + .or_insert_with(|| this_result.clone()); + assert_same_export_func_result(&existing_result, &this_result, name); + } + } +} + +fn init_hang_limit(instance: &Instance) { + match instance.get_export("hangLimitInitializer") { + None => return, + Some(Extern::Func(f)) => { + f.call(&[]) + .expect("initializing the hang limit should not fail"); + } + Some(_) => panic!("unexpected hangLimitInitializer export"), + } +} + +fn assert_same_export_func_result( + lhs: &Result, Trap>, + rhs: &Result, Trap>, + func_name: &str, +) { + let fail = || { + panic!( + "differential fuzzing failed: exported func {} returned two \ + different results: {:?} != {:?}", + func_name, lhs, rhs + ) + }; + + match (lhs, rhs) { + (Err(_), Err(_)) => {} + (Ok(lhs), Ok(rhs)) => { + if lhs.len() != rhs.len() { + fail(); + } + for (lhs, rhs) in lhs.iter().zip(rhs.iter()) { + match (lhs, rhs) { + (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue, + (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue, + (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue, + (Val::F32(lhs), Val::F32(rhs)) => { + let lhs = f32::from_bits(*lhs); + let rhs = f32::from_bits(*rhs); + if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { + continue; + } else { + fail() + } + } + (Val::F64(lhs), Val::F64(rhs)) => { + let lhs = f64::from_bits(*lhs); + let rhs = f64::from_bits(*rhs); + if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { + continue; + } else { + fail() + } + } + (Val::AnyRef(_), Val::AnyRef(_)) | (Val::FuncRef(_), Val::FuncRef(_)) => { + continue + } + _ => fail(), + } + } + } + _ => fail(), + } +} + /// Invoke the given API calls. pub fn make_api_calls(api: crate::generators::api::ApiCalls) { use crate::generators::api::ApiCall; @@ -170,12 +344,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { let nth = nth % funcs.len(); let f = &funcs[nth]; let ty = f.ty(); - let params = match ty - .params() - .iter() - .map(|valty| dummy_value(valty)) - .collect::, _>>() - { + let params = match dummy_values(ty.params()) { Ok(p) => p, Err(_) => continue, }; diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index b51e80e1f4..b06297d376 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -70,6 +70,11 @@ pub fn dummy_value(val_ty: &ValType) -> Result { }) } +/// Construct a sequence of dummy values for the given types. +pub fn dummy_values(val_tys: &[ValType]) -> Result, Trap> { + val_tys.iter().map(dummy_value).collect() +} + /// Construct a dummy global for the given global type. pub fn dummy_global(store: &Store, ty: GlobalType) -> Result { let val = dummy_value(ty.content())?; diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0d28aae9a1..2e2fe38cbf 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -39,3 +39,7 @@ name = "api_calls" path = "fuzz_targets/api_calls.rs" test = false doc = false + +[[bin]] +name = "differential" +path = "fuzz_targets/differential.rs" diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs new file mode 100755 index 0000000000..5cf14f0523 --- /dev/null +++ b/fuzz/fuzz_targets/differential.rs @@ -0,0 +1,13 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use wasmtime_fuzzing::{generators, oracles}; + +fuzz_target!(|data: ( + generators::DifferentialConfig, + generators::DifferentialConfig, + generators::WasmOptTtf +)| { + let (lhs, rhs, wasm) = data; + oracles::differential_execution(&wasm, &[lhs, rhs]); +}); From 1667c462c59c90808d5a9ba82229055ed4448482 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 17 Jan 2020 16:19:38 -0800 Subject: [PATCH 2/3] CI: run a sample of our differential fuzz test corpus in CI --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1dd398f965..bc13378761 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,6 +98,11 @@ jobs: | shuf \ | head -n 100 \ | xargs cargo fuzz run api_calls --release --debug-assertions + - run: | + find fuzz/corpus/differential -type f \ + | shuf \ + | head -n 100 \ + | xargs cargo fuzz run differential --release --debug-assertions # Install wasm32-unknown-emscripten target, and ensure `crates/wasi-common` # compiles to Emscripten. From 386c3a94b00fc4542b132d878eef6e456dfa0c60 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 21 Jan 2020 09:32:52 -0800 Subject: [PATCH 3/3] fuzz: Don't run tests or docs for `differential` fuzz target --- fuzz/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 2e2fe38cbf..b286c21d2d 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -43,3 +43,5 @@ doc = false [[bin]] name = "differential" path = "fuzz_targets/differential.rs" +test = false +doc = false