Browse Source

Add differential fuzzing against wasmi (a Wasm interpreter).

This PR adds a new fuzz target, `differential_wasmi`, that runs a
Cranelift-based Wasm backend alongside a simple third-party Wasm
interpeter crate (`wasmi`).  The fuzzing runs the first function in a
given module to completion on each side, and then diffs the return value
and linear memory contents.

This strategy should provide end-to-end coverage including both the Wasm
translation to CLIF (which has seen some subtle and scary bugs at
times), the lowering from CLIF to VCode, the register allocation, and
the final code emission.

This PR also adds a feature `experimental_x64` to the fuzzing crate (and
the chain of dependencies down to `cranelift-codegen`) so that we can
fuzz the new x86-64 backend as well as the current one.
pull/2453/head
Chris Fallin 4 years ago
parent
commit
bbdea06e2d
No known key found for this signature in database GPG Key ID: 31649E4FE65EB465
  1. 80
      Cargo.lock
  2. 6
      crates/fuzzing/Cargo.toml
  3. 202
      crates/fuzzing/src/oracles.rs
  4. 3
      crates/wasmtime/Cargo.toml
  5. 11
      fuzz/Cargo.toml
  6. 13
      fuzz/fuzz_targets/differential_wasmi.rs

80
Cargo.lock

@ -766,6 +766,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "downcast-rs"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
[[package]]
name = "dynasm"
version = "1.0.0"
@ -1238,6 +1244,12 @@ dependencies = [
"autocfg",
]
[[package]]
name = "memory_units"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882"
[[package]]
name = "miniz_oxide"
version = "0.4.3"
@ -1264,6 +1276,17 @@ dependencies = [
"version_check",
]
[[package]]
name = "num-bigint"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-integer"
version = "0.1.44"
@ -1274,6 +1297,18 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg",
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
@ -1352,6 +1387,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "parity-wasm"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865"
[[package]]
name = "peeking_take_while"
version = "0.1.2"
@ -2312,14 +2353,48 @@ dependencies = [
"yanix",
]
[[package]]
name = "wasm-encoder"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49891b7c581cf9e0090b25cd274e6498ad478f0a7819319ea96da2f253caaacb"
dependencies = [
"leb128",
]
[[package]]
name = "wasm-smith"
version = "0.1.10"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd5e4720bb44dc5e46a917139dc9dfa4bb6fee023bf9f77d2c55ec12eeaf9930"
checksum = "c7fdf8c9ba2fdc0d8ffe3f7e5b23b5aac377eeca817a9885c058f27c8de5c500"
dependencies = [
"arbitrary",
"leb128",
"wasm-encoder",
]
[[package]]
name = "wasmi"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6825d9b2147105789adb4c2d84b9b568719713f3ac39618b637b4dafc86c4"
dependencies = [
"downcast-rs",
"libc",
"memory_units",
"num-rational",
"num-traits",
"parity-wasm",
"wasmi-validation",
]
[[package]]
name = "wasmi-validation"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93"
dependencies = [
"parity-wasm",
]
[[package]]
@ -2522,6 +2597,7 @@ dependencies = [
"log",
"rayon",
"wasm-smith",
"wasmi",
"wasmparser 0.68.0",
"wasmprinter",
"wasmtime",

6
crates/fuzzing/Cargo.toml

@ -16,7 +16,11 @@ wasmparser = "0.68.0"
wasmprinter = "0.2.15"
wasmtime = { path = "../wasmtime" }
wasmtime-wast = { path = "../wast" }
wasm-smith = "0.1.10"
wasm-smith = "0.1.12"
wasmi = "0.7.0"
[dev-dependencies]
wat = "1.0.28"
[features]
experimental_x64 = ["wasmtime/experimental_x64"]

202
crates/fuzzing/src/oracles.rs

@ -12,7 +12,9 @@
pub mod dummy;
use arbitrary::Arbitrary;
use dummy::dummy_imports;
use log::debug;
use std::cell::Cell;
use std::rc::Rc;
use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
@ -249,24 +251,8 @@ pub fn differential_execution(
(Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue,
(Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue,
(Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue,
(Val::F32(lhs), Val::F32(rhs)) => {
let lhs = f32::from_bits(*lhs);
let rhs = f32::from_bits(*rhs);
if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) {
continue;
} else {
fail()
}
}
(Val::F64(lhs), Val::F64(rhs)) => {
let lhs = f64::from_bits(*lhs);
let rhs = f64::from_bits(*rhs);
if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) {
continue;
} else {
fail()
}
}
(Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue,
(Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue,
(Val::ExternRef(_), Val::ExternRef(_))
| (Val::FuncRef(_), Val::FuncRef(_)) => continue,
_ => fail(),
@ -278,6 +264,18 @@ pub fn differential_execution(
}
}
fn f32_equal(a: u32, b: u32) -> bool {
let a = f32::from_bits(a);
let b = f32::from_bits(b);
a == b || (a.is_nan() && b.is_nan())
}
fn f64_equal(a: u64, b: u64) -> bool {
let a = f64::from_bits(a);
let b = f64::from_bits(b);
a == b || (a.is_nan() && b.is_nan())
}
/// Invoke the given API calls.
pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
use crate::generators::api::ApiCall;
@ -479,3 +477,171 @@ pub fn table_ops(config: crate::generators::Config, ops: crate::generators::tabl
}
}
}
/// Configuration options for wasm-smith such that generated modules always
/// conform to certain specifications.
#[derive(Default, Debug, Arbitrary)]
pub struct DifferentialWasmiModuleConfig;
impl wasm_smith::Config for DifferentialWasmiModuleConfig {
fn allow_start_export(&self) -> bool {
false
}
fn min_funcs(&self) -> usize {
1
}
fn max_funcs(&self) -> usize {
1
}
fn min_memories(&self) -> u32 {
1
}
fn max_memories(&self) -> u32 {
1
}
fn max_imports(&self) -> usize {
0
}
fn min_exports(&self) -> usize {
2
}
fn max_memory_pages(&self) -> u32 {
1
}
fn memory_max_size_required(&self) -> bool {
true
}
}
/// Perform differential execution between Cranelift and wasmi, diffing the
/// resulting memory image when execution terminates. This relies on the
/// module-under-test to be instrumented to bound the execution time. Invoke
/// with a module generated by `wasm-smith` using the
/// `DiferentialWasmiModuleConfig` configuration type for best results.
///
/// May return `None` if we early-out due to a rejected fuzz config; these
/// should be rare if modules are generated appropriately.
pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> {
crate::init_fuzzing();
// Instantiate wasmi module and instance.
let wasmi_module = wasmi::Module::from_buffer(&wasm[..]).ok()?;
let wasmi_instance =
wasmi::ModuleInstance::new(&wasmi_module, &wasmi::ImportsBuilder::default()).ok()?;
let wasmi_instance = wasmi_instance.assert_no_start();
// TODO(paritytech/wasmi#19): wasmi does not currently canonicalize NaNs. To avoid spurious
// fuzz failures, for now let's fuzz only integer Wasm programs.
if wasmi_module.deny_floating_point().is_err() {
return None;
}
// Instantiate wasmtime module and instance.
let mut wasmtime_config = config.to_wasmtime();
wasmtime_config.cranelift_nan_canonicalization(true);
let wasmtime_engine = Engine::new(&wasmtime_config);
let wasmtime_store = Store::new(&wasmtime_engine);
let wasmtime_module =
Module::new(&wasmtime_engine, &wasm).expect("Wasmtime can compile module");
let wasmtime_instance = Instance::new(&wasmtime_store, &wasmtime_module, &[])
.expect("Wasmtime can instantiate module");
// Introspect wasmtime module to find name of an exported function and of an
// exported memory. Stop when we have one of each. (According to the config
// above, there should be at most one of each.)
let (func_name, memory_name) = {
let mut func_name = None;
let mut memory_name = None;
for e in wasmtime_module.exports() {
match e.ty() {
wasmtime::ExternType::Func(..) => func_name = Some(e.name().to_string()),
wasmtime::ExternType::Memory(..) => memory_name = Some(e.name().to_string()),
_ => {}
}
if func_name.is_some() && memory_name.is_some() {
break;
}
}
(func_name?, memory_name?)
};
let wasmi_mem_export = wasmi_instance.export_by_name(&memory_name[..]).unwrap();
let wasmi_mem = wasmi_mem_export.as_memory().unwrap();
let wasmi_main_export = wasmi_instance.export_by_name(&func_name[..]).unwrap();
let wasmi_main = wasmi_main_export.as_func().unwrap();
let wasmi_val = wasmi::FuncInstance::invoke(&wasmi_main, &[], &mut wasmi::NopExternals);
let wasmtime_mem = wasmtime_instance
.get_memory(&memory_name[..])
.expect("memory export is present");
let wasmtime_main = wasmtime_instance
.get_func(&func_name[..])
.expect("function export is present");
let wasmtime_vals = wasmtime_main.call(&[]);
let wasmtime_val = wasmtime_vals.map(|v| v.iter().next().cloned());
debug!(
"Successful execution: wasmi returned {:?}, wasmtime returned {:?}",
wasmi_val, wasmtime_val
);
let show_wat = || {
if let Ok(s) = wasmprinter::print_bytes(&wasm[..]) {
eprintln!("wat:\n{}\n", s);
}
};
match (&wasmi_val, &wasmtime_val) {
(&Ok(Some(wasmi::RuntimeValue::I32(a))), &Ok(Some(Val::I32(b)))) if a == b => {}
(&Ok(Some(wasmi::RuntimeValue::F32(a))), &Ok(Some(Val::F32(b))))
if f32_equal(a.to_bits(), b) => {}
(&Ok(Some(wasmi::RuntimeValue::I64(a))), &Ok(Some(Val::I64(b)))) if a == b => {}
(&Ok(Some(wasmi::RuntimeValue::F64(a))), &Ok(Some(Val::F64(b))))
if f64_equal(a.to_bits(), b) => {}
(&Ok(None), &Ok(None)) => {}
(&Err(_), &Err(_)) => {}
_ => {
show_wat();
panic!(
"Values do not match: wasmi returned {:?}; wasmtime returned {:?}",
wasmi_val, wasmtime_val
);
}
}
if wasmi_mem.current_size().0 != wasmtime_mem.size() as usize {
show_wat();
panic!("resulting memories are not the same size");
}
// Wasmi memory may be stored non-contiguously; copy it out to a contiguous chunk.
let mut wasmi_buf: Vec<u8> = vec![0; wasmtime_mem.data_size()];
wasmi_mem
.get_into(0, &mut wasmi_buf[..])
.expect("can access wasmi memory");
let wasmtime_slice = unsafe { wasmtime_mem.data_unchecked() };
if wasmi_buf.len() >= 64 {
debug!("-> First 64 bytes of wasmi heap: {:?}", &wasmi_buf[0..64]);
debug!(
"-> First 64 bytes of Wasmtime heap: {:?}",
&wasmtime_slice[0..64]
);
}
if &wasmi_buf[..] != &wasmtime_slice[..] {
show_wat();
panic!("memory contents are not equal");
}
Some(())
}

3
crates/wasmtime/Cargo.toml

@ -59,3 +59,6 @@ parallel-compilation = ["wasmtime-jit/parallel-compilation"]
# Enables support for automatic cache configuration to be enabled in `Config`.
cache = ["wasmtime-cache"]
# Enables support for new x64 backend.
experimental_x64 = ["wasmtime-jit/experimental_x64"]

11
fuzz/Cargo.toml

@ -17,7 +17,10 @@ target-lexicon = "0.11"
peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true }
wasmtime = { path = "../crates/wasmtime" }
wasmtime-fuzzing = { path = "../crates/fuzzing" }
wasm-smith = "0.1.5"
wasm-smith = "0.1.12"
[features]
experimental_x64 = ["wasmtime-fuzzing/experimental_x64"]
[[bin]]
name = "compile"
@ -43,6 +46,12 @@ path = "fuzz_targets/differential.rs"
test = false
doc = false
[[bin]]
name = "differential_wasmi"
path = "fuzz_targets/differential_wasmi.rs"
test = false
doc = false
[[bin]]
name = "spectests"
path = "fuzz_targets/spectests.rs"

13
fuzz/fuzz_targets/differential_wasmi.rs

@ -0,0 +1,13 @@
#![no_main]
use libfuzzer_sys::fuzz_target;
use wasmtime_fuzzing::{generators, oracles};
fuzz_target!(|data: (
generators::Config,
wasm_smith::ConfiguredModule<oracles::DifferentialWasmiModuleConfig>
)| {
let (config, mut wasm) = data;
wasm.ensure_termination(1000);
oracles::differential_wasmi_execution(&wasm.to_bytes()[..], &config);
});
Loading…
Cancel
Save