From bab59a2cd2c8baeccdda81c23f4a4df6dcfca14b Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 25 Nov 2019 16:20:59 -0800 Subject: [PATCH] Fuzzing: Add test case logging and regression test template When the test case that causes the failure can successfully be disassembled to WAT, we get logs like this: ``` [2019-11-26T18:48:46Z INFO wasmtime_fuzzing] Wrote WAT disassembly to: /home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wat [2019-11-26T18:48:46Z INFO wasmtime_fuzzing] If this fuzz test fails, copy `/home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wat` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wat` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`: ``` #[test] fn my_fuzzing_regression_test() { let data = wat::parse_str( include_str!("./regressions/my-regression.wat") ).unwrap(); oracles::instantiate(data, CompilationStrategy::Auto) } ``` ``` If the test case cannot be disassembled to WAT, then we get logs like this: ``` [2019-11-26T18:48:46Z INFO wasmtime_fuzzing] Wrote Wasm test case to: /home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wasm [2019-11-26T18:48:46Z INFO wasmtime_fuzzing] Failed to disassemble Wasm into WAT: Bad magic number (at offset 0) Stack backtrace: Run with RUST_LIB_BACKTRACE=1 env variable to display a backtrace [2019-11-26T18:48:46Z INFO wasmtime_fuzzing] If this fuzz test fails, copy `/home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wasm` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wasm` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`: ``` #[test] fn my_fuzzing_regression_test() { let data = include_bytes!("./regressions/my-regression.wasm"); oracles::instantiate(data, CompilationStrategy::Auto) } ``` ``` --- crates/fuzzing/Cargo.toml | 4 + crates/fuzzing/src/generators.rs | 1 + crates/fuzzing/src/lib.rs | 163 ++++++++++++++++++++ crates/fuzzing/tests/regressions.rs | 9 ++ crates/fuzzing/tests/regressions/README.md | 2 + fuzz/Cargo.toml | 2 +- fuzz/fuzz_targets/compile.rs | 12 +- fuzz/fuzz_targets/instantiate.rs | 9 +- fuzz/fuzz_targets/instantiate_translated.rs | 7 +- 9 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 crates/fuzzing/tests/regressions.rs create mode 100644 crates/fuzzing/tests/regressions/README.md diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index a4956a1b01..5f710c4357 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -9,9 +9,13 @@ version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.22" arbitrary = "0.2.0" binaryen = "0.8.2" cranelift-codegen = "0.50.0" cranelift-native = "0.50.0" +env_logger = { version = "0.7.1", optional = true } +log = "0.4.8" wasmparser = "0.42.1" +wasmprinter = "0.2.0" wasmtime-jit = { path = "../jit" } diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index e9ff714ce6..043ce47baa 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -11,6 +11,7 @@ use arbitrary::{Arbitrary, Unstructured}; /// A Wasm test case generator that is powered by Binaryen's `wasm-opt -ttf`. +#[derive(Debug)] pub struct WasmOptTtf { /// The raw, encoded Wasm bytes. pub wasm: Vec, diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index c523a9bfbd..df232f8072 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -1,2 +1,165 @@ +//! Fuzzing infrastructure for Wasmtime. + +#![deny(missing_docs, missing_debug_implementations)] + pub mod generators; pub mod oracles; + +use anyhow::Context; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process; +use std::sync::{atomic, Once}; + +/// Run a fuzz test on Wasm test case with automatic logging. +/// +/// This is intended for defining the body of a `libfuzzer_sys::fuzz_target!` +/// invocation. +/// +/// Automatically prints out how to create a regression test that runs the exact +/// same set of oracles. +/// +/// It also binds the expression getting the wasm bytes to the variable, for +/// example below the `wasm` variable is assigned the value +/// `&my_test_case.as_wasm_bytes()`. This variable can be used within the body. +/// +/// ```ignore +/// use wasmtime_fuzzing::{oracles, with_log_wasm_test_case}; +/// +/// with_log_wasm_test_case!(&my_test_case.as_wasm_bytes(), |wasm| { +/// oracles::compile(wasm); +/// oracles::instantiate(wasm); +/// }); +/// ``` +#[macro_export] +macro_rules! with_log_wasm_test_case { + ( $wasm:expr , |$wasm_var:ident| $oracle:expr ) => {{ + let $wasm_var = $wasm; + wasmtime_fuzzing::log_wasm_test_case( + &$wasm_var, + stringify!($wasm_var), + stringify!($oracle), + ); + $oracle; + }}; +} + +/// Given that we are going to do a fuzz test of the given Wasm buffer, log the +/// Wasm and its WAT disassembly, and preserve them to the filesystem so that if +/// we panic or crash, we can easily inspect the test case. +/// +/// This is intended to be used via the `with_log_wasm_test_case` macro. +pub fn log_wasm_test_case(wasm: &[u8], wasm_var: &'static str, oracle_expr: &'static str) { + init_logging(); + + let wasm_path = wasm_test_case_path(); + fs::write(&wasm_path, wasm) + .with_context(|| format!("Failed to write wasm to {}", wasm_path.display())) + .unwrap(); + log::info!("Wrote Wasm test case to: {}", wasm_path.display()); + + match wasmprinter::print_bytes(wasm) { + Ok(wat) => { + log::info!("WAT disassembly:\n{}", wat); + + let wat_path = wat_disassembly_path(); + fs::write(&wat_path, &wat) + .with_context(|| { + format!("Failed to write WAT disassembly to {}", wat_path.display()) + }) + .unwrap(); + log::info!("Wrote WAT disassembly to: {}", wat_path.display()); + + log::info!( + "If this fuzz test fails, copy `{wat_path}` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wat` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`: + +``` +#[test] +fn my_fuzzing_regression_test() {{ + let {wasm_var} = wat::parse_str( + include_str!(\"./regressions/my-regression.wat\") + ).unwrap(); + {oracle_expr} +}} +```", + wat_path = wat_path.display(), + wasm_var = wasm_var, + oracle_expr = oracle_expr + ); + } + Err(e) => { + log::info!("Failed to disassemble Wasm into WAT:\n{:?}", e); + log::info!( + "If this fuzz test fails, copy `{wasm_path}` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wasm` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`: + +``` +#[test] +fn my_fuzzing_regression_test() {{ + let {wasm_var} = include_bytes!(\"./regressions/my-regression.wasm\"); + {oracle_expr} +}} +```", + wasm_path = wasm_path.display(), + wasm_var = wasm_var, + oracle_expr = oracle_expr + ); + } + } +} + +fn scratch_dir() -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("scratch"); + + static CREATE: Once = Once::new(); + CREATE.call_once(|| { + fs::create_dir_all(&dir) + .with_context(|| format!("Failed to create {}", dir.display())) + .unwrap(); + }); + + dir +} + +fn wasm_test_case_path() -> PathBuf { + static WASM_TEST_CASE_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(0); + + thread_local! { + static WASM_TEST_CASE_PATH: PathBuf = { + let dir = scratch_dir(); + dir.join(format!("{}-{}.wasm", + process::id(), + WASM_TEST_CASE_COUNTER.fetch_add(1, atomic::Ordering::SeqCst) + )) + }; + } + + WASM_TEST_CASE_PATH.with(|p| p.clone()) +} + +fn wat_disassembly_path() -> PathBuf { + static WAT_DISASSEMBLY_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(0); + + thread_local! { + static WAT_DISASSEMBLY_PATH: PathBuf = { + let dir = scratch_dir(); + dir.join(format!( + "{}-{}.wat", + process::id(), + WAT_DISASSEMBLY_COUNTER.fetch_add(1, atomic::Ordering::SeqCst) + )) + }; + } + + WAT_DISASSEMBLY_PATH.with(|p| p.clone()) +} + +#[cfg(feature = "env_logger")] +fn init_logging() { + static INIT_LOGGING: Once = Once::new(); + INIT_LOGGING.call_once(|| env_logger::init()); +} + +#[cfg(not(feature = "env_logger"))] +fn init_logging() {} diff --git a/crates/fuzzing/tests/regressions.rs b/crates/fuzzing/tests/regressions.rs new file mode 100644 index 0000000000..c463e1422e --- /dev/null +++ b/crates/fuzzing/tests/regressions.rs @@ -0,0 +1,9 @@ +//! Regression tests for bugs found via fuzzing. +//! +//! The `#[test]` goes in here, the Wasm binary goes in +//! `./regressions/some-descriptive-name.wasm`, and then the `#[test]` should +//! use the Wasm binary by including it via +//! `include_bytes!("./regressions/some-descriptive-name.wasm")`. + +#[allow(unused_imports)] // Until we actually have some regression tests... +use wasmtime_fuzzing::*; diff --git a/crates/fuzzing/tests/regressions/README.md b/crates/fuzzing/tests/regressions/README.md new file mode 100644 index 0000000000..3a1630279f --- /dev/null +++ b/crates/fuzzing/tests/regressions/README.md @@ -0,0 +1,2 @@ +This directory contains `.wasm` binaries generated during fuzzing that uncovered +a bug, and which we now use as regression tests in `../regressions.rs`. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e0bc9ba101..3fb5420242 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,7 +10,7 @@ cargo-fuzz = true [dependencies] arbitrary = "0.2.0" -wasmtime-fuzzing = { path = "../crates/fuzzing" } +wasmtime-fuzzing = { path = "../crates/fuzzing", features = ["env_logger"] } wasmtime-jit = { path = "../crates/jit" } libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git" } diff --git a/fuzz/fuzz_targets/compile.rs b/fuzz/fuzz_targets/compile.rs index 67bc4f0131..2b495f63dc 100644 --- a/fuzz/fuzz_targets/compile.rs +++ b/fuzz/fuzz_targets/compile.rs @@ -1,14 +1,20 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use wasmtime_fuzzing::oracles; +use wasmtime_fuzzing::{oracles, with_log_wasm_test_case}; use wasmtime_jit::CompilationStrategy; fuzz_target!(|data: &[u8]| { - oracles::compile(data, CompilationStrategy::Cranelift); + with_log_wasm_test_case!(data, |data| oracles::compile( + data, + CompilationStrategy::Cranelift + )); }); #[cfg(feature = "lightbeam")] fuzz_target!(|data: &[u8]| { - oracles::compile(data, CompilationStrategy::Lightbeam); + with_log_wasm_test_case!(data, |data| oracles::compile( + data, + CompilationStrategy::Lightbeam + )); }); diff --git a/fuzz/fuzz_targets/instantiate.rs b/fuzz/fuzz_targets/instantiate.rs index c43ea5e9b4..36fb104b87 100644 --- a/fuzz/fuzz_targets/instantiate.rs +++ b/fuzz/fuzz_targets/instantiate.rs @@ -1,9 +1,12 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use wasmtime_fuzzing::oracles; -use wasmtime_jit::{CompilationStrategy}; +use wasmtime_fuzzing::{oracles, with_log_wasm_test_case}; +use wasmtime_jit::CompilationStrategy; fuzz_target!(|data: &[u8]| { - oracles::instantiate(data, CompilationStrategy::Auto); + with_log_wasm_test_case!(data, |data| oracles::instantiate( + data, + CompilationStrategy::Auto + )); }); diff --git a/fuzz/fuzz_targets/instantiate_translated.rs b/fuzz/fuzz_targets/instantiate_translated.rs index 693490509b..19e313adea 100644 --- a/fuzz/fuzz_targets/instantiate_translated.rs +++ b/fuzz/fuzz_targets/instantiate_translated.rs @@ -1,9 +1,12 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use wasmtime_fuzzing::{generators, oracles}; +use wasmtime_fuzzing::{generators, oracles, with_log_wasm_test_case}; use wasmtime_jit::CompilationStrategy; fuzz_target!(|data: generators::WasmOptTtf| { - oracles::instantiate(&data.wasm, CompilationStrategy::Auto); + with_log_wasm_test_case!(&data.wasm, |wasm| oracles::instantiate( + wasm, + CompilationStrategy::Auto + )); });