Support disabling backtraces at compile time (#3932)

* Support disabling backtraces at compile time

This commit adds support to Wasmtime to disable, at compile time, the
gathering of backtraces on traps. The `wasmtime` crate now sports a
`wasm-backtrace` feature which, when disabled, will mean that backtraces
are never collected at compile time nor are unwinding tables inserted
into compiled objects.

The motivation for this commit stems from the fact that generating a
backtrace is quite a slow operation. Currently backtrace generation is
done with libunwind and `_Unwind_Backtrace` typically found in glibc or
other system libraries. When thousands of modules are loaded into the
same process though this means that the initial backtrace can take
nearly half a second and all subsequent backtraces can take upwards of
hundreds of milliseconds. Relative to all other operations in Wasmtime
this is extremely expensive at this time. In the future we'd like to
implement a more performant backtrace scheme but such an implementation
would require coordination with Cranelift and is a big chunk of work
that may take some time, so in the meantime if embedders don't need a
backtrace they can still use this option to disable backtraces at
compile time and avoid the performance pitfalls of collecting
backtraces.

In general I tried to originally make this a runtime configuration
option but ended up opting for a compile-time option because `Trap::new`
otherwise has no arguments and always captures a backtrace. By making
this a compile-time option it was possible to configure, statically, the
behavior of `Trap::new`. Additionally I also tried to minimize the
amount of `#[cfg]` necessary by largely only having it at the producer
and consumer sites.

Also a noteworthy restriction of this implementation is that if
backtrace support is disabled at compile time then reference types
support will be unconditionally disabled at runtime. With backtrace
support disabled there's no way to trace the stack of wasm frames which
means that GC can't happen given our current implementation.

* Always enable backtraces for the C API

.github/workflows/main.yml

@@ -137,7 +137,8 @@ jobs:
     - run: cargo check -p wasmtime --no-default-features --features uffd
     - run: cargo check -p wasmtime --no-default-features --features pooling-allocator
     - run: cargo check -p wasmtime --no-default-features --features cranelift
-    - run: cargo check -p wasmtime --no-default-features --features cranelift,wat,async,cache
+    - run: cargo check -p wasmtime --no-default-features --features wasm-backtrace
+    - run: cargo check -p wasmtime --no-default-features --features cranelift,wat,async,cache,wasm-backtrace
 
     # Check that benchmarks of the cranelift project build
     - run: cargo check --benches -p cranelift-codegen

Cargo.toml

@@ -99,6 +99,7 @@ default = [
   "wasi-nn",
   "pooling-allocator",
   "memory-init-cow",
+  "wasm-backtrace",
 ]
 jitdump = ["wasmtime/jitdump"]
 vtune = ["wasmtime/vtune"]
@@ -109,6 +110,7 @@ memory-init-cow = ["wasmtime/memory-init-cow"]
 pooling-allocator = ["wasmtime/pooling-allocator"]
 all-arch = ["wasmtime/all-arch"]
 posix-signals-on-macos = ["wasmtime/posix-signals-on-macos"]
+wasm-backtrace = ["wasmtime/wasm-backtrace"]
 
 # Stub feature that does nothing, for Cargo-features compatibility: the new
 # backend is the default now.

crates/c-api/Cargo.toml

@@ -20,7 +20,7 @@ doctest = false
 env_logger = "0.8"
 anyhow = "1.0"
 once_cell = "1.3"
-wasmtime = { path = "../wasmtime", default-features = false, features = ['cranelift'] }
+wasmtime = { path = "../wasmtime", default-features = false, features = ['cranelift', 'wasm-backtrace'] }
 wasmtime-c-api-macros = { path = "macros" }
 
 # Optional dependency for the `wat2wasm` API

crates/cranelift/src/compiler.rs

@@ -188,9 +188,13 @@ impl wasmtime_environ::Compiler for Compiler {
 
         let stack_maps = mach_stack_maps_to_stack_maps(result.buffer.stack_maps());
 
-        let unwind_info = context
-            .create_unwind_info(isa)
-            .map_err(|error| CompileError::Codegen(pretty_error(&context.func, error)))?;
+        let unwind_info = if isa.flags().unwind_info() {
+            context
+                .create_unwind_info(isa)
+                .map_err(|error| CompileError::Codegen(pretty_error(&context.func, error)))?
+        } else {
+            None
+        };
 
         let address_transform =
             self.get_function_address_map(&context, &input, code_buf.len() as u32, tunables);
@@ -566,9 +570,13 @@ impl Compiler {
             .relocs()
             .is_empty());
 
-        let unwind_info = context
-            .create_unwind_info(isa)
-            .map_err(|error| CompileError::Codegen(pretty_error(&context.func, error)))?;
+        let unwind_info = if isa.flags().unwind_info() {
+            context
+                .create_unwind_info(isa)
+                .map_err(|error| CompileError::Codegen(pretty_error(&context.func, error)))?
+        } else {
+            None
+        };
 
         Ok(CompiledFunction {
             body: code_buf,

crates/runtime/Cargo.toml

@@ -22,7 +22,7 @@ indexmap = "1.0.2"
 thiserror = "1.0.4"
 more-asserts = "0.2.1"
 cfg-if = "1.0"
-backtrace = "0.3.61"
+backtrace = { version = "0.3.61", optional = true }
 rand = "0.8.3"
 anyhow = "1.0.38"
 memfd = { version = "0.4.1", optional = true }
@@ -46,8 +46,8 @@ cc = "1.0"
 maintenance = { status = "actively-developed" }
 
 [features]
-default = []
 memory-init-cow = ['memfd']
+wasm-backtrace = ["backtrace"]
 
 async = ["wasmtime-fiber"]
 

crates/runtime/src/externref.rs

@@ -704,6 +704,7 @@ impl VMExternRefActivationsTable {
         }
     }
 
+    #[cfg_attr(not(feature = "wasm-backtrace"), allow(dead_code))]
     fn insert_precise_stack_root(
         precise_stack_roots: &mut HashSet<VMExternRefWithTraits>,
         root: NonNull<VMExternData>,
@@ -866,6 +867,7 @@ impl<T> std::ops::DerefMut for DebugOnly<T> {
 ///
 /// Additionally, you must have registered the stack maps for every Wasm module
 /// that has frames on the stack with the given `stack_maps_registry`.
+#[cfg_attr(not(feature = "wasm-backtrace"), allow(unused_mut, unused_variables))]
 pub unsafe fn gc(
     module_info_lookup: &dyn ModuleInfoLookup,
     externref_activations_table: &mut VMExternRefActivationsTable,
@@ -893,6 +895,7 @@ pub unsafe fn gc(
         None => {
             if cfg!(debug_assertions) {
                 // Assert that there aren't any Wasm frames on the stack.
+                #[cfg(feature = "wasm-backtrace")]
                 backtrace::trace(|frame| {
                     assert!(module_info_lookup.lookup(frame.ip() as usize).is_none());
                     true
@@ -917,7 +920,7 @@ pub unsafe fn gc(
     //   newly-discovered precise set.
 
     // The SP of the previous (younger) frame we processed.
-    let mut last_sp = None;
+    let mut last_sp: Option<usize> = None;
 
     // Whether we have found our stack canary or not yet.
     let mut found_canary = false;
@@ -934,6 +937,7 @@ pub unsafe fn gc(
         });
     }
 
+    #[cfg(feature = "wasm-backtrace")]
     backtrace::trace(|frame| {
         let pc = frame.ip() as usize;
         let sp = frame.sp() as usize;

crates/runtime/src/lib.rs

@@ -61,7 +61,7 @@ pub use crate::mmap_vec::MmapVec;
 pub use crate::table::{Table, TableElement};
 pub use crate::traphandlers::{
     catch_traps, init_traps, raise_lib_trap, raise_user_trap, resume_panic, tls_eager_initialize,
-    SignalHandler, TlsRestore, Trap,
+    Backtrace, SignalHandler, TlsRestore, Trap,
 };
 pub use crate::vmcontext::{
     VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, VMGlobalDefinition,

crates/runtime/src/libcalls.rs

@@ -61,7 +61,6 @@ use crate::instance::Instance;
 use crate::table::{Table, TableElementType};
 use crate::traphandlers::{raise_lib_trap, resume_panic, Trap};
 use crate::vmcontext::{VMCallerCheckedAnyfunc, VMContext};
-use backtrace::Backtrace;
 use std::mem;
 use std::ptr::{self, NonNull};
 use wasmtime_environ::{
@@ -588,10 +587,7 @@ unsafe fn validate_atomic_addr(
     addr: usize,
 ) -> Result<(), Trap> {
     if addr > instance.get_memory(memory).current_length {
-        return Err(Trap::Wasm {
-            trap_code: TrapCode::HeapOutOfBounds,
-            backtrace: Backtrace::new_unresolved(),
-        });
+        return Err(Trap::wasm(TrapCode::HeapOutOfBounds));
     }
     Ok(())
 }

crates/runtime/src/traphandlers.rs

@@ -3,7 +3,6 @@
 
 use crate::VMContext;
 use anyhow::Error;
-use backtrace::Backtrace;
 use std::any::Any;
 use std::cell::{Cell, UnsafeCell};
 use std::mem::MaybeUninit;
@@ -143,10 +142,9 @@ impl Trap {
     ///
     /// Internally saves a backtrace when constructed.
     pub fn wasm(trap_code: TrapCode) -> Self {
-        let backtrace = Backtrace::new_unresolved();
         Trap::Wasm {
             trap_code,
-            backtrace,
+            backtrace: Backtrace::new(),
         }
     }
 
@@ -154,8 +152,38 @@ impl Trap {
     ///
     /// Internally saves a backtrace when constructed.
     pub fn oom() -> Self {
-        let backtrace = Backtrace::new_unresolved();
-        Trap::OOM { backtrace }
+        Trap::OOM {
+            backtrace: Backtrace::new(),
+        }
+    }
+}
+
+/// A crate-local backtrace type which conditionally, at compile time, actually
+/// contains a backtrace from the `backtrace` crate or nothing.
+#[derive(Debug)]
+pub struct Backtrace {
+    #[cfg(feature = "wasm-backtrace")]
+    trace: backtrace::Backtrace,
+}
+
+impl Backtrace {
+    /// Captures a new backtrace
+    ///
+    /// Note that this function does nothing if the `wasm-backtrace` feature is
+    /// disabled.
+    pub fn new() -> Backtrace {
+        Backtrace {
+            #[cfg(feature = "wasm-backtrace")]
+            trace: backtrace::Backtrace::new_unresolved(),
+        }
+    }
+
+    /// Returns the backtrace frames associated with this backtrace. Note that
+    /// this is conditionally defined and not present when `wasm-backtrace` is
+    /// not present.
+    #[cfg(feature = "wasm-backtrace")]
+    pub fn frames(&self) -> &[backtrace::BacktraceFrame] {
+        self.trace.frames()
     }
 }
 
@@ -299,7 +327,7 @@ impl CallThreadState {
     }
 
     fn capture_backtrace(&self, pc: *const u8) {
-        let backtrace = Backtrace::new_unresolved();
+        let backtrace = Backtrace::new();
         unsafe {
             (*self.unwind.get())
                 .as_mut_ptr()

crates/wasmtime/Cargo.toml

@@ -25,7 +25,7 @@ anyhow = "1.0.19"
 region = "2.2.0"
 libc = "0.2"
 cfg-if = "1.0"
-backtrace = "0.3.61"
+backtrace = { version = "0.3.61", optional = true }
 log = "0.4.8"
 wat = { version = "1.0.36", optional = true }
 serde = { version = "1.0.94", features = ["derive"] }
@@ -61,6 +61,7 @@ default = [
   'pooling-allocator',
   'memory-init-cow',
   'vtune',
+  'wasm-backtrace',
 ]
 
 # An on-by-default feature enabling runtime compilation of WebAssembly modules
@@ -108,3 +109,9 @@ posix-signals-on-macos = ["wasmtime-runtime/posix-signals-on-macos"]
 # Enabling this feature has no effect on unsupported platforms or when the
 # `uffd` feature is enabled.
 memory-init-cow = ["wasmtime-runtime/memory-init-cow"]
+
+# Enables runtime support necessary to capture backtraces of WebAssembly code
+# that is running.
+#
+# This is enabled by default.
+wasm-backtrace = ["wasmtime-runtime/wasm-backtrace", "backtrace"]

crates/wasmtime/src/config.rs

@@ -141,7 +141,9 @@ impl Config {
             ret.cranelift_debug_verifier(false);
             ret.cranelift_opt_level(OptLevel::Speed);
         }
+        #[cfg(feature = "wasm-backtrace")]
         ret.wasm_reference_types(true);
+        ret.features.reference_types = cfg!(feature = "wasm-backtrace");
         ret.wasm_multi_value(true);
         ret.wasm_bulk_memory(true);
         ret.wasm_simd(true);
@@ -502,10 +504,14 @@ impl Config {
     /// Note that enabling the reference types feature will also enable the bulk
     /// memory feature.
     ///
-    /// This is `true` by default on x86-64, and `false` by default on other
-    /// architectures.
+    /// This feature is `true` by default. If the `wasm-backtrace` feature is
+    /// disabled at compile time, however, then this is `false` by default and
+    /// it cannot be turned on since GC currently requires backtraces to work.
+    /// Note that the `wasm-backtrace` feature is on by default, however.
     ///
     /// [proposal]: https://github.com/webassembly/reference-types
+    #[cfg(feature = "wasm-backtrace")]
+    #[cfg_attr(nightlydoc, doc(cfg(feature = "wasm-backtrace")))]
     pub fn wasm_reference_types(&mut self, enable: bool) -> &mut Self {
         self.features.reference_types = enable;
 
@@ -1272,9 +1278,20 @@ impl Config {
 
 #[cfg(compiler)]
 fn compiler_builder(strategy: Strategy) -> Result<Box<dyn CompilerBuilder>> {
-    match strategy {
-        Strategy::Auto | Strategy::Cranelift => Ok(wasmtime_cranelift::builder()),
-    }
+    let mut builder = match strategy {
+        Strategy::Auto | Strategy::Cranelift => wasmtime_cranelift::builder(),
+    };
+    builder
+        .set(
+            "unwind_info",
+            if cfg!(feature = "wasm-backtrace") {
+                "true"
+            } else {
+                "false"
+            },
+        )
+        .unwrap();
+    Ok(builder)
 }
 
 fn round_up_to_pages(val: u64) -> u64 {

crates/wasmtime/src/engine.rs

@@ -304,7 +304,7 @@ impl Engine {
             // can affect the way the generated code performs or behaves at
             // runtime.
             "avoid_div_traps" => *value == FlagValue::Bool(true),
-            "unwind_info" => *value == FlagValue::Bool(true),
+            "unwind_info" => *value == FlagValue::Bool(cfg!(feature = "wasm-backtrace")),
             "libcall_call_conv" => *value == FlagValue::Enum("isa_default".into()),
 
             // Features wasmtime doesn't use should all be disabled, since

crates/wasmtime/src/lib.rs

@@ -290,6 +290,14 @@
 //!   run-time via [`Config::memory_init_cow`] (which is also enabled by
 //!   default).
 //!
+//! * `wasm-backtrace` - Enabled by default, this feature builds in support to
+//!   generate backtraces at runtime for WebAssembly modules. This means that
+//!   unwinding information is compiled into wasm modules and necessary runtime
+//!   dependencies are enabled as well. If this is turned off then some methods
+//!   to look at trap frames will not be available. Additionally at this time
+//!   disabling this feature means that the reference types feature is always
+//!   disabled as well.
+//!
 //! ## Examples
 //!
 //! In addition to the examples below be sure to check out the [online embedding

crates/wasmtime/src/trap.rs

@@ -1,10 +1,10 @@
 use crate::module::GlobalModuleRegistry;
 use crate::FrameInfo;
-use backtrace::Backtrace;
 use std::fmt;
 use std::sync::Arc;
 use wasmtime_environ::TrapCode as EnvTrapCode;
 use wasmtime_jit::{demangle_function_name, demangle_function_name_or_index};
+use wasmtime_runtime::Backtrace;
 
 /// A struct representing an aborted instruction execution, with a message
 /// indicating the cause.
@@ -129,8 +129,10 @@ impl fmt::Display for TrapCode {
 
 struct TrapInner {
     reason: TrapReason,
+    #[cfg(feature = "wasm-backtrace")]
     wasm_trace: Vec<FrameInfo>,
     native_trace: Backtrace,
+    #[cfg(feature = "wasm-backtrace")]
     hint_wasm_backtrace_details_env: bool,
 }
 
@@ -148,18 +150,14 @@ impl Trap {
     #[cold] // traps are exceptional, this helps move handling off the main path
     pub fn new<I: Into<String>>(message: I) -> Self {
         let reason = TrapReason::Message(message.into());
-        Trap::new_with_trace(None, reason, Backtrace::new_unresolved())
+        Trap::new_with_trace(None, reason, Backtrace::new())
     }
 
     /// Creates a new `Trap` representing an explicit program exit with a classic `i32`
     /// exit status value.
     #[cold] // see Trap::new
     pub fn i32_exit(status: i32) -> Self {
-        Trap::new_with_trace(
-            None,
-            TrapReason::I32Exit(status),
-            Backtrace::new_unresolved(),
-        )
+        Trap::new_with_trace(None, TrapReason::I32Exit(status), Backtrace::new())
     }
 
     #[cold] // see Trap::new
@@ -212,10 +210,12 @@ impl Trap {
     /// * `native_trace` - this is a captured backtrace from when the trap
     ///   occurred, and this will iterate over the frames to find frames that
     ///   lie in wasm jit code.
+    #[cfg_attr(not(feature = "wasm-backtrace"), allow(unused_mut, unused_variables))]
     fn new_with_trace(trap_pc: Option<usize>, reason: TrapReason, native_trace: Backtrace) -> Self {
-        let mut wasm_trace = Vec::new();
+        let mut wasm_trace = Vec::<FrameInfo>::new();
         let mut hint_wasm_backtrace_details_env = false;
 
+        #[cfg(feature = "wasm-backtrace")]
         GlobalModuleRegistry::with(|registry| {
             for frame in native_trace.frames() {
                 let pc = frame.ip() as usize;
@@ -253,8 +253,10 @@ impl Trap {
         Trap {
             inner: Arc::new(TrapInner {
                 reason,
-                wasm_trace,
                 native_trace,
+                #[cfg(feature = "wasm-backtrace")]
+                wasm_trace,
+                #[cfg(feature = "wasm-backtrace")]
                 hint_wasm_backtrace_details_env,
             }),
         }
@@ -281,6 +283,8 @@ impl Trap {
 
     /// Returns a list of function frames in WebAssembly code that led to this
     /// trap happening.
+    #[cfg(feature = "wasm-backtrace")]
+    #[cfg_attr(nightlydoc, doc(cfg(feature = "wasm-backtrace")))]
     pub fn trace(&self) -> &[FrameInfo] {
         &self.inner.wasm_trace
     }
@@ -297,65 +301,77 @@ impl Trap {
 
 impl fmt::Debug for Trap {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("Trap")
-            .field("reason", &self.inner.reason)
-            .field("wasm_trace", &self.inner.wasm_trace)
-            .field("native_trace", &self.inner.native_trace)
-            .finish()
+        let mut f = f.debug_struct("Trap");
+        f.field("reason", &self.inner.reason);
+        #[cfg(feature = "wasm-backtrace")]
+        {
+            f.field("wasm_trace", &self.inner.wasm_trace)
+                .field("native_trace", &self.inner.native_trace);
+        }
+        f.finish()
     }
 }
 
 impl fmt::Display for Trap {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}", self.inner.reason)?;
-        let trace = self.trace();
-        if trace.is_empty() {
-            return Ok(());
-        }
-        writeln!(f, "\nwasm backtrace:")?;
-        for (i, frame) in self.trace().iter().enumerate() {
-            let name = frame.module_name().unwrap_or("<unknown>");
-            write!(f, "  {:>3}: ", i)?;
 
-            if let Some(offset) = frame.module_offset() {
-                write!(f, "{:#6x} - ", offset)?;
+        #[cfg(feature = "wasm-backtrace")]
+        {
+            let trace = self.trace();
+            if trace.is_empty() {
+                return Ok(());
             }
+            writeln!(f, "\nwasm backtrace:")?;
 
-            let write_raw_func_name = |f: &mut fmt::Formatter<'_>| {
-                demangle_function_name_or_index(f, frame.func_name(), frame.func_index() as usize)
-            };
-            if frame.symbols().is_empty() {
-                write!(f, "{}!", name)?;
-                write_raw_func_name(f)?;
-                writeln!(f, "")?;
-            } else {
-                for (i, symbol) in frame.symbols().iter().enumerate() {
-                    if i > 0 {
-                        write!(f, "              - ")?;
-                    } else {
-                        // ...
-                    }
-                    match symbol.name() {
-                        Some(name) => demangle_function_name(f, name)?,
-                        None if i == 0 => write_raw_func_name(f)?,
-                        None => write!(f, "<inlined function>")?,
-                    }
+            for (i, frame) in self.trace().iter().enumerate() {
+                let name = frame.module_name().unwrap_or("<unknown>");
+                write!(f, "  {:>3}: ", i)?;
+
+                if let Some(offset) = frame.module_offset() {
+                    write!(f, "{:#6x} - ", offset)?;
+                }
+
+                let write_raw_func_name = |f: &mut fmt::Formatter<'_>| {
+                    demangle_function_name_or_index(
+                        f,
+                        frame.func_name(),
+                        frame.func_index() as usize,
+                    )
+                };
+                if frame.symbols().is_empty() {
+                    write!(f, "{}!", name)?;
+                    write_raw_func_name(f)?;
                     writeln!(f, "")?;
-                    if let Some(file) = symbol.file() {
-                        write!(f, "                    at {}", file)?;
-                        if let Some(line) = symbol.line() {
-                            write!(f, ":{}", line)?;
-                            if let Some(col) = symbol.column() {
-                                write!(f, ":{}", col)?;
+                } else {
+                    for (i, symbol) in frame.symbols().iter().enumerate() {
+                        if i > 0 {
+                            write!(f, "              - ")?;
+                        } else {
+                            // ...
+                        }
+                        match symbol.name() {
+                            Some(name) => demangle_function_name(f, name)?,
+                            None if i == 0 => write_raw_func_name(f)?,
+                            None => write!(f, "<inlined function>")?,
+                        }
+                        writeln!(f, "")?;
+                        if let Some(file) = symbol.file() {
+                            write!(f, "                    at {}", file)?;
+                            if let Some(line) = symbol.line() {
+                                write!(f, ":{}", line)?;
+                                if let Some(col) = symbol.column() {
+                                    write!(f, ":{}", col)?;
+                                }
                             }
                         }
+                        writeln!(f, "")?;
                     }
-                    writeln!(f, "")?;
                 }
             }
-        }
-        if self.inner.hint_wasm_backtrace_details_env {
-            writeln!(f, "note: using the `WASMTIME_BACKTRACE_DETAILS=1` environment variable to may show more debugging information")?;
+            if self.inner.hint_wasm_backtrace_details_env {
+                writeln!(f, "note: using the `WASMTIME_BACKTRACE_DETAILS=1` environment variable to may show more debugging information")?;
+            }
         }
         Ok(())
     }
@@ -388,7 +404,7 @@ impl From<Box<dyn std::error::Error + Send + Sync>> for Trap {
             trap.clone()
         } else {
             let reason = TrapReason::Error(e.into());
-            Trap::new_with_trace(None, reason, Backtrace::new_unresolved())
+            Trap::new_with_trace(None, reason, Backtrace::new())
         }
     }
 }

src/lib.rs

@@ -384,7 +384,9 @@ impl CommonOptions {
             config.wasm_bulk_memory(enable);
         }
         if let Some(enable) = reference_types {
+            #[cfg(feature = "wasm-backtrace")]
             config.wasm_reference_types(enable);
+            drop(enable); // suppress unused warnings
         }
         if let Some(enable) = multi_value {
             config.wasm_multi_value(enable);