diff --git a/crates/api/src/callable.rs b/crates/api/src/callable.rs index 2e59b39e12..601b0ed763 100644 --- a/crates/api/src/callable.rs +++ b/crates/api/src/callable.rs @@ -160,7 +160,7 @@ impl WrappedCallable for WasmtimeFn { let exec_code_buf = self .store .compiler_mut() - .get_published_trampoline(body, &signature, value_size) + .get_published_trampoline(&signature, value_size) .map_err(|e| Trap::new(format!("trampoline error: {:?}", e)))?; // Call the trampoline. @@ -169,6 +169,7 @@ impl WrappedCallable for WasmtimeFn { vmctx, ptr::null_mut(), exec_code_buf, + body, values_vec.as_mut_ptr() as *mut u8, ) } { diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index 21a274eb23..0a01749838 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -22,6 +22,7 @@ use wasmtime_environ::{ }; use wasmtime_runtime::{ InstantiationError, SignatureRegistry, TrapRegistration, TrapRegistry, VMFunctionBody, + VMSharedSignatureIndex, }; /// Select which kind of compilation to use. @@ -51,7 +52,7 @@ pub struct Compiler { code_memory: CodeMemory, trap_registry: TrapRegistry, - trampoline_park: HashMap<*const VMFunctionBody, *const VMFunctionBody>, + trampoline_park: HashMap, signatures: SignatureRegistry, strategy: CompilationStrategy, cache_config: CacheConfig, @@ -200,37 +201,31 @@ impl Compiler { /// Create a trampoline for invoking a function. pub(crate) fn get_trampoline( &mut self, - callee_address: *const VMFunctionBody, signature: &ir::Signature, value_size: usize, ) -> Result<*const VMFunctionBody, SetupError> { - use std::collections::hash_map::Entry::{Occupied, Vacant}; - Ok(match self.trampoline_park.entry(callee_address) { - Occupied(entry) => *entry.get(), - Vacant(entry) => { - let body = make_trampoline( - &*self.isa, - &mut self.code_memory, - &mut self.fn_builder_ctx, - callee_address, - signature, - value_size, - )?; - - entry.insert(body); - body - } - }) + let index = self.signatures.register(signature); + if let Some(trampoline) = self.trampoline_park.get(&index) { + return Ok(*trampoline); + } + let body = make_trampoline( + &*self.isa, + &mut self.code_memory, + &mut self.fn_builder_ctx, + signature, + value_size, + )?; + self.trampoline_park.insert(index, body); + return Ok(body); } /// Create and publish a trampoline for invoking a function. pub fn get_published_trampoline( &mut self, - callee_address: *const VMFunctionBody, signature: &ir::Signature, value_size: usize, ) -> Result<*const VMFunctionBody, SetupError> { - let result = self.get_trampoline(callee_address, signature, value_size)?; + let result = self.get_trampoline(signature, value_size)?; self.publish_compiled_code(); Ok(result) } @@ -256,7 +251,6 @@ fn make_trampoline( isa: &dyn TargetIsa, code_memory: &mut CodeMemory, fn_builder_ctx: &mut FunctionBuilderContext, - callee_address: *const VMFunctionBody, signature: &ir::Signature, value_size: usize, ) -> Result<*const VMFunctionBody, SetupError> { @@ -272,6 +266,9 @@ fn make_trampoline( // Add the caller `vmctx` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); + // Add the `callee_address` parameter. + wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); + // Add the `values_vec` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); @@ -287,9 +284,9 @@ fn make_trampoline( builder.switch_to_block(block0); builder.seal_block(block0); - let (vmctx_ptr_val, caller_vmctx_ptr_val, values_vec_ptr_val) = { + let (vmctx_ptr_val, caller_vmctx_ptr_val, callee_value, values_vec_ptr_val) = { let params = builder.func.dfg.block_params(block0); - (params[0], params[1], params[2]) + (params[0], params[1], params[2], params[3]) }; // Load the argument values out of `values_vec`. @@ -318,10 +315,6 @@ fn make_trampoline( let new_sig = builder.import_signature(signature.clone()); - // TODO: It's possible to make this a direct call. We just need Cranelift - // to support functions declared with an immediate integer address. - // ExternalName::Absolute(u64). Let's do it. - let callee_value = builder.ins().iconst(pointer_type, callee_address as i64); let call = builder .ins() .call_indirect(new_sig, callee_value, &callee_args); diff --git a/crates/runtime/signalhandlers/Trampolines.cpp b/crates/runtime/signalhandlers/Trampolines.cpp index e554823652..c76db87fd2 100644 --- a/crates/runtime/signalhandlers/Trampolines.cpp +++ b/crates/runtime/signalhandlers/Trampolines.cpp @@ -7,7 +7,8 @@ int WasmtimeCallTrampoline( void **buf_storage, void *vmctx, void *caller_vmctx, - void (*body)(void*, void*, void*), + void (*trampoline)(void*, void*, void*, void*), + void *body, void *args) { jmp_buf buf; @@ -15,12 +16,16 @@ int WasmtimeCallTrampoline( return 0; } *buf_storage = &buf; - body(vmctx, caller_vmctx, args); + trampoline(vmctx, caller_vmctx, body, args); return 1; } extern "C" -int WasmtimeCall(void **buf_storage, void *vmctx, void *caller_vmctx, void (*body)(void*, void*)) { +int WasmtimeCall( + void **buf_storage, + void *vmctx, + void *caller_vmctx, + void (*body)(void*, void*)) { jmp_buf buf; if (setjmp(buf) != 0) { return 0; diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 6137c1aef9..c1195ffa21 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -347,10 +347,14 @@ impl Instance { &*self.host_state } - fn invoke_function(&self, index: FuncIndex) -> Result<(), InstantiationError> { - // TODO: Check that the callee's calling convention matches what we expect. + /// Invoke the WebAssembly start function of the instance, if one is present. + fn invoke_start_function(&self) -> Result<(), InstantiationError> { + let start_index = match self.module.start_func { + Some(idx) => idx, + None => return Ok(()), + }; - let (callee_address, callee_vmctx) = match self.module.defined_func_index(index) { + let (callee_address, callee_vmctx) = match self.module.defined_func_index(start_index) { Some(defined_index) => { let body = *self .finished_functions @@ -359,8 +363,8 @@ impl Instance { (body as *const _, self.vmctx_ptr()) } None => { - assert_lt!(index.index(), self.module.imported_funcs.len()); - let import = self.imported_function(index); + assert_lt!(start_index.index(), self.module.imported_funcs.len()); + let import = self.imported_function(start_index); (import.body, import.vmctx) } }; @@ -370,15 +374,6 @@ impl Instance { .map_err(InstantiationError::StartTrap) } - /// Invoke the WebAssembly start function of the instance, if one is present. - fn invoke_start_function(&self) -> Result<(), InstantiationError> { - if let Some(start_index) = self.module.start_func { - self.invoke_function(start_index) - } else { - Ok(()) - } - } - /// Return the offset from the vmctx pointer to its containing Instance. pub(crate) fn vmctx_offset() -> isize { offset_of!(Self, vmctx) as isize diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 0a2b91b6e6..c76a37dc97 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -17,6 +17,7 @@ extern "C" { jmp_buf: *mut *const u8, vmctx: *mut u8, caller_vmctx: *mut u8, + trampoline: *const VMFunctionBody, callee: *const VMFunctionBody, values_vec: *mut u8, ) -> i32; @@ -133,13 +134,23 @@ impl fmt::Display for Trap { impl std::error::Error for Trap {} -/// Call the wasm function pointed to by `callee`. `values_vec` points to -/// a buffer which holds the incoming arguments, and to which the outgoing -/// return values will be written. -#[no_mangle] -pub unsafe extern "C" fn wasmtime_call_trampoline( +/// Call the wasm function pointed to by `callee`. +/// +/// * `vmctx` - the callee vmctx argument +/// * `caller_vmctx` - the caller vmctx argument +/// * `trampoline` - the jit-generated trampoline whose ABI takes 4 values, the +/// callee vmctx, the caller vmctx, the `callee` argument below, and then the +/// `values_vec` argument. +/// * `callee` - the third argument to the `trampoline` function +/// * `values_vec` - points to a buffer which holds the incoming arguments, and to +/// which the outgoing return values will be written. +/// +/// Wildly unsafe because it calls raw function pointers and reads/writes raw +/// function pointers. +pub unsafe fn wasmtime_call_trampoline( vmctx: *mut VMContext, caller_vmctx: *mut VMContext, + trampoline: *const VMFunctionBody, callee: *const VMFunctionBody, values_vec: *mut u8, ) -> Result<(), Trap> { @@ -148,6 +159,7 @@ pub unsafe extern "C" fn wasmtime_call_trampoline( cx.jmp_buf.as_ptr(), vmctx as *mut u8, caller_vmctx as *mut u8, + trampoline, callee, values_vec, ) @@ -156,8 +168,7 @@ pub unsafe extern "C" fn wasmtime_call_trampoline( /// Call the wasm function pointed to by `callee`, which has no arguments or /// return values. -#[no_mangle] -pub unsafe extern "C" fn wasmtime_call( +pub unsafe fn wasmtime_call( vmctx: *mut VMContext, caller_vmctx: *mut VMContext, callee: *const VMFunctionBody,