Exit through Cranelift-generated trampolines for builtins (#8152)

* Exit through Cranelift-generated trampolines for builtins

This commit changes how builtin functions in Wasmtime (think
`memory.grow`) are implemented. These functions are required to exit
through some manner of trampoline to handle runtime requirements for
backtracing right now. Currently this is done via inline assembly for
each architecture (or external assembly for s390x). This is a bit
unfortunate as it's a lot of hand-coding and making sure everything is
right, and it's not easy to update as it's multiple platforms to update.

The change in this commit is to instead use Cranelift-generated
trampolines for this purpose instead. The path for invoking a builtin
function now looks like:

* Wasm code calls a statically known symbol for each builtin.
* The statically known symbol will perform exit trampoline duties (e.g.
  pc/fp/etc) and then load a function pointer to the host
  implementation.
* The host implementation is invoked and then proceeds as usual.

The main new piece for this PR is that all wasm modules and functions
are compiled in parallel but an output of this compilation phase is what
builtin functions are required. All builtin functions are then unioned
together into one set and then anything required is generated just
afterwards. That means that only one builtin-trampoline per-module is
generated per-builtin.

This work is inspired by #8135 and my own personal desire to have as
much about our ABI details flowing through Cranelift as we can. This in
theory makes it more flexible to deal with future improvements to our
ABI.

prtest:full

* Fix some build issues

* Update winch test expectations

* Update Winch to use new builtin shims.

This commit refactors the Winch compiler to use the new trampolines for
all Wasmtime builtins created in the previous commits. This required a
fair bit of refactoring to handle plumbing through a new kind of
relocation and function call.

Winch's `FuncEnv` now contains a `PrimaryMap` from `UserExternalNameRef`
to `UserExternalName`. This is because there's now more than one kind of
name than just wasm function relocations, so the raw index space of
`UserExternalNameRef` is no longer applicable. This required threading
`FuncEnv` to more locations along with some refactorings to ensure that
lifetimes work out ok.

The `CompiledFunction` no longer stores a trait object of how to map
name refs to names and now directly has a `Primarymap`. This also means
that Winch's return value from its `TargetIsa` is a `CompiledFunction`
as opposed to the previous just-a-`MachBuffer` so it can also package up
all the relocation information. This ends up having `winch-codegen`
depend on `wasmtime-cranelift-shared` as a new dependency.

* Review feedback

Cargo.lock

@@ -4064,6 +4064,7 @@ dependencies = [
  "smallvec",
  "target-lexicon",
  "wasmparser",
+ "wasmtime-cranelift",
  "wasmtime-environ",
 ]
 

crates/cranelift/src/compiled_function.rs

@@ -1,17 +1,9 @@
-use std::any::Any;
-
 use crate::{mach_reloc_to_reloc, mach_trap_to_trap, Relocation};
 use cranelift_codegen::{
-    ir, ir::UserExternalNameRef, isa::unwind::CfaUnwindInfo, isa::unwind::UnwindInfo, Final,
-    MachBufferFinalized, MachSrcLoc, ValueLabelsRanges,
+    ir, isa::unwind::CfaUnwindInfo, isa::unwind::UnwindInfo, Final, MachBufferFinalized,
+    MachSrcLoc, ValueLabelsRanges,
 };
-use wasmtime_environ::{FilePos, InstructionAddressMap, TrapInformation};
-
-/// Trait used in the [CompiledFunction] to resolve the locations of
-/// external name references in a compiled function.
-pub trait CompiledFuncEnv {
-    fn resolve_user_external_name_ref(&self, external: UserExternalNameRef) -> (u32, u32);
-}
+use wasmtime_environ::{FilePos, InstructionAddressMap, PrimaryMap, TrapInformation};
 
 #[derive(Debug, Clone, PartialEq, Eq, Default)]
 /// Metadata to translate from binary offsets back to the original
@@ -64,8 +56,8 @@ pub struct CompiledFunctionMetadata {
 pub struct CompiledFunction {
     /// The machine code buffer for this function.
     pub buffer: MachBufferFinalized<Final>,
-    /// The environment for the compiled function.
-    env: Box<dyn CompiledFuncEnv + Send + 'static>,
+    /// What names each name ref corresponds to.
+    name_map: PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
     /// The alignment for the compiled function.
     pub alignment: u32,
     /// The metadata for the compiled function, including unwind information
@@ -77,13 +69,14 @@ impl CompiledFunction {
     /// Creates a [CompiledFunction] from a [cranelift_codegen::MachBufferFinalized<Final>]
     /// This function uses the information in the machine buffer to derive the traps and relocations
     /// fields. The compiled function metadata is loaded with the default values.
-    pub fn new<E>(buffer: MachBufferFinalized<Final>, env: E, alignment: u32) -> Self
-    where
-        E: Any + CompiledFuncEnv + Send + 'static,
-    {
+    pub fn new(
+        buffer: MachBufferFinalized<Final>,
+        name_map: PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
+        alignment: u32,
+    ) -> Self {
         Self {
             buffer,
-            env: Box::new(env),
+            name_map,
             alignment,
             metadata: Default::default(),
         }
@@ -91,11 +84,10 @@ impl CompiledFunction {
 
     /// Returns an iterator to the function's relocation information.
     pub fn relocations(&self) -> impl Iterator<Item = Relocation> + '_ {
-        self.buffer.relocs().iter().map(|r| {
-            mach_reloc_to_reloc(r, |external| {
-                self.env.resolve_user_external_name_ref(external)
-            })
-        })
+        self.buffer
+            .relocs()
+            .iter()
+            .map(|r| mach_reloc_to_reloc(r, &self.name_map))
     }
 
     /// Returns an iterator to the function's trap information.

crates/cranelift/src/compiler.rs

@@ -1,12 +1,10 @@
 use crate::debug::{DwarfSectionRelocTarget, ModuleMemoryOffset};
 use crate::func_environ::FuncEnvironment;
 use crate::{array_call_signature, native_call_signature, DEBUG_ASSERT_TRAP_CODE};
-use crate::{builder::LinkOptions, value_type, wasm_call_signature};
+use crate::{builder::LinkOptions, value_type, wasm_call_signature, BuiltinFunctionSignatures};
 use crate::{CompiledFunction, ModuleTextBuilder};
 use anyhow::{Context as _, Result};
-use cranelift_codegen::ir::{
-    self, InstBuilder, MemFlags, UserExternalName, UserExternalNameRef, UserFuncName, Value,
-};
+use cranelift_codegen::ir::{self, InstBuilder, MemFlags, UserExternalName, UserFuncName, Value};
 use cranelift_codegen::isa::{
     unwind::{UnwindInfo, UnwindInfoKind},
     OwnedTargetIsa, TargetIsa,
@@ -17,8 +15,7 @@ use cranelift_codegen::{CompiledCode, MachStackMap};
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_frontend::FunctionBuilder;
 use cranelift_wasm::{
-    DefinedFuncIndex, FuncIndex, FuncTranslator, MemoryIndex, OwnedMemoryIndex, WasmFuncType,
-    WasmValType,
+    DefinedFuncIndex, FuncTranslator, MemoryIndex, OwnedMemoryIndex, WasmFuncType, WasmValType,
 };
 use object::write::{Object, StandardSegment, SymbolId};
 use object::{RelocationEncoding, RelocationFlags, RelocationKind, SectionKind};
@@ -30,9 +27,9 @@ use std::path;
 use std::sync::{Arc, Mutex};
 use wasmparser::{FuncValidatorAllocations, FunctionBody};
 use wasmtime_environ::{
-    AddressMapSection, CacheStore, CompileError, FlagValue, FunctionBodyData, FunctionLoc,
-    ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapInformation, TrapEncodingBuilder,
-    Tunables, VMOffsets, WasmFunctionInfo,
+    AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, FlagValue, FunctionBodyData,
+    FunctionLoc, ModuleTranslation, ModuleTypesBuilder, PtrSize, RelocationTarget,
+    StackMapInformation, TrapEncodingBuilder, Tunables, VMOffsets, WasmFunctionInfo,
 };
 
 #[cfg(feature = "component-model")]
@@ -143,7 +140,7 @@ impl wasmtime_environ::Compiler for Compiler {
         let context = &mut compiler.cx.codegen_context;
         context.func.signature = wasm_call_signature(isa, wasm_func_ty, &self.tunables);
         context.func.name = UserFuncName::User(UserExternalName {
-            namespace: 0,
+            namespace: crate::NS_WASM_FUNC,
             index: func_index.as_u32(),
         });
 
@@ -448,7 +445,7 @@ impl wasmtime_environ::Compiler for Compiler {
         &self,
         obj: &mut Object<'static>,
         funcs: &[(String, Box<dyn Any + Send>)],
-        resolve_reloc: &dyn Fn(usize, FuncIndex) -> usize,
+        resolve_reloc: &dyn Fn(usize, RelocationTarget) -> usize,
     ) -> Result<Vec<(SymbolId, FunctionLoc)>> {
         let mut builder =
             ModuleTextBuilder::new(obj, self, self.isa.text_section_builder(funcs.len()));
@@ -629,6 +626,68 @@ impl wasmtime_environ::Compiler for Compiler {
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         self.isa.create_systemv_cie()
     }
+
+    fn compile_wasm_to_builtin(
+        &self,
+        index: BuiltinFunctionIndex,
+    ) -> Result<Box<dyn Any + Send>, CompileError> {
+        let isa = &*self.isa;
+        let ptr_size = isa.pointer_bytes();
+        let pointer_type = isa.pointer_type();
+        let sig = BuiltinFunctionSignatures::new(isa).signature(index);
+
+        let mut compiler = self.function_compiler();
+        let func = ir::Function::with_name_signature(Default::default(), sig.clone());
+        let (mut builder, block0) = compiler.builder(func);
+        let vmctx = builder.block_params(block0)[0];
+
+        // Debug-assert that this is the right kind of vmctx, and then
+        // additionally perform the "routine of the exit trampoline" of saving
+        // fp/pc/etc.
+        debug_assert_vmctx_kind(isa, &mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC);
+        let limits = builder.ins().load(
+            pointer_type,
+            MemFlags::trusted(),
+            vmctx,
+            ptr_size.vmcontext_runtime_limits(),
+        );
+        save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, limits);
+
+        // Now it's time to delegate to the actual builtin. Builtins are stored
+        // in an array in all `VMContext`s. First load the base pointer of the
+        // array and then load the entry of the array that correspons to this
+        // builtin.
+        let mem_flags = ir::MemFlags::trusted().with_readonly();
+        let array_addr = builder.ins().load(
+            pointer_type,
+            mem_flags,
+            vmctx,
+            i32::try_from(ptr_size.vmcontext_builtin_functions()).unwrap(),
+        );
+        let body_offset = i32::try_from(index.index() * pointer_type.bytes()).unwrap();
+        let func_addr = builder
+            .ins()
+            .load(pointer_type, mem_flags, array_addr, body_offset);
+
+        // Forward all our own arguments to the libcall itself, and then return
+        // all the same results as the libcall.
+        let block_params = builder.block_params(block0).to_vec();
+        let sig = builder.func.import_signature(sig);
+        let call = builder.ins().call_indirect(sig, func_addr, &block_params);
+        let results = builder.func.dfg.inst_results(call).to_vec();
+        builder.ins().return_(&results);
+        builder.finalize();
+
+        Ok(Box::new(compiler.finish()?))
+    }
+
+    fn compiled_function_relocation_targets<'a>(
+        &'a self,
+        func: &'a dyn Any,
+    ) -> Box<dyn Iterator<Item = RelocationTarget> + 'a> {
+        let func = func.downcast_ref::<CompiledFunction>().unwrap();
+        Box::new(func.relocations().map(|r| r.reloc_target))
+    }
 }
 
 #[cfg(feature = "incremental-cache")]
@@ -976,20 +1035,6 @@ impl Compiler {
     }
 }
 
-/// The compiled function environment.
-pub struct CompiledFuncEnv {
-    /// Map to resolve external name references.
-    map: PrimaryMap<UserExternalNameRef, UserExternalName>,
-}
-
-impl crate::CompiledFuncEnv for CompiledFuncEnv {
-    fn resolve_user_external_name_ref(&self, external: ir::UserExternalNameRef) -> (u32, u32) {
-        let UserExternalName { index, namespace } = self.map[external];
-
-        (namespace, index)
-    }
-}
-
 struct FunctionCompiler<'a> {
     compiler: &'a Compiler,
     cx: CompilerContext,
@@ -1036,11 +1081,11 @@ impl FunctionCompiler<'_> {
         };
 
         let alignment = compiled_code.buffer.alignment.max(preferred_alignment);
-        let env = CompiledFuncEnv {
-            map: context.func.params.user_named_funcs().clone(),
-        };
-        let mut compiled_function =
-            CompiledFunction::new(compiled_code.buffer.clone(), env, alignment);
+        let mut compiled_function = CompiledFunction::new(
+            compiled_code.buffer.clone(),
+            context.func.params.user_named_funcs().clone(),
+            alignment,
+        );
 
         if let Some((body, tunables)) = body_and_tunables {
             let data = body.get_binary_reader();
@@ -1131,7 +1176,7 @@ fn declare_and_call(
 ) -> ir::Inst {
     let name = ir::ExternalName::User(builder.func.declare_imported_user_function(
         ir::UserExternalName {
-            namespace: 0,
+            namespace: crate::NS_WASM_FUNC,
             index: func_index,
         },
     ));

crates/cranelift/src/func_environ.rs

@@ -1,13 +1,12 @@
+use crate::BuiltinFunctionSignatures;
 use cranelift_codegen::cursor::FuncCursor;
 use cranelift_codegen::ir;
 use cranelift_codegen::ir::condcodes::*;
 use cranelift_codegen::ir::immediates::{Imm64, Offset32};
 use cranelift_codegen::ir::pcc::Fact;
 use cranelift_codegen::ir::types::*;
-use cranelift_codegen::ir::{
-    AbiParam, ArgumentPurpose, Function, InstBuilder, MemFlags, Signature,
-};
-use cranelift_codegen::isa::{self, CallConv, TargetFrontendConfig, TargetIsa};
+use cranelift_codegen::ir::{ArgumentPurpose, Function, InstBuilder, MemFlags};
+use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa};
 use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap};
 use cranelift_frontend::FunctionBuilder;
 use cranelift_frontend::Variable;
@@ -24,103 +23,59 @@ use wasmtime_environ::{
 };
 use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
 
-macro_rules! declare_function_signatures {
-    (
-        $(
-            $( #[$attr:meta] )*
-            $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
-        )*
-    ) => {
-        /// A struct with an `Option<ir::SigRef>` member for every builtin
-        /// function, to de-duplicate constructing/getting its signature.
-        #[allow(unused_doc_comments)]
-        struct BuiltinFunctionSignatures {
-            pointer_type: ir::Type,
+/// A struct with an `Option<ir::FuncRef>` member for every builtin
+/// function, to de-duplicate constructing/getting its function.
+struct BuiltinFunctions {
+    types: BuiltinFunctionSignatures,
 
-            #[cfg(feature = "gc")]
-            reference_type: ir::Type,
-
-            call_conv: isa::CallConv,
+    builtins:
+        [Option<ir::FuncRef>; BuiltinFunctionIndex::builtin_functions_total_number() as usize],
+}
 
-            $(
-                $( #[$attr] )*
-                $name: Option<ir::SigRef>,
-            )*
+impl BuiltinFunctions {
+    fn new(isa: &dyn TargetIsa) -> Self {
+        Self {
+            types: BuiltinFunctionSignatures::new(isa),
+            builtins: [None; BuiltinFunctionIndex::builtin_functions_total_number() as usize],
         }
+    }
 
-        #[allow(unused_doc_comments)]
-        impl BuiltinFunctionSignatures {
-            fn new(
-                pointer_type: ir::Type,
-                reference_type: ir::Type,
-                call_conv: isa::CallConv,
-            ) -> Self {
-                #[cfg(not(feature = "gc"))]
-                let _ = reference_type;
-
-                Self {
-                    pointer_type,
-
-                    #[cfg(feature = "gc")]
-                    reference_type,
-
-                    call_conv,
-
-                    $(
-                        $( #[$attr] )*
-                        $name: None,
-                    )*
-                }
-            }
-
-            fn vmctx(&self) -> AbiParam {
-                AbiParam::special(self.pointer_type, ArgumentPurpose::VMContext)
-            }
-
-            #[cfg(feature = "gc")]
-            fn reference(&self) -> AbiParam {
-                AbiParam::new(self.reference_type)
-            }
-
-            fn pointer(&self) -> AbiParam {
-                AbiParam::new(self.pointer_type)
-            }
-
-            fn i32(&self) -> AbiParam {
-                // Some platform ABIs require i32 values to be zero- or sign-
-                // extended to the full register width.  We need to indicate
-                // this here by using the appropriate .uext or .sext attribute.
-                // The attribute can be added unconditionally; platforms whose
-                // ABI does not require such extensions will simply ignore it.
-                // Note that currently all i32 arguments or return values used
-                // by builtin functions are unsigned, so we always use .uext.
-                // If that ever changes, we will have to add a second type
-                // marker here.
-                AbiParam::new(I32).uext()
-            }
+    fn load_builtin(&mut self, func: &mut Function, index: BuiltinFunctionIndex) -> ir::FuncRef {
+        let cache = &mut self.builtins[index.index() as usize];
+        if let Some(f) = cache {
+            return *f;
+        }
+        let signature = func.import_signature(self.types.signature(index));
+        let name =
+            ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName {
+                namespace: crate::NS_WASMTIME_BUILTIN,
+                index: index.index(),
+            }));
+        let f = func.import_function(ir::ExtFuncData {
+            name,
+            signature,
+            colocated: true,
+        });
+        *cache = Some(f);
+        f
+    }
+}
 
-            fn i64(&self) -> AbiParam {
-                AbiParam::new(I64)
+// Generate helper methods on `BuiltinFunctions` above for each named builtin
+// as well.
+macro_rules! declare_function_signatures {
+    ($(
+        $( #[$attr:meta] )*
+        $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
+    )*) => {
+        $(impl BuiltinFunctions {
+            $( #[$attr] )*
+            fn $name(&mut self, func: &mut Function) -> ir::FuncRef {
+                self.load_builtin(func, BuiltinFunctionIndex::$name())
             }
-
-            $(
-                $( #[$attr] )*
-                fn $name(&mut self, func: &mut Function) -> ir::SigRef {
-                    let sig = self.$name.unwrap_or_else(|| {
-                        func.import_signature(Signature {
-                            params: vec![ $( self.$param() ),* ],
-                            returns: vec![ $( self.$result() )? ],
-                            call_conv: self.call_conv,
-                        })
-                    });
-                    self.$name = Some(sig);
-                    sig
-                }
-            )*
-        }
+        })*
     };
 }
-
 wasmtime_environ::foreach_builtin_function!(declare_function_signatures);
 
 /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`.
@@ -146,7 +101,7 @@ pub struct FuncEnvironment<'module_environment> {
     pcc_vmctx_memtype: Option<ir::MemoryType>,
 
     /// Caches of signatures for builtin functions.
-    builtin_function_signatures: BuiltinFunctionSignatures,
+    builtin_functions: BuiltinFunctions,
 
     /// Offsets to struct fields accessed by JIT code.
     pub(crate) offsets: VMOffsets<u8>,
@@ -192,15 +147,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         tunables: &'module_environment Tunables,
         wmemcheck: bool,
     ) -> Self {
-        let builtin_function_signatures = BuiltinFunctionSignatures::new(
-            isa.pointer_type(),
-            match isa.pointer_type() {
-                ir::types::I32 => ir::types::R32,
-                ir::types::I64 => ir::types::R64,
-                _ => panic!(),
-            },
-            CallConv::triple_default(isa.triple()),
-        );
+        let builtin_functions = BuiltinFunctions::new(isa);
 
         // Avoid unused warning in default build.
         #[cfg(not(feature = "wmemcheck"))]
@@ -214,7 +161,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             tables: SecondaryMap::default(),
             vmctx: None,
             pcc_vmctx_memtype: None,
-            builtin_function_signatures,
+            builtin_functions,
             offsets: VMOffsets::new(isa.pointer_bytes(), &translation.module),
             tunables,
             fuel_var: Variable::new(0),
@@ -262,98 +209,45 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         })
     }
 
+    fn vmctx_val(&mut self, pos: &mut FuncCursor<'_>) -> ir::Value {
+        let pointer_type = self.pointer_type();
+        let vmctx = self.vmctx(&mut pos.func);
+        pos.ins().global_value(pointer_type, vmctx)
+    }
+
     fn get_table_copy_func(
         &mut self,
         func: &mut Function,
         dst_table_index: TableIndex,
         src_table_index: TableIndex,
-    ) -> (ir::SigRef, usize, usize, BuiltinFunctionIndex) {
-        let sig = self.builtin_function_signatures.table_copy(func);
+    ) -> (ir::FuncRef, usize, usize) {
+        let sig = self.builtin_functions.table_copy(func);
         (
             sig,
             dst_table_index.as_u32() as usize,
             src_table_index.as_u32() as usize,
-            BuiltinFunctionIndex::table_copy(),
         )
     }
 
-    fn get_table_init_func(
-        &mut self,
-        func: &mut Function,
-        table_index: TableIndex,
-    ) -> (ir::SigRef, usize, BuiltinFunctionIndex) {
-        let sig = self.builtin_function_signatures.table_init(func);
-        let table_index = table_index.as_u32() as usize;
-        (sig, table_index, BuiltinFunctionIndex::table_init())
-    }
-
-    fn get_elem_drop_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) {
-        let sig = self.builtin_function_signatures.elem_drop(func);
-        (sig, BuiltinFunctionIndex::elem_drop())
-    }
-
     fn get_memory_atomic_wait(
         &mut self,
         func: &mut Function,
         memory_index: MemoryIndex,
         ty: ir::Type,
-    ) -> (ir::SigRef, usize, BuiltinFunctionIndex) {
+    ) -> (ir::FuncRef, usize) {
         match ty {
             I32 => (
-                self.builtin_function_signatures.memory_atomic_wait32(func),
+                self.builtin_functions.memory_atomic_wait32(func),
                 memory_index.index(),
-                BuiltinFunctionIndex::memory_atomic_wait32(),
             ),
             I64 => (
-                self.builtin_function_signatures.memory_atomic_wait64(func),
+                self.builtin_functions.memory_atomic_wait64(func),
                 memory_index.index(),
-                BuiltinFunctionIndex::memory_atomic_wait64(),
             ),
             x => panic!("get_memory_atomic_wait unsupported type: {:?}", x),
         }
     }
 
-    fn get_memory_init_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) {
-        (
-            self.builtin_function_signatures.memory_init(func),
-            BuiltinFunctionIndex::memory_init(),
-        )
-    }
-
-    fn get_data_drop_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) {
-        (
-            self.builtin_function_signatures.data_drop(func),
-            BuiltinFunctionIndex::data_drop(),
-        )
-    }
-
-    /// Translates load of builtin function and returns a pair of values `vmctx`
-    /// and address of the loaded function.
-    fn translate_load_builtin_function_address(
-        &mut self,
-        pos: &mut FuncCursor<'_>,
-        callee_func_idx: BuiltinFunctionIndex,
-    ) -> (ir::Value, ir::Value) {
-        // We use an indirect call so that we don't have to patch the code at runtime.
-        let pointer_type = self.pointer_type();
-        let vmctx = self.vmctx(&mut pos.func);
-        let base = pos.ins().global_value(pointer_type, vmctx);
-
-        let mem_flags = ir::MemFlags::trusted().with_readonly();
-
-        // Load the base of the array of builtin functions
-        let array_offset = i32::try_from(self.offsets.vmctx_builtin_functions()).unwrap();
-        let array_addr = pos.ins().load(pointer_type, mem_flags, base, array_offset);
-
-        // Load the callee address.
-        let body_offset = i32::try_from(callee_func_idx.index() * pointer_type.bytes()).unwrap();
-        let func_addr = pos
-            .ins()
-            .load(pointer_type, mem_flags, array_addr, body_offset);
-
-        (base, func_addr)
-    }
-
     /// Generate code to increment or decrement the given `externref`'s
     /// reference count.
     ///
@@ -621,14 +515,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         // intrinsic may alter how much fuel is in the system.
         builder.switch_to_block(out_of_gas_block);
         self.fuel_save_from_var(builder);
-        let out_of_gas_sig = self.builtin_function_signatures.out_of_gas(builder.func);
-        let (vmctx, out_of_gas) = self.translate_load_builtin_function_address(
-            &mut builder.cursor(),
-            BuiltinFunctionIndex::out_of_gas(),
-        );
-        builder
-            .ins()
-            .call_indirect(out_of_gas_sig, out_of_gas, &[vmctx]);
+        let out_of_gas = self.builtin_functions.out_of_gas(builder.func);
+        let vmctx = self.vmctx_val(&mut builder.cursor());
+        builder.ins().call(out_of_gas, &[vmctx]);
         self.fuel_load_into_var(builder);
         builder.ins().jump(continuation_block, &[]);
         builder.seal_block(continuation_block);
@@ -665,7 +554,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
 
     #[cfg(feature = "wmemcheck")]
     fn hook_malloc_exit(&mut self, builder: &mut FunctionBuilder, retvals: &[Value]) {
-        let check_malloc_sig = self.builtin_function_signatures.check_malloc(builder.func);
+        let check_malloc_sig = self.builtin_functions.check_malloc(builder.func);
         let (vmctx, check_malloc) = self.translate_load_builtin_function_address(
             &mut builder.cursor(),
             BuiltinFunctionIndex::check_malloc(),
@@ -693,7 +582,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
 
     #[cfg(feature = "wmemcheck")]
     fn hook_free_exit(&mut self, builder: &mut FunctionBuilder) {
-        let check_free_sig = self.builtin_function_signatures.check_free(builder.func);
+        let check_free_sig = self.builtin_functions.check_free(builder.func);
         let (vmctx, check_free) = self.translate_load_builtin_function_address(
             &mut builder.cursor(),
             BuiltinFunctionIndex::check_free(),
@@ -798,16 +687,11 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         builder.seal_block(new_epoch_doublecheck_block);
 
         builder.switch_to_block(new_epoch_doublecheck_block);
-        let new_epoch_sig = self.builtin_function_signatures.new_epoch(builder.func);
-        let (vmctx, new_epoch) = self.translate_load_builtin_function_address(
-            &mut builder.cursor(),
-            BuiltinFunctionIndex::new_epoch(),
-        );
+        let new_epoch = self.builtin_functions.new_epoch(builder.func);
+        let vmctx = self.vmctx_val(&mut builder.cursor());
         // new_epoch() returns the new deadline, so we don't have to
         // reload it.
-        let call = builder
-            .ins()
-            .call_indirect(new_epoch_sig, new_epoch, &[vmctx]);
+        let call = builder.ins().call(new_epoch, &[vmctx]);
         let new_deadline = *builder.func.dfg.inst_results(call).first().unwrap();
         builder.def_var(self.epoch_deadline_var, new_deadline);
         builder.ins().jump(continuation_block, &[]);
@@ -984,16 +868,11 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
 
         builder.switch_to_block(null_block);
         let table_index = builder.ins().iconst(I32, table_index.index() as i64);
-        let builtin_idx = BuiltinFunctionIndex::table_get_lazy_init_func_ref();
-        let builtin_sig = self
-            .builtin_function_signatures
+        let lazy_init = self
+            .builtin_functions
             .table_get_lazy_init_func_ref(builder.func);
-        let (vmctx, builtin_addr) =
-            self.translate_load_builtin_function_address(&mut builder.cursor(), builtin_idx);
-        let call_inst =
-            builder
-                .ins()
-                .call_indirect(builtin_sig, builtin_addr, &[vmctx, table_index, index]);
+        let vmctx = self.vmctx_val(&mut builder.cursor());
+        let call_inst = builder.ins().call(lazy_init, &[vmctx, table_index, index]);
         let returned_entry = builder.func.dfg.inst_results(call_inst)[0];
         builder.ins().jump(continuation_block, &[returned_entry]);
         builder.seal_block(continuation_block);
@@ -1544,36 +1423,29 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         delta: ir::Value,
         init_value: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let (func_idx, func_sig) =
-            match self.module.table_plans[table_index].table.wasm_ty.heap_type {
-                WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => (
-                    BuiltinFunctionIndex::table_grow_func_ref(),
-                    self.builtin_function_signatures
-                        .table_grow_func_ref(&mut pos.func),
-                ),
-                #[cfg(feature = "gc")]
-                WasmHeapType::Extern => (
-                    BuiltinFunctionIndex::table_grow_externref(),
-                    self.builtin_function_signatures
-                        .table_grow_externref(&mut pos.func),
-                ),
-                #[cfg(not(feature = "gc"))]
-                WasmHeapType::Extern => {
-                    return Err(cranelift_wasm::wasm_unsupported!(
-                        "support for `externref` disabled at compile time because \
+        let grow = match self.module.table_plans[table_index].table.wasm_ty.heap_type {
+            WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => {
+                self.builtin_functions.table_grow_func_ref(&mut pos.func)
+            }
+
+            #[cfg(feature = "gc")]
+            WasmHeapType::Extern => self.builtin_functions.table_grow_externref(&mut pos.func),
+
+            #[cfg(not(feature = "gc"))]
+            WasmHeapType::Extern => {
+                return Err(cranelift_wasm::wasm_unsupported!(
+                    "support for `externref` disabled at compile time because \
                      the `gc` cargo feature was not enabled",
-                    ))
-                }
-            };
+                ))
+            }
+        };
 
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+        let vmctx = self.vmctx_val(&mut pos);
 
         let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64);
-        let call_inst = pos.ins().call_indirect(
-            func_sig,
-            func_addr,
-            &[vmctx, table_index_arg, delta, init_value],
-        );
+        let call_inst = pos
+            .ins()
+            .call(grow, &[vmctx, table_index_arg, delta, init_value]);
 
         Ok(pos.func.dfg.first_result(call_inst))
     }
@@ -1677,15 +1549,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
                     .ins()
                     .brif(at_capacity, gc_block, &[], no_gc_block, &[]);
                 builder.switch_to_block(gc_block);
-                let builtin_idx = BuiltinFunctionIndex::activations_table_insert_with_gc();
-                let builtin_sig = self
-                    .builtin_function_signatures
+                let insert = self
+                    .builtin_functions
                     .activations_table_insert_with_gc(builder.func);
-                let (vmctx, builtin_addr) = self
-                    .translate_load_builtin_function_address(&mut builder.cursor(), builtin_idx);
-                builder
-                    .ins()
-                    .call_indirect(builtin_sig, builtin_addr, &[vmctx, elem]);
+                let vmctx = self.vmctx_val(&mut builder.cursor());
+                builder.ins().call(insert, &[vmctx, elem]);
                 builder.ins().jump(continue_block, &[]);
 
                 // If `next != end`, then:
@@ -1876,15 +1744,9 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
                 // Call the `drop_externref` builtin to (you guessed it) drop
                 // the `externref`.
                 builder.switch_to_block(drop_block);
-                let builtin_idx = BuiltinFunctionIndex::drop_externref();
-                let builtin_sig = self
-                    .builtin_function_signatures
-                    .drop_externref(builder.func);
-                let (vmctx, builtin_addr) = self
-                    .translate_load_builtin_function_address(&mut builder.cursor(), builtin_idx);
-                builder
-                    .ins()
-                    .call_indirect(builtin_sig, builtin_addr, &[vmctx, current_elem]);
+                let drop_externref = self.builtin_functions.drop_externref(builder.func);
+                let vmctx = self.vmctx_val(&mut builder.cursor());
+                builder.ins().call(drop_externref, &[vmctx, current_elem]);
                 builder.ins().jump(continue_block, &[]);
 
                 builder.switch_to_block(continue_block);
@@ -1916,37 +1778,26 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         val: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
-        let (builtin_idx, builtin_sig) =
-            match self.module.table_plans[table_index].table.wasm_ty.heap_type {
-                WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => (
-                    BuiltinFunctionIndex::table_fill_func_ref(),
-                    self.builtin_function_signatures
-                        .table_fill_func_ref(&mut pos.func),
-                ),
-                #[cfg(feature = "gc")]
-                WasmHeapType::Extern => (
-                    BuiltinFunctionIndex::table_fill_externref(),
-                    self.builtin_function_signatures
-                        .table_fill_externref(&mut pos.func),
-                ),
-                #[cfg(not(feature = "gc"))]
-                WasmHeapType::Extern => {
-                    return Err(cranelift_wasm::wasm_unsupported!(
-                        "support for `externref` disabled at compile time because the \
+        let libcall = match self.module.table_plans[table_index].table.wasm_ty.heap_type {
+            WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => {
+                self.builtin_functions.table_fill_func_ref(&mut pos.func)
+            }
+            #[cfg(feature = "gc")]
+            WasmHeapType::Extern => self.builtin_functions.table_fill_externref(&mut pos.func),
+            #[cfg(not(feature = "gc"))]
+            WasmHeapType::Extern => {
+                return Err(cranelift_wasm::wasm_unsupported!(
+                    "support for `externref` disabled at compile time because the \
                          `gc` cargo feature was not enabled",
-                    ));
-                }
-            };
+                ));
+            }
+        };
 
-        let (vmctx, builtin_addr) =
-            self.translate_load_builtin_function_address(&mut pos, builtin_idx);
+        let vmctx = self.vmctx_val(&mut pos);
 
         let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64);
-        pos.ins().call_indirect(
-            builtin_sig,
-            builtin_addr,
-            &[vmctx, table_index_arg, dst, val, len],
-        );
+        pos.ins()
+            .call(libcall, &[vmctx, table_index_arg, dst, val, len]);
 
         Ok(())
     }
@@ -1989,14 +1840,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         func_index: FuncIndex,
     ) -> WasmResult<ir::Value> {
         let func_index = pos.ins().iconst(I32, func_index.as_u32() as i64);
-        let builtin_index = BuiltinFunctionIndex::ref_func();
-        let builtin_sig = self.builtin_function_signatures.ref_func(&mut pos.func);
-        let (vmctx, builtin_addr) =
-            self.translate_load_builtin_function_address(&mut pos, builtin_index);
+        let ref_func = self.builtin_functions.ref_func(&mut pos.func);
+        let vmctx = self.vmctx_val(&mut pos);
 
-        let call_inst = pos
-            .ins()
-            .call_indirect(builtin_sig, builtin_addr, &[vmctx, func_index]);
+        let call_inst = pos.ins().call(ref_func, &[vmctx, func_index]);
         Ok(pos.func.dfg.first_result(call_inst))
     }
 
@@ -2012,18 +1859,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
             "We only use GlobalVariable::Custom for externref"
         );
 
-        let builtin_index = BuiltinFunctionIndex::externref_global_get();
-        let builtin_sig = self
-            .builtin_function_signatures
-            .externref_global_get(&mut pos.func);
+        let externref_global_get = self.builtin_functions.externref_global_get(&mut pos.func);
 
-        let (vmctx, builtin_addr) =
-            self.translate_load_builtin_function_address(&mut pos, builtin_index);
+        let vmctx = self.vmctx_val(&mut pos);
 
         let global_index_arg = pos.ins().iconst(I32, index.as_u32() as i64);
-        let call_inst =
-            pos.ins()
-                .call_indirect(builtin_sig, builtin_addr, &[vmctx, global_index_arg]);
+        let call_inst = pos
+            .ins()
+            .call(externref_global_get, &[vmctx, global_index_arg]);
 
         Ok(pos.func.dfg.first_result(call_inst))
     }
@@ -2058,17 +1901,13 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
             "We only use GlobalVariable::Custom for externref"
         );
 
-        let builtin_index = BuiltinFunctionIndex::externref_global_set();
-        let builtin_sig = self
-            .builtin_function_signatures
-            .externref_global_set(&mut pos.func);
+        let externref_global_set = self.builtin_functions.externref_global_set(&mut pos.func);
 
-        let (vmctx, builtin_addr) =
-            self.translate_load_builtin_function_address(&mut pos, builtin_index);
+        let vmctx = self.vmctx_val(&mut pos);
 
         let global_index_arg = pos.ins().iconst(I32, index.as_u32() as i64);
         pos.ins()
-            .call_indirect(builtin_sig, builtin_addr, &[vmctx, global_index_arg, value]);
+            .call(externref_global_set, &[vmctx, global_index_arg, value]);
 
         Ok(())
     }
@@ -2393,7 +2232,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         let signature = func.import_signature(sig);
         let name =
             ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName {
-                namespace: 0,
+                namespace: crate::NS_WASM_FUNC,
                 index: index.as_u32(),
             }));
         Ok(func.import_function(ir::ExtFuncData {
@@ -2498,21 +2337,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _heap: Heap,
         val: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let func_sig = self
-            .builtin_function_signatures
-            .memory32_grow(&mut pos.func);
+        let memory_grow = self.builtin_functions.memory32_grow(&mut pos.func);
         let index_arg = index.index();
 
         let memory_index = pos.ins().iconst(I32, index_arg as i64);
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(
-            &mut pos,
-            BuiltinFunctionIndex::memory32_grow(),
-        );
+        let vmctx = self.vmctx_val(&mut pos);
 
         let val = self.cast_memory_index_to_i64(&mut pos, val, index);
-        let call_inst = pos
-            .ins()
-            .call_indirect(func_sig, func_addr, &[vmctx, val, memory_index]);
+        let call_inst = pos.ins().call(memory_grow, &[vmctx, val, memory_index]);
         let result = *pos.func.dfg.inst_results(call_inst).first().unwrap();
         Ok(self.cast_pointer_to_memory_index(pos, result, index))
     }
@@ -2604,10 +2436,9 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         src: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
-        let (vmctx, func_addr) = self
-            .translate_load_builtin_function_address(&mut pos, BuiltinFunctionIndex::memory_copy());
+        let vmctx = self.vmctx_val(&mut pos);
 
-        let func_sig = self.builtin_function_signatures.memory_copy(&mut pos.func);
+        let memory_copy = self.builtin_functions.memory_copy(&mut pos.func);
         let dst = self.cast_memory_index_to_i64(&mut pos, dst, dst_index);
         let src = self.cast_memory_index_to_i64(&mut pos, src, src_index);
         // The length is 32-bit if either memory is 32-bit, but if they're both
@@ -2624,11 +2455,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         };
         let src_index = pos.ins().iconst(I32, i64::from(src_index.as_u32()));
         let dst_index = pos.ins().iconst(I32, i64::from(dst_index.as_u32()));
-        pos.ins().call_indirect(
-            func_sig,
-            func_addr,
-            &[vmctx, dst_index, dst, src_index, src, len],
-        );
+        pos.ins()
+            .call(memory_copy, &[vmctx, dst_index, dst, src_index, src, len]);
 
         Ok(())
     }
@@ -2642,19 +2470,15 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         val: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
-        let func_sig = self.builtin_function_signatures.memory_fill(&mut pos.func);
+        let memory_fill = self.builtin_functions.memory_fill(&mut pos.func);
         let dst = self.cast_memory_index_to_i64(&mut pos, dst, memory_index);
         let len = self.cast_memory_index_to_i64(&mut pos, len, memory_index);
         let memory_index_arg = pos.ins().iconst(I32, i64::from(memory_index.as_u32()));
 
-        let (vmctx, func_addr) = self
-            .translate_load_builtin_function_address(&mut pos, BuiltinFunctionIndex::memory_fill());
+        let vmctx = self.vmctx_val(&mut pos);
 
-        pos.ins().call_indirect(
-            func_sig,
-            func_addr,
-            &[vmctx, memory_index_arg, dst, val, len],
-        );
+        pos.ins()
+            .call(memory_fill, &[vmctx, memory_index_arg, dst, val, len]);
 
         Ok(())
     }
@@ -2669,18 +2493,17 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         src: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
-        let (func_sig, func_idx) = self.get_memory_init_func(&mut pos.func);
+        let memory_init = self.builtin_functions.memory_init(&mut pos.func);
 
         let memory_index_arg = pos.ins().iconst(I32, memory_index.index() as i64);
         let seg_index_arg = pos.ins().iconst(I32, seg_index as i64);
 
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+        let vmctx = self.vmctx_val(&mut pos);
 
         let dst = self.cast_memory_index_to_i64(&mut pos, dst, memory_index);
 
-        pos.ins().call_indirect(
-            func_sig,
-            func_addr,
+        pos.ins().call(
+            memory_init,
             &[vmctx, memory_index_arg, seg_index_arg, dst, src, len],
         );
 
@@ -2688,11 +2511,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
     }
 
     fn translate_data_drop(&mut self, mut pos: FuncCursor, seg_index: u32) -> WasmResult<()> {
-        let (func_sig, func_idx) = self.get_data_drop_func(&mut pos.func);
+        let data_drop = self.builtin_functions.data_drop(&mut pos.func);
         let seg_index_arg = pos.ins().iconst(I32, seg_index as i64);
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
-        pos.ins()
-            .call_indirect(func_sig, func_addr, &[vmctx, seg_index_arg]);
+        let vmctx = self.vmctx_val(&mut pos);
+        pos.ins().call(data_drop, &[vmctx, seg_index_arg]);
         Ok(())
     }
 
@@ -2715,17 +2537,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         src: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
-        let (func_sig, dst_table_index_arg, src_table_index_arg, func_idx) =
+        let (table_copy, dst_table_index_arg, src_table_index_arg) =
             self.get_table_copy_func(&mut pos.func, dst_table_index, src_table_index);
 
         let dst_table_index_arg = pos.ins().iconst(I32, dst_table_index_arg as i64);
         let src_table_index_arg = pos.ins().iconst(I32, src_table_index_arg as i64);
-
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
-
-        pos.ins().call_indirect(
-            func_sig,
-            func_addr,
+        let vmctx = self.vmctx_val(&mut pos);
+        pos.ins().call(
+            table_copy,
             &[
                 vmctx,
                 dst_table_index_arg,
@@ -2748,17 +2567,12 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         src: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
-        let (func_sig, table_index_arg, func_idx) =
-            self.get_table_init_func(&mut pos.func, table_index);
-
-        let table_index_arg = pos.ins().iconst(I32, table_index_arg as i64);
-        let seg_index_arg = pos.ins().iconst(I32, seg_index as i64);
-
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
-
-        pos.ins().call_indirect(
-            func_sig,
-            func_addr,
+        let table_init = self.builtin_functions.table_init(&mut pos.func);
+        let table_index_arg = pos.ins().iconst(I32, i64::from(table_index.as_u32()));
+        let seg_index_arg = pos.ins().iconst(I32, i64::from(seg_index));
+        let vmctx = self.vmctx_val(&mut pos);
+        pos.ins().call(
+            table_init,
             &[vmctx, table_index_arg, seg_index_arg, dst, src, len],
         );
 
@@ -2766,15 +2580,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
     }
 
     fn translate_elem_drop(&mut self, mut pos: FuncCursor, elem_index: u32) -> WasmResult<()> {
-        let (func_sig, func_idx) = self.get_elem_drop_func(&mut pos.func);
-
+        let elem_drop = self.builtin_functions.elem_drop(&mut pos.func);
         let elem_index_arg = pos.ins().iconst(I32, elem_index as i64);
-
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
-
-        pos.ins()
-            .call_indirect(func_sig, func_addr, &[vmctx, elem_index_arg]);
-
+        let vmctx = self.vmctx_val(&mut pos);
+        pos.ins().call(elem_drop, &[vmctx, elem_index_arg]);
         Ok(())
     }
 
@@ -2789,16 +2598,15 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
     ) -> WasmResult<ir::Value> {
         let addr = self.cast_memory_index_to_i64(&mut pos, addr, memory_index);
         let implied_ty = pos.func.dfg.value_type(expected);
-        let (func_sig, memory_index, func_idx) =
+        let (wait_func, memory_index) =
             self.get_memory_atomic_wait(&mut pos.func, memory_index, implied_ty);
 
         let memory_index_arg = pos.ins().iconst(I32, memory_index as i64);
 
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+        let vmctx = self.vmctx_val(&mut pos);
 
-        let call_inst = pos.ins().call_indirect(
-            func_sig,
-            func_addr,
+        let call_inst = pos.ins().call(
+            wait_func,
             &[vmctx, memory_index_arg, addr, expected, timeout],
         );
 
@@ -2814,20 +2622,13 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         count: ir::Value,
     ) -> WasmResult<ir::Value> {
         let addr = self.cast_memory_index_to_i64(&mut pos, addr, memory_index);
-        let func_sig = self
-            .builtin_function_signatures
-            .memory_atomic_notify(&mut pos.func);
+        let atomic_notify = self.builtin_functions.memory_atomic_notify(&mut pos.func);
 
         let memory_index_arg = pos.ins().iconst(I32, memory_index.index() as i64);
-
-        let (vmctx, func_addr) = self.translate_load_builtin_function_address(
-            &mut pos,
-            BuiltinFunctionIndex::memory_atomic_notify(),
-        );
-
-        let call_inst =
-            pos.ins()
-                .call_indirect(func_sig, func_addr, &[vmctx, memory_index_arg, addr, count]);
+        let vmctx = self.vmctx_val(&mut pos);
+        let call_inst = pos
+            .ins()
+            .call(atomic_notify, &[vmctx, memory_index_arg, addr, count]);
 
         Ok(*pos.func.dfg.inst_results(call_inst).first().unwrap())
     }

crates/cranelift/src/lib.rs

@@ -3,18 +3,21 @@
 //! This crate provides an implementation of the `wasmtime_environ::Compiler`
 //! and `wasmtime_environ::CompilerBuilder` traits.
 
+use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Signature};
 use cranelift_codegen::{
     binemit,
-    ir::{self, ExternalName, UserExternalNameRef},
+    ir::{self, ExternalName},
     isa::{CallConv, TargetIsa},
     settings, FinalizedMachReloc, FinalizedRelocTarget, MachTrap,
 };
 use cranelift_entity::PrimaryMap;
 use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmFuncType, WasmValType};
 use target_lexicon::Architecture;
+use wasmtime_environ::{
+    BuiltinFunctionIndex, FlagValue, RelocationTarget, Trap, TrapInformation, Tunables,
+};
 
 pub use builder::builder;
-use wasmtime_environ::{FlagValue, Trap, TrapInformation, Tunables};
 
 pub mod isa_builder;
 mod obj;
@@ -203,6 +206,17 @@ fn reference_type(wasm_ht: cranelift_wasm::WasmHeapType, pointer_type: ir::Type)
     }
 }
 
+// List of namespaces which are processed in `mach_reloc_to_reloc` below.
+
+/// Namespace corresponding to wasm functions, the index is the index of the
+/// defined function that's being referenced.
+pub const NS_WASM_FUNC: u32 = 0;
+
+/// Namespace for builtin function trampolines. The index is the index of the
+/// builtin that's being referenced. These trampolines invoke the real host
+/// function through an indirect function call loaded by the `VMContext`.
+pub const NS_WASMTIME_BUILTIN: u32 = 1;
+
 /// A record of a relocation to perform.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Relocation {
@@ -216,15 +230,6 @@ pub struct Relocation {
     pub addend: binemit::Addend,
 }
 
-/// Destination function. Can be either user function or some special one, like `memory.grow`.
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub enum RelocationTarget {
-    /// The user function index.
-    UserFunc(FuncIndex),
-    /// A compiler-generated libcall.
-    LibCall(ir::LibCall),
-}
-
 /// Converts cranelift_codegen settings to the wasmtime_environ equivalent.
 pub fn clif_flags_to_wasmtime(
     flags: impl IntoIterator<Item = settings::Value>,
@@ -289,10 +294,10 @@ pub fn mach_trap_to_trap(trap: &MachTrap) -> Option<TrapInformation> {
 
 /// Converts machine relocations to relocation information
 /// to perform.
-fn mach_reloc_to_reloc<F>(reloc: &FinalizedMachReloc, transform_user_func_ref: F) -> Relocation
-where
-    F: Fn(UserExternalNameRef) -> (u32, u32),
-{
+fn mach_reloc_to_reloc(
+    reloc: &FinalizedMachReloc,
+    name_map: &PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
+) -> Relocation {
     let &FinalizedMachReloc {
         offset,
         kind,
@@ -301,12 +306,18 @@ where
     } = reloc;
     let reloc_target = match *target {
         FinalizedRelocTarget::ExternalName(ExternalName::User(user_func_ref)) => {
-            let (namespace, index) = transform_user_func_ref(user_func_ref);
-            debug_assert_eq!(namespace, 0);
-            RelocationTarget::UserFunc(FuncIndex::from_u32(index))
+            let name = &name_map[user_func_ref];
+            match name.namespace {
+                NS_WASM_FUNC => RelocationTarget::Wasm(FuncIndex::from_u32(name.index)),
+                NS_WASMTIME_BUILTIN => {
+                    RelocationTarget::Builtin(BuiltinFunctionIndex::from_u32(name.index))
+                }
+                _ => panic!("unknown namespace {}", name.namespace),
+            }
         }
         FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
-            RelocationTarget::LibCall(libcall)
+            let libcall = libcall_cranelift_to_wasmtime(libcall);
+            RelocationTarget::HostLibcall(libcall)
         }
         _ => panic!("unrecognized external name"),
     };
@@ -317,3 +328,104 @@ where
         addend,
     }
 }
+
+fn libcall_cranelift_to_wasmtime(call: ir::LibCall) -> wasmtime_environ::obj::LibCall {
+    use wasmtime_environ::obj::LibCall as LC;
+    match call {
+        ir::LibCall::FloorF32 => LC::FloorF32,
+        ir::LibCall::FloorF64 => LC::FloorF64,
+        ir::LibCall::NearestF32 => LC::NearestF32,
+        ir::LibCall::NearestF64 => LC::NearestF64,
+        ir::LibCall::CeilF32 => LC::CeilF32,
+        ir::LibCall::CeilF64 => LC::CeilF64,
+        ir::LibCall::TruncF32 => LC::TruncF32,
+        ir::LibCall::TruncF64 => LC::TruncF64,
+        ir::LibCall::FmaF32 => LC::FmaF32,
+        ir::LibCall::FmaF64 => LC::FmaF64,
+        ir::LibCall::X86Pshufb => LC::X86Pshufb,
+        _ => panic!("cranelift emitted a libcall wasmtime does not support: {call:?}"),
+    }
+}
+
+/// Helper structure for creating a `Signature` for all builtins.
+struct BuiltinFunctionSignatures {
+    pointer_type: ir::Type,
+
+    #[cfg(feature = "gc")]
+    reference_type: ir::Type,
+
+    call_conv: CallConv,
+}
+
+impl BuiltinFunctionSignatures {
+    fn new(isa: &dyn TargetIsa) -> Self {
+        Self {
+            pointer_type: isa.pointer_type(),
+            #[cfg(feature = "gc")]
+            reference_type: match isa.pointer_type() {
+                ir::types::I32 => ir::types::R32,
+                ir::types::I64 => ir::types::R64,
+                _ => panic!(),
+            },
+            call_conv: CallConv::triple_default(isa.triple()),
+        }
+    }
+
+    fn vmctx(&self) -> AbiParam {
+        AbiParam::special(self.pointer_type, ArgumentPurpose::VMContext)
+    }
+
+    #[cfg(feature = "gc")]
+    fn reference(&self) -> AbiParam {
+        AbiParam::new(self.reference_type)
+    }
+
+    fn pointer(&self) -> AbiParam {
+        AbiParam::new(self.pointer_type)
+    }
+
+    fn i32(&self) -> AbiParam {
+        // Some platform ABIs require i32 values to be zero- or sign-
+        // extended to the full register width.  We need to indicate
+        // this here by using the appropriate .uext or .sext attribute.
+        // The attribute can be added unconditionally; platforms whose
+        // ABI does not require such extensions will simply ignore it.
+        // Note that currently all i32 arguments or return values used
+        // by builtin functions are unsigned, so we always use .uext.
+        // If that ever changes, we will have to add a second type
+        // marker here.
+        AbiParam::new(ir::types::I32).uext()
+    }
+
+    fn i64(&self) -> AbiParam {
+        AbiParam::new(ir::types::I64)
+    }
+
+    fn signature(&self, builtin: BuiltinFunctionIndex) -> Signature {
+        let mut _cur = 0;
+        macro_rules! iter {
+            (
+                $(
+                    $( #[$attr:meta] )*
+                    $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
+                )*
+            ) => {
+                $(
+                    $( #[$attr] )*
+                    if _cur == builtin.index() {
+                        return Signature {
+                            params: vec![ $( self.$param() ),* ],
+                            returns: vec![ $( self.$result() )? ],
+                            call_conv: self.call_conv,
+                        };
+                    }
+                    _cur += 1;
+                )*
+            };
+        }
+
+        wasmtime_environ::foreach_builtin_function!(iter);
+
+        unreachable!();
+    }
+}

crates/cranelift/src/obj.rs

@@ -16,7 +16,6 @@
 use crate::{CompiledFunction, RelocationTarget};
 use anyhow::Result;
 use cranelift_codegen::binemit::Reloc;
-use cranelift_codegen::ir::LibCall;
 use cranelift_codegen::isa::unwind::{systemv, UnwindInfo};
 use cranelift_codegen::TextSectionBuilder;
 use cranelift_control::ControlPlane;
@@ -26,7 +25,8 @@ use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, Symbol
 use object::{Architecture, SectionKind, SymbolFlags, SymbolKind, SymbolScope};
 use std::collections::HashMap;
 use std::ops::Range;
-use wasmtime_environ::{Compiler, FuncIndex};
+use wasmtime_environ::obj::LibCall;
+use wasmtime_environ::Compiler;
 
 const TEXT_SECTION_NAME: &[u8] = b".text";
 
@@ -108,7 +108,7 @@ impl<'a> ModuleTextBuilder<'a> {
         &mut self,
         name: &str,
         compiled_func: &'a CompiledFunction,
-        resolve_reloc_target: impl Fn(FuncIndex) -> usize,
+        resolve_reloc_target: impl Fn(wasmtime_environ::RelocationTarget) -> usize,
     ) -> (SymbolId, Range<u64>) {
         let body = compiled_func.buffer.data();
         let alignment = compiled_func.alignment;
@@ -140,8 +140,8 @@ impl<'a> ModuleTextBuilder<'a> {
                 // resolve this relocation before we actually emit an object
                 // file, but if it can't handle it then we pass through the
                 // relocation.
-                RelocationTarget::UserFunc(index) => {
-                    let target = resolve_reloc_target(index);
+                RelocationTarget::Wasm(_) | RelocationTarget::Builtin(_) => {
+                    let target = resolve_reloc_target(r.reloc_target);
                     if self
                         .text
                         .resolve_reloc(off + u64::from(r.offset), r.reloc, r.addend, target)
@@ -157,7 +157,8 @@ impl<'a> ModuleTextBuilder<'a> {
                     // the final object file as well.
                     panic!(
                         "unresolved relocation could not be processed against \
-                         {index:?}: {r:?}"
+                         {:?}: {r:?}",
+                        r.reloc_target,
                     );
                 }
 
@@ -170,10 +171,10 @@ impl<'a> ModuleTextBuilder<'a> {
                 // 8-byte relocations so that's asserted here and then encoded
                 // directly into the object as a normal object relocation. This
                 // is processed at module load time to resolve the relocations.
-                RelocationTarget::LibCall(call) => {
+                RelocationTarget::HostLibcall(call) => {
                     let symbol = *self.libcall_symbols.entry(call).or_insert_with(|| {
                         self.obj.add_symbol(Symbol {
-                            name: libcall_name(call).as_bytes().to_vec(),
+                            name: call.symbol().as_bytes().to_vec(),
                             value: 0,
                             size: 0,
                             kind: SymbolKind::Text,
@@ -541,22 +542,3 @@ impl<'a> UnwindInfoBuilder<'a> {
         }
     }
 }
-
-fn libcall_name(call: LibCall) -> &'static str {
-    use wasmtime_environ::obj::LibCall as LC;
-    let other = match call {
-        LibCall::FloorF32 => LC::FloorF32,
-        LibCall::FloorF64 => LC::FloorF64,
-        LibCall::NearestF32 => LC::NearestF32,
-        LibCall::NearestF64 => LC::NearestF64,
-        LibCall::CeilF32 => LC::CeilF32,
-        LibCall::CeilF64 => LC::CeilF64,
-        LibCall::TruncF32 => LC::TruncF32,
-        LibCall::TruncF64 => LC::TruncF64,
-        LibCall::FmaF32 => LC::FmaF32,
-        LibCall::FmaF64 => LC::FmaF64,
-        LibCall::X86Pshufb => LC::X86Pshufb,
-        _ => panic!("unknown libcall to give a name to: {call:?}"),
-    };
-    other.symbol()
-}

crates/environ/src/builtin.rs

@@ -92,7 +92,7 @@ macro_rules! foreach_builtin_function {
 }
 
 /// An index type for builtin functions.
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct BuiltinFunctionIndex(u32);
 
 impl BuiltinFunctionIndex {
@@ -120,6 +120,17 @@ macro_rules! declare_indexes {
                 0;
                 $( $( #[$attr] )* $name; )*
             );
+
+            /// Returns a symbol name for this builtin.
+            pub fn name(&self) -> &'static str {
+                $(
+                    $( #[$attr] )*
+                    if *self == BuiltinFunctionIndex::$name() {
+                        return stringify!($name);
+                    }
+                )*
+                unreachable!()
+            }
         }
     };
 

crates/environ/src/compilation.rs

@@ -3,8 +3,8 @@
 
 use crate::{obj, Tunables};
 use crate::{
-    DefinedFuncIndex, FilePos, FuncIndex, FunctionBodyData, ModuleTranslation, ModuleTypesBuilder,
-    PrimaryMap, StackMap, WasmError, WasmFuncType,
+    BuiltinFunctionIndex, DefinedFuncIndex, FilePos, FuncIndex, FunctionBodyData,
+    ModuleTranslation, ModuleTypesBuilder, PrimaryMap, StackMap, WasmError, WasmFuncType,
 };
 use anyhow::Result;
 use object::write::{Object, SymbolId};
@@ -65,6 +65,19 @@ pub enum CompileError {
     DebugInfoNotSupported,
 }
 
+/// What relocations can be applied against.
+///
+/// Each wasm function may refer to various other `RelocationTarget` entries.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum RelocationTarget {
+    /// This is a reference to another defined wasm function in the same module.
+    Wasm(FuncIndex),
+    /// This is a reference to a trampoline for a builtin function.
+    Builtin(BuiltinFunctionIndex),
+    /// A compiler-generated libcall.
+    HostLibcall(obj::LibCall),
+}
+
 /// Implementation of an incremental compilation's key/value cache store.
 ///
 /// In theory, this could just be Cranelift's `CacheKvStore` trait, but it is not as we want to
@@ -174,6 +187,12 @@ pub trait Compiler: Send + Sync {
     /// The body of the function is available in `data` and configuration
     /// values are also passed in via `tunables`. Type information in
     /// `translation` is all relative to `types`.
+    ///
+    /// This function returns a tuple:
+    ///
+    /// 1. Metadata about the wasm function itself.
+    /// 2. The function itself, as an `Any` to get downcasted later when passed
+    ///    to `append_code`.
     fn compile_function(
         &self,
         translation: &ModuleTranslation<'_>,
@@ -216,6 +235,27 @@ pub trait Compiler: Send + Sync {
         wasm_func_ty: &WasmFuncType,
     ) -> Result<Box<dyn Any + Send>, CompileError>;
 
+    /// Creates a tramopline that can be used to call Wasmtime's implementation
+    /// of the builtin function specified by `index`.
+    ///
+    /// The trampoline created can technically have any ABI but currently has
+    /// the native ABI. This will then perform all the necessary duties of an
+    /// exit trampoline from wasm and then perform the actual dispatch to the
+    /// builtin function. Builtin functions in Wasmtime are stored in an array
+    /// in all `VMContext` pointers, so the call to the host is an indirect
+    /// call.
+    fn compile_wasm_to_builtin(
+        &self,
+        index: BuiltinFunctionIndex,
+    ) -> Result<Box<dyn Any + Send>, CompileError>;
+
+    /// Returns the list of relocations required for a function from one of the
+    /// previous `compile_*` functions above.
+    fn compiled_function_relocation_targets<'a>(
+        &'a self,
+        func: &'a dyn Any,
+    ) -> Box<dyn Iterator<Item = RelocationTarget> + 'a>;
+
     /// Appends a list of compiled functions to an in-memory object.
     ///
     /// This function will receive the same `Box<dyn Any>` produced as part of
@@ -238,7 +278,7 @@ pub trait Compiler: Send + Sync {
     ///
     /// 1. First, the index within `funcs` that is being resolved,
     ///
-    /// 2. and next the `FuncIndex` which is the relocation target to
+    /// 2. and next the `RelocationTarget` which is the relocation target to
     /// resolve.
     ///
     /// The return value is an index within `funcs` that the relocation points
@@ -247,7 +287,7 @@ pub trait Compiler: Send + Sync {
         &self,
         obj: &mut Object<'static>,
         funcs: &[(String, Box<dyn Any + Send>)],
-        resolve_reloc: &dyn Fn(usize, FuncIndex) -> usize,
+        resolve_reloc: &dyn Fn(usize, RelocationTarget) -> usize,
     ) -> Result<Vec<(SymbolId, FunctionLoc)>>;
 
     /// Inserts two trampolines into `obj` for a array-call host function:

crates/environ/src/obj.rs

@@ -131,6 +131,7 @@ pub const ELF_WASMTIME_DWARF: &str = ".wasmtime.dwarf";
 
 macro_rules! libcalls {
     ($($rust:ident = $sym:tt)*) => (
+        #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)]
         #[allow(missing_docs)]
         pub enum LibCall {
             $($rust,)*

crates/environ/src/vmoffsets.rs

@@ -7,10 +7,10 @@
 //      magic: u32,
 //      _padding: u32, // (On 64-bit systems)
 //      runtime_limits: *const VMRuntimeLimits,
+//      builtins: *mut VMBuiltinFunctionsArray,
 //      callee: *mut VMFunctionBody,
 //      externref_activations_table: *mut VMExternRefActivationsTable,
 //      store: *mut dyn Store,
-//      builtins: *mut VMBuiltinFunctionsArray,
 //      type_ids: *const VMSharedTypeIndex,
 //      imported_functions: [VMFunctionImport; module.num_imported_functions],
 //      imported_tables: [VMTableImport; module.num_imported_tables],
@@ -74,11 +74,11 @@ pub struct VMOffsets<P> {
     // precalculated offsets of various member fields
     magic: u32,
     runtime_limits: u32,
+    builtin_functions: u32,
     callee: u32,
     epoch_ptr: u32,
     externref_activations_table: u32,
     store: u32,
-    builtin_functions: u32,
     type_ids: u32,
     imported_functions: u32,
     imported_tables: u32,
@@ -106,6 +106,11 @@ pub trait PtrSize {
         .unwrap()
     }
 
+    /// The offset of the `VMContext::builtin_functions` field
+    fn vmcontext_builtin_functions(&self) -> u8 {
+        self.vmcontext_runtime_limits() + self.size()
+    }
+
     /// The offset of the `native_call` field.
     #[inline]
     fn vm_func_ref_native_call(&self) -> u8 {
@@ -355,11 +360,11 @@ impl<P: PtrSize> VMOffsets<P> {
             imported_tables: "imported tables",
             imported_functions: "imported functions",
             type_ids: "module types",
-            builtin_functions: "jit builtin functions state",
             store: "jit store state",
             externref_activations_table: "jit host externref state",
             epoch_ptr: "jit current epoch state",
             callee: "callee function pointer",
+            builtin_functions: "jit builtin functions state",
             runtime_limits: "jit runtime limits state",
             magic: "magic value",
         }
@@ -381,11 +386,11 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
             num_escaped_funcs: fields.num_escaped_funcs,
             magic: 0,
             runtime_limits: 0,
+            builtin_functions: 0,
             callee: 0,
             epoch_ptr: 0,
             externref_activations_table: 0,
             store: 0,
-            builtin_functions: 0,
             type_ids: 0,
             imported_functions: 0,
             imported_tables: 0,
@@ -432,11 +437,11 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
             size(magic) = 4u32,
             align(u32::from(ret.ptr.size())),
             size(runtime_limits) = ret.ptr.size(),
+            size(builtin_functions) = ret.ptr.size(),
             size(callee) = ret.ptr.size(),
             size(epoch_ptr) = ret.ptr.size(),
             size(externref_activations_table) = ret.ptr.size(),
             size(store) = ret.ptr.size() * 2,
-            size(builtin_functions) = ret.pointer_size(),
             size(type_ids) = ret.ptr.size(),
             size(imported_functions)
                 = cmul(ret.num_imported_functions, ret.size_of_vmfunction_import()),

crates/runtime/build.rs

@@ -18,10 +18,6 @@ fn main() {
     build.define(&format!("CFG_TARGET_OS_{}", os), None);
     build.define(&format!("CFG_TARGET_ARCH_{}", arch), None);
     build.define("VERSIONED_SUFFIX", Some(versioned_suffix!()));
-    if arch == "s390x" {
-        println!("cargo:rerun-if-changed=src/trampolines/s390x.S");
-        build.file("src/arch/s390x.S");
-    }
     println!("cargo:rerun-if-changed=src/helpers.c");
     build.file("src/helpers.c");
     build.compile("wasmtime-helpers");

crates/runtime/src/arch/aarch64.rs

@@ -74,47 +74,3 @@ pub fn assert_fp_is_aligned(_fp: usize) {
     //
     // [0]: https://github.com/ARM-software/abi-aa/blob/2022Q1/aapcs64/aapcs64.rst#the-frame-pointer
 }
-
-#[rustfmt::skip]
-macro_rules! wasm_to_libcall_trampoline {
-    ($libcall:ident ; $libcall_impl:ident) => {
-        wasmtime_asm_macros::asm_func!(
-            wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall),
-            "
-                .cfi_startproc
-                bti c
-
-                // Load the pointer to `VMRuntimeLimits` in `x9`.
-                ldur x9, [x0, #8]
-
-                // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits.
-                stur fp, [x9, #24]
-
-                // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits.
-                stur lr, [x9, #32]
-
-                // Tail call to the actual implementation of this libcall.
-                b {}
-
-                .cfi_endproc
-            ",
-            sym $libcall_impl
-        );
-    };
-}
-pub(crate) use wasm_to_libcall_trampoline;
-
-#[cfg(test)]
-mod wasm_to_libcall_trampoline_offsets_tests {
-    use wasmtime_environ::{Module, PtrSize, VMOffsets};
-
-    #[test]
-    fn test() {
-        let module = Module::new();
-        let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module);
-
-        assert_eq!(8, offsets.vmctx_runtime_limits());
-        assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp());
-        assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc());
-    }
-}

crates/runtime/src/arch/riscv64.rs

@@ -39,50 +39,3 @@ pub fn assert_entry_sp_is_aligned(sp: usize) {
 pub fn assert_fp_is_aligned(fp: usize) {
     assert_eq!(fp % 16, 0, "stack should always be aligned to 16");
 }
-
-#[rustfmt::skip]
-macro_rules! wasm_to_libcall_trampoline {
-    ($libcall:ident ; $libcall_impl:ident) => {
-        wasmtime_asm_macros::asm_func!(
-            wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall),
-            concat!(
-                "
-                    .cfi_startproc
-
-                    // Load the pointer to `VMRuntimeLimits` in `t0`.
-                    ld t0, 8(a0)
-
-                    // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits.
-                    sd fp, 24(t0)
-
-                    // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits.
-                    sd ra, 32(t0)
-
-                    // Tail call to the actual implementation of this libcall.
-                .Lhi_{0}:
-                    auipc t0, %pcrel_hi({0})
-                    jalr x0, %pcrel_lo(.Lhi_{0})(t0)
-
-                    .cfi_endproc
-                ",
-            ),
-            sym $libcall_impl,
-        );
-    };
-}
-pub(crate) use wasm_to_libcall_trampoline;
-
-#[cfg(test)]
-mod wasm_to_libcall_trampoline_offsets_tests {
-    use wasmtime_environ::{Module, PtrSize, VMOffsets};
-
-    #[test]
-    fn test() {
-        let module = Module::new();
-        let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module);
-
-        assert_eq!(8, offsets.vmctx_runtime_limits());
-        assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp());
-        assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc());
-    }
-}

crates/runtime/src/arch/s390x.S

@@ -1,70 +0,0 @@
-// Currently `global_asm!` isn't stable on s390x, so this is an external
-// assembler file built with the `build.rs`.
-
-        .machine z13
-        .text
-
-        .hidden host_to_wasm_trampoline
-        .globl host_to_wasm_trampoline
-        .type host_to_wasm_trampoline,@function
-        .p2align 2
-
-#define CONCAT2(a, b) a ## b
-#define CONCAT(a, b) CONCAT2(a , b)
-#define VERSIONED_SYMBOL(a) CONCAT(a, VERSIONED_SUFFIX)
-
-#define LIBCALL_TRAMPOLINE(libcall, libcall_impl)                                 \
-        .hidden VERSIONED_SYMBOL(libcall) ;                                       \
-        .globl VERSIONED_SYMBOL(libcall) ;                                        \
-        .type VERSIONED_SYMBOL(libcall),@function ;                               \
-        .p2align 2 ;                                                              \
-VERSIONED_SYMBOL(libcall): ;                                                      \
-        .cfi_startproc ;                                                          \
-                                                                                  \
-        /* Load the pointer to `VMRuntimeLimits` in `%r1`.  */                    \
-        lg %r1, 8(%r2) ;                                                          \
-                                                                                  \
-        /* Store the last Wasm FP into the `last_wasm_exit_fp` in the limits.  */ \
-        lg %r0, 0(%r15) ;                                                         \
-        stg %r0, 24(%r1) ;                                                        \
-                                                                                  \
-        /* Store the last Wasm PC into the `last_wasm_exit_pc` in the limits.  */ \
-        stg %r14, 32(%r1) ;                                                       \
-                                                                                  \
-        /* Tail call to the actual implementation of this libcall.  */            \
-        jg VERSIONED_SYMBOL(libcall_impl) ;                                       \
-                                                                                  \
-        .cfi_endproc ;                                                            \
-        .size VERSIONED_SYMBOL(libcall),.-VERSIONED_SYMBOL(libcall)
-
-LIBCALL_TRAMPOLINE(memory32_grow, impl_memory32_grow)
-LIBCALL_TRAMPOLINE(table_grow_func_ref, impl_table_grow_func_ref)
-LIBCALL_TRAMPOLINE(table_grow_externref, impl_table_grow_externref)
-LIBCALL_TRAMPOLINE(table_fill_func_ref, impl_table_fill_func_ref)
-LIBCALL_TRAMPOLINE(table_fill_externref, impl_table_fill_externref)
-LIBCALL_TRAMPOLINE(table_copy, impl_table_copy)
-LIBCALL_TRAMPOLINE(table_init, impl_table_init)
-LIBCALL_TRAMPOLINE(elem_drop, impl_elem_drop)
-LIBCALL_TRAMPOLINE(memory_copy, impl_memory_copy)
-LIBCALL_TRAMPOLINE(memory_fill, impl_memory_fill)
-LIBCALL_TRAMPOLINE(memory_init, impl_memory_init)
-LIBCALL_TRAMPOLINE(ref_func, impl_ref_func)
-LIBCALL_TRAMPOLINE(data_drop, impl_data_drop)
-LIBCALL_TRAMPOLINE(table_get_lazy_init_func_ref, impl_table_get_lazy_init_func_ref)
-LIBCALL_TRAMPOLINE(drop_externref, impl_drop_externref)
-LIBCALL_TRAMPOLINE(activations_table_insert_with_gc, impl_activations_table_insert_with_gc)
-LIBCALL_TRAMPOLINE(externref_global_get, impl_externref_global_get)
-LIBCALL_TRAMPOLINE(externref_global_set, impl_externref_global_set)
-LIBCALL_TRAMPOLINE(memory_atomic_notify, impl_memory_atomic_notify)
-LIBCALL_TRAMPOLINE(memory_atomic_wait32, impl_memory_atomic_wait32)
-LIBCALL_TRAMPOLINE(memory_atomic_wait64, impl_memory_atomic_wait64)
-LIBCALL_TRAMPOLINE(out_of_gas, impl_out_of_gas)
-LIBCALL_TRAMPOLINE(new_epoch, impl_new_epoch)
-LIBCALL_TRAMPOLINE(check_malloc, impl_check_malloc)
-LIBCALL_TRAMPOLINE(check_free, impl_check_free)
-LIBCALL_TRAMPOLINE(check_load, impl_check_load)
-LIBCALL_TRAMPOLINE(check_store, impl_check_store)
-LIBCALL_TRAMPOLINE(malloc_start, impl_malloc_start)
-LIBCALL_TRAMPOLINE(free_start, impl_free_start)
-LIBCALL_TRAMPOLINE(update_stack_pointer, impl_update_stack_pointer)
-LIBCALL_TRAMPOLINE(update_mem_size, impl_update_mem_size)

crates/runtime/src/arch/s390x.rs

@@ -32,30 +32,3 @@ pub fn assert_entry_sp_is_aligned(sp: usize) {
 pub fn assert_fp_is_aligned(fp: usize) {
     assert_eq!(fp % 8, 0, "stack should always be aligned to 8");
 }
-
-// The implementation for libcall trampolines is in the s390x.S
-// file.  We provide this dummy definition of wasm_to_libcall_trampoline
-// here to make libcalls.rs compile on s390x.  Note that this means we
-// have to duplicate the list of libcalls used in the assembler file.
-
-macro_rules! wasm_to_libcall_trampoline {
-    ($libcall:ident ; $libcall_impl:ident) => {};
-}
-pub(crate) use wasm_to_libcall_trampoline;
-
-// The wasm_to_host_trampoline implementation is in the s390x.S
-// file, but we still want to have this unit test here.
-#[cfg(test)]
-mod wasm_to_libcall_trampoline_offsets_tests {
-    use wasmtime_environ::{Module, PtrSize, VMOffsets};
-
-    #[test]
-    fn test() {
-        let module = Module::new();
-        let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module);
-
-        assert_eq!(8, offsets.vmctx_runtime_limits());
-        assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp());
-        assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc());
-    }
-}

crates/runtime/src/arch/x86_64.rs

@@ -39,66 +39,3 @@ pub fn assert_entry_sp_is_aligned(sp: usize) {
 pub fn assert_fp_is_aligned(fp: usize) {
     assert_eq!(fp % 16, 0, "stack should always be aligned to 16");
 }
-
-// Helper macros for getting the first and second arguments according to the
-// system calling convention, as well as some callee-saved scratch registers we
-// can safely use in the trampolines.
-cfg_if::cfg_if! {
-    if #[cfg(windows)] {
-        macro_rules! callee_vmctx { () => ("rcx") }
-        macro_rules! scratch0 { () => ("r10") }
-        macro_rules! scratch1 { () => ("r11") }
-    } else {
-        macro_rules! callee_vmctx { () => ("rdi") }
-        macro_rules! scratch0 { () => ("r10") }
-        macro_rules! scratch1 { () => ("r11") }
-    }
-}
-pub(crate) use {callee_vmctx, scratch0, scratch1};
-
-#[rustfmt::skip]
-macro_rules! wasm_to_libcall_trampoline {
-    ($libcall:ident ; $libcall_impl:ident) => {
-        wasmtime_asm_macros::asm_func!(
-            wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall),
-            concat!(
-                "
-                   .cfi_startproc simple
-                   .cfi_def_cfa_offset 0
-
-                    // Load the pointer to `VMRuntimeLimits` in `scratch0!()`.
-                    mov ", crate::arch::scratch0!(), ", 8[", crate::arch::callee_vmctx!(), "]
-
-                    // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits.
-                    mov 24[", crate::arch::scratch0!(), "], rbp
-
-                    // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits.
-                    mov ", crate::arch::scratch1!(), ", [rsp]
-                    mov 32[", crate::arch::scratch0!(), "], ", crate::arch::scratch1!(), "
-
-                    // Tail call to the actual implementation of this libcall.
-                    jmp {}
-
-                    .cfi_endproc
-                ",
-            ),
-            sym $libcall_impl
-        );
-    };
-}
-pub(crate) use wasm_to_libcall_trampoline;
-
-#[cfg(test)]
-mod wasm_to_libcall_trampoline_offsets_tests {
-    use wasmtime_environ::{Module, PtrSize, VMOffsets};
-
-    #[test]
-    fn test() {
-        let module = Module::new();
-        let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module);
-
-        assert_eq!(8, offsets.vmctx_runtime_limits());
-        assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp());
-        assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc());
-    }
-}

crates/runtime/src/libcalls.rs

@@ -68,18 +68,25 @@ use wasmtime_wmemcheck::AccessError::{
     DoubleMalloc, InvalidFree, InvalidRead, InvalidWrite, OutOfBounds,
 };
 
-/// Actually public trampolines which are used by the runtime as the entrypoint
-/// for libcalls.
+/// Raw functions which are actually called from compiled code.
 ///
-/// Note that the trampolines here are actually defined in inline assembly right
-/// now to ensure that the fp/sp on exit are recorded for backtraces to work
-/// properly.
-pub mod trampolines {
+/// Invocation of a builtin currently looks like:
+///
+/// * A wasm function calls a cranelift-compiled trampoline that's generated
+///   once-per-builtin.
+/// * The cranelift-compiled trampoline performs any necessary actions to exit
+///   wasm, such as dealing with fp/pc/etc.
+/// * The cranelift-compiled trampoline loads a function pointer from an array
+///   stored in `VMContext` That function pointer is defined in this module.
+/// * This module runs, handling things like `catch_unwind` and `Result` and
+///   such.
+/// * This module delegates to the outer module (this file) which has the actual
+///   implementation.
+pub mod raw {
     // Allow these things because of the macro and how we can't differentiate
     // between doc comments and `cfg`s.
     #![allow(unused_doc_comments, unused_attributes)]
 
-    use crate::arch::wasm_to_libcall_trampoline;
     use crate::{Instance, TrapReason, VMContext};
 
     macro_rules! libcall {
@@ -88,54 +95,32 @@ pub mod trampolines {
                 $( #[cfg($attr:meta)] )?
                 $name:ident( vmctx: vmctx $(, $pname:ident: $param:ident )* ) $( -> $result:ident )?;
             )*
-        ) => {paste::paste! {
+        ) => {
             $(
-                // The actual libcall itself, which has the `pub` name here, is
-                // defined via the `wasm_to_libcall_trampoline!` macro on
-                // supported platforms or otherwise in inline assembly for
-                // platforms like s390x which don't have stable `global_asm!`
-                // yet.
-                extern "C" {
-                    #[allow(missing_docs)]
-                    #[allow(improper_ctypes)]
-                    #[wasmtime_versioned_export_macros::versioned_link]
-                    pub fn $name(
-                        vmctx: *mut VMContext,
-                        $( $pname: libcall!(@ty $param), )*
-                    ) $(-> libcall!(@ty $result))?;
-                }
-
-                wasm_to_libcall_trampoline!($name ; [<impl_ $name>]);
-
-                // This is the direct entrypoint from the inline assembly which
-                // still has the same raw signature as the trampoline itself.
+                // This is the direct entrypoint from the compiled module which
+                // still has the raw signature.
+                //
                 // This will delegate to the outer module to the actual
                 // implementation and automatically perform `catch_unwind` along
                 // with conversion of the return value in the face of traps.
-                //
-                // Note that rust targets which support `global_asm!` can use
-                // the `sym` operator to get the symbol here, but other targets
-                // like s390x need to use outlined assembly files which requires
-                // `no_mangle`.
-                #[cfg_attr(target_arch = "s390x", wasmtime_versioned_export_macros::versioned_export)]
-                #[allow(unused_variables)]
-                unsafe extern "C" fn [<impl_ $name>](
+                #[allow(unused_variables, missing_docs)]
+                pub unsafe extern "C" fn $name(
                     vmctx: *mut VMContext,
                     $( $pname : libcall!(@ty $param), )*
                 ) $( -> libcall!(@ty $result))? {
                     $(#[cfg($attr)])?
                     {
-                    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-                        Instance::from_vmctx(vmctx, |instance| {
-                            {
-                                super::$name(instance, $($pname),*)
-                            }
-                        })
-                    }));
-                    match result {
-                        Ok(ret) => LibcallResult::convert(ret),
-                        Err(panic) => crate::traphandlers::resume_panic(panic),
-                    }
+                        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                            Instance::from_vmctx(vmctx, |instance| {
+                                {
+                                    super::$name(instance, $($pname),*)
+                                }
+                            })
+                        }));
+                        match result {
+                            Ok(ret) => LibcallResult::convert(ret),
+                            Err(panic) => crate::traphandlers::resume_panic(panic),
+                        }
                     }
                     $(
                         #[cfg(not($attr))]
@@ -147,14 +132,15 @@ pub mod trampolines {
                 // will sometimes strip out some of these symbols resulting
                 // in a linking failure.
                 #[allow(non_upper_case_globals)]
-                #[used]
-                static [<impl_ $name _ref>]: unsafe extern "C" fn(
-                    *mut VMContext,
-                    $( $pname : libcall!(@ty $param), )*
-                ) $( -> libcall!(@ty $result))? = [<impl_ $name>];
-
+                const _: () = {
+                    #[used]
+                    static I_AM_USED: unsafe extern "C" fn(
+                        *mut VMContext,
+                        $( $pname : libcall!(@ty $param), )*
+                    ) $( -> libcall!(@ty $result))? = $name;
+                };
             )*
-        }};
+        };
 
         (@ty i32) => (u32);
         (@ty i64) => (u64);

crates/runtime/src/vmcontext.rs

@@ -723,7 +723,7 @@ macro_rules! define_builtin_array {
             #[allow(unused_doc_comments)]
             pub const INIT: VMBuiltinFunctionsArray = VMBuiltinFunctionsArray {
                 $(
-                    $name: crate::libcalls::trampolines::$name,
+                    $name: crate::libcalls::raw::$name,
                 )*
             };
         }

crates/wasmtime/src/compile.rs

@@ -26,16 +26,16 @@ use crate::Engine;
 use anyhow::{Context, Result};
 use std::{
     any::Any,
-    collections::{btree_map, BTreeMap, BTreeSet, HashMap},
+    collections::{btree_map, BTreeMap, BTreeSet, HashMap, HashSet},
     mem,
 };
 #[cfg(feature = "component-model")]
 use wasmtime_environ::component::Translator;
 use wasmtime_environ::{
-    CompiledFunctionInfo, CompiledModuleInfo, Compiler, DefinedFuncIndex, FinishedObject,
-    FuncIndex, FunctionBodyData, ModuleEnvironment, ModuleInternedTypeIndex, ModuleTranslation,
-    ModuleType, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap, StaticModuleIndex,
-    WasmFunctionInfo,
+    BuiltinFunctionIndex, CompiledFunctionInfo, CompiledModuleInfo, Compiler, DefinedFuncIndex,
+    FinishedObject, FunctionBodyData, ModuleEnvironment, ModuleInternedTypeIndex,
+    ModuleTranslation, ModuleType, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap,
+    RelocationTarget, StaticModuleIndex, WasmFunctionInfo,
 };
 
 mod code_builder;
@@ -217,6 +217,7 @@ impl CompileKey {
     const ARRAY_TO_WASM_TRAMPOLINE_KIND: u32 = Self::new_kind(1);
     const NATIVE_TO_WASM_TRAMPOLINE_KIND: u32 = Self::new_kind(2);
     const WASM_TO_NATIVE_TRAMPOLINE_KIND: u32 = Self::new_kind(3);
+    const WASM_TO_BUILTIN_TRAMPOLINE_KIND: u32 = Self::new_kind(4);
 
     const fn new_kind(kind: u32) -> u32 {
         assert!(kind < (1 << Self::KIND_BITS));
@@ -255,12 +256,19 @@ impl CompileKey {
             index: index.as_u32(),
         }
     }
+
+    fn wasm_to_builtin_trampoline(index: BuiltinFunctionIndex) -> Self {
+        Self {
+            namespace: Self::WASM_TO_BUILTIN_TRAMPOLINE_KIND,
+            index: index.index(),
+        }
+    }
 }
 
 #[cfg(feature = "component-model")]
 impl CompileKey {
-    const TRAMPOLINE_KIND: u32 = Self::new_kind(4);
-    const RESOURCE_DROP_WASM_TO_NATIVE_KIND: u32 = Self::new_kind(5);
+    const TRAMPOLINE_KIND: u32 = Self::new_kind(5);
+    const RESOURCE_DROP_WASM_TO_NATIVE_KIND: u32 = Self::new_kind(6);
 
     fn trampoline(index: wasmtime_environ::component::TrampolineIndex) -> Self {
         Self {
@@ -496,7 +504,13 @@ impl<'a> CompileInputs<'a> {
         let compiler = engine.compiler();
 
         // Compile each individual input in parallel.
-        let raw_outputs = engine.run_maybe_parallel(self.inputs, |f| f(compiler))?;
+        let mut raw_outputs = engine.run_maybe_parallel(self.inputs, |f| f(compiler))?;
+
+        // Now that all functions have been compiled see if any
+        // wasmtime-builtin functions are necessary. If so those need to be
+        // collected and then those trampolines additionally need to be
+        // compiled.
+        compile_required_builtins(engine, &mut raw_outputs)?;
 
         // Bucket the outputs by kind.
         let mut outputs: BTreeMap<u32, Vec<CompileOutput>> = BTreeMap::new();
@@ -504,24 +518,48 @@ impl<'a> CompileInputs<'a> {
             outputs.entry(output.key.kind()).or_default().push(output);
         }
 
-        // Assert that the elements within a bucket are all sorted as we expect
-        // them to be.
-        fn is_sorted_by_key<T, K>(items: &[T], f: impl Fn(&T) -> K) -> bool
-        where
-            K: PartialOrd,
-        {
-            items
-                .windows(2)
-                .all(|window| f(&window[0]) <= f(&window[1]))
-        }
-        debug_assert!(outputs
-            .values()
-            .all(|funcs| is_sorted_by_key(funcs, |x| x.key)));
-
         Ok(UnlinkedCompileOutputs { outputs })
     }
 }
 
+fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutput>) -> Result<()> {
+    let compiler = engine.compiler();
+    let mut builtins = HashSet::new();
+    let mut new_inputs: Vec<CompileInput<'_>> = Vec::new();
+
+    let compile_builtin = |builtin: BuiltinFunctionIndex| {
+        Box::new(move |compiler: &dyn Compiler| {
+            let symbol = format!("wasmtime_builtin_{}", builtin.name());
+            Ok(CompileOutput {
+                key: CompileKey::wasm_to_builtin_trampoline(builtin),
+                symbol,
+                function: CompiledFunction::Function(compiler.compile_wasm_to_builtin(builtin)?),
+                info: None,
+            })
+        })
+    };
+
+    for output in raw_outputs.iter() {
+        let f = match &output.function {
+            CompiledFunction::Function(f) => f,
+            #[cfg(feature = "component-model")]
+            CompiledFunction::AllCallFunc(_) => continue,
+        };
+        for reloc in compiler.compiled_function_relocation_targets(&**f) {
+            match reloc {
+                RelocationTarget::Builtin(i) => {
+                    if builtins.insert(i) {
+                        new_inputs.push(compile_builtin(i));
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+    raw_outputs.extend(engine.run_maybe_parallel(new_inputs, |c| c(compiler))?);
+    Ok(())
+}
+
 #[derive(Default)]
 struct UnlinkedCompileOutputs {
     // A map from kind to `CompileOutput`.
@@ -617,19 +655,28 @@ impl FunctionIndices {
         let symbol_ids_and_locs = compiler.append_code(
             &mut obj,
             &compiled_funcs,
-            &|caller_index: usize, callee_index: FuncIndex| {
-                let module = self
-                    .compiled_func_index_to_module
-                    .get(&caller_index)
-                    .copied()
-                    .expect("should only reloc inside wasm function callers");
-                let def_func_index = translations[module]
-                    .module
-                    .defined_func_index(callee_index)
-                    .unwrap();
-                self.indices[&CompileKey::WASM_FUNCTION_KIND]
-                    [&CompileKey::wasm_function(module, def_func_index)]
-                    .unwrap_function()
+            &|caller_index: usize, callee: RelocationTarget| match callee {
+                RelocationTarget::Wasm(callee_index) => {
+                    let module = self
+                        .compiled_func_index_to_module
+                        .get(&caller_index)
+                        .copied()
+                        .expect("should only reloc inside wasm function callers");
+                    let def_func_index = translations[module]
+                        .module
+                        .defined_func_index(callee_index)
+                        .unwrap();
+                    self.indices[&CompileKey::WASM_FUNCTION_KIND]
+                        [&CompileKey::wasm_function(module, def_func_index)]
+                        .unwrap_function()
+                }
+                RelocationTarget::Builtin(builtin) => self.indices
+                    [&CompileKey::WASM_TO_BUILTIN_TRAMPOLINE_KIND]
+                    [&CompileKey::wasm_to_builtin_trampoline(builtin)]
+                    .unwrap_function(),
+                RelocationTarget::HostLibcall(_) => {
+                    unreachable!("relocation is resolved at runtime, not compile time");
+                }
             },
         )?;
 
@@ -669,6 +716,13 @@ impl FunctionIndices {
         let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables);
         let mut artifacts = Artifacts::default();
 
+        // Remove this as it's not needed by anything below and we'll debug
+        // assert `self.indices` is empty, so this is acknowledgement that this
+        // is a pure runtime implementation detail and not needed in any
+        // metadata generated below.
+        self.indices
+            .remove(&CompileKey::WASM_TO_BUILTIN_TRAMPOLINE_KIND);
+
         // Finally, build our binary artifacts that map things like `FuncIndex`
         // to a function location and all of that using the indices we saved
         // earlier and the function locations we just received after appending

crates/winch/src/compiler.rs

@@ -7,9 +7,9 @@ use std::sync::Mutex;
 use wasmparser::FuncValidatorAllocations;
 use wasmtime_cranelift::{CompiledFunction, ModuleTextBuilder};
 use wasmtime_environ::{
-    CompileError, DefinedFuncIndex, FilePos, FuncIndex, FunctionBodyData, FunctionLoc,
-    ModuleTranslation, ModuleTypesBuilder, PrimaryMap, TrapEncodingBuilder, VMOffsets,
-    WasmFunctionInfo,
+    BuiltinFunctionIndex, CompileError, DefinedFuncIndex, FilePos, FunctionBodyData, FunctionLoc,
+    ModuleTranslation, ModuleTypesBuilder, PrimaryMap, RelocationTarget, TrapEncodingBuilder,
+    VMOffsets, WasmFunctionInfo,
 };
 use winch_codegen::{BuiltinFunctions, TargetIsa};
 
@@ -29,17 +29,6 @@ pub(crate) struct Compiler {
     contexts: Mutex<Vec<CompilationContext>>,
 }
 
-/// The compiled function environment.
-pub struct CompiledFuncEnv;
-impl wasmtime_cranelift::CompiledFuncEnv for CompiledFuncEnv {
-    fn resolve_user_external_name_ref(
-        &self,
-        external: cranelift_codegen::ir::UserExternalNameRef,
-    ) -> (u32, u32) {
-        (0, external.as_u32())
-    }
-}
-
 impl Compiler {
     pub fn new(isa: Box<dyn TargetIsa>, trampolines: Box<dyn wasmtime_environ::Compiler>) -> Self {
         Self {
@@ -109,7 +98,7 @@ impl wasmtime_environ::Compiler for Compiler {
         );
         let mut context = self.get_context(translation);
         let mut validator = validator.into_validator(mem::take(&mut context.allocations));
-        let buffer = self
+        let func = self
             .isa
             .compile_function(
                 ty,
@@ -121,13 +110,10 @@ impl wasmtime_environ::Compiler for Compiler {
             )
             .map_err(|e| CompileError::Codegen(format!("{e:?}")));
         self.save_context(context, validator.into_allocations());
-        let buffer = buffer?;
-
-        let mut compiled_function =
-            CompiledFunction::new(buffer, CompiledFuncEnv {}, self.isa.function_alignment());
+        let mut func = func?;
 
         if self.isa.flags().unwind_info() {
-            self.emit_unwind_info(&mut compiled_function)?;
+            self.emit_unwind_info(&mut func)?;
         }
 
         Ok((
@@ -135,7 +121,7 @@ impl wasmtime_environ::Compiler for Compiler {
                 start_srcloc,
                 stack_maps: Box::new([]),
             },
-            Box::new(compiled_function),
+            Box::new(func),
         ))
     }
 
@@ -171,7 +157,7 @@ impl wasmtime_environ::Compiler for Compiler {
         &self,
         obj: &mut Object<'static>,
         funcs: &[(String, Box<dyn Any + Send>)],
-        resolve_reloc: &dyn Fn(usize, FuncIndex) -> usize,
+        resolve_reloc: &dyn Fn(usize, wasmtime_environ::RelocationTarget) -> usize,
     ) -> Result<Vec<(SymbolId, FunctionLoc)>> {
         let mut builder =
             ModuleTextBuilder::new(obj, self, self.isa.text_section_builder(funcs.len()));
@@ -240,4 +226,18 @@ impl wasmtime_environ::Compiler for Compiler {
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         self.isa.create_systemv_cie()
     }
+
+    fn compile_wasm_to_builtin(
+        &self,
+        index: BuiltinFunctionIndex,
+    ) -> Result<Box<dyn Any + Send>, CompileError> {
+        self.trampolines.compile_wasm_to_builtin(index)
+    }
+
+    fn compiled_function_relocation_targets<'a>(
+        &'a self,
+        func: &'a dyn Any,
+    ) -> Box<dyn Iterator<Item = RelocationTarget> + 'a> {
+        self.trampolines.compiled_function_relocation_targets(func)
+    }
 }

tests/disas/icall-loop.wat

@@ -30,55 +30,54 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64) -> i32 fast
 ;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
+;;     fn0 = colocated u1:9 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;;                                     v16 -> v0
-;;                                     v20 -> v0
-;;                                     v29 -> v0
+;;                                     v18 -> v0
+;;                                     v27 -> v0
 ;;                                     v3 -> v2
-;;                                     v28 -> v3
+;;                                     v26 -> v3
 ;; @002b                               v4 = iconst.i32 2
 ;; @002b                               v5 = icmp uge v2, v4  ; v4 = 2
 ;; @002b                               v10 = iconst.i64 0
 ;; @002b                               v6 = uextend.i64 v2
-;;                                     v30 = iconst.i64 3
-;; @002b                               v8 = ishl v6, v30  ; v30 = 3
-;;                                     v31 = iconst.i64 -2
-;; @002b                               v17 = load.i64 notrap aligned readonly v0+56
-;; @002b                               v18 = load.i64 notrap aligned readonly v17+72
+;;                                     v28 = iconst.i64 3
+;; @002b                               v8 = ishl v6, v28  ; v28 = 3
+;;                                     v29 = iconst.i64 -2
 ;; @002b                               v15 = iconst.i32 0
-;; @002b                               v21 = load.i64 notrap aligned readonly v0+64
-;; @002b                               v22 = load.i32 notrap aligned readonly v21
+;; @002b                               v19 = load.i64 notrap aligned readonly v0+64
+;; @002b                               v20 = load.i32 notrap aligned readonly v19
 ;; @0027                               jump block2
 ;;
 ;;                                 block2:
 ;; @002b                               v7 = load.i64 notrap aligned v0+72
 ;; @002b                               v9 = iadd v7, v8
-;;                                     v32 = iconst.i64 0
-;;                                     v33 = select_spectre_guard v5, v32, v9  ; v32 = 0
-;; @002b                               v12 = load.i64 table_oob aligned table v33
-;;                                     v34 = iconst.i64 -2
-;;                                     v35 = band v12, v34  ; v34 = -2
-;; @002b                               brif v12, block5(v35), block4
+;;                                     v30 = iconst.i64 0
+;;                                     v31 = select_spectre_guard v5, v30, v9  ; v30 = 0
+;; @002b                               v12 = load.i64 table_oob aligned table v31
+;;                                     v32 = iconst.i64 -2
+;;                                     v33 = band v12, v32  ; v32 = -2
+;; @002b                               brif v12, block5(v33), block4
 ;;
 ;;                                 block4 cold:
-;;                                     v36 = iconst.i32 0
-;; @002b                               v19 = call_indirect.i64 sig1, v18(v0, v36, v2)  ; v36 = 0
-;; @002b                               jump block5(v19)
+;;                                     v34 = iconst.i32 0
+;; @002b                               v17 = call fn0(v0, v34, v2)  ; v34 = 0
+;; @002b                               jump block5(v17)
 ;;
 ;;                                 block5(v14: i64):
-;; @002b                               v23 = load.i32 icall_null aligned readonly v14+24
-;; @002b                               v24 = icmp eq v23, v22
-;; @002b                               brif v24, block7, block6
+;; @002b                               v21 = load.i32 icall_null aligned readonly v14+24
+;; @002b                               v22 = icmp eq v21, v20
+;; @002b                               brif v22, block7, block6
 ;;
 ;;                                 block6 cold:
 ;; @002b                               trap bad_sig
 ;;
 ;;                                 block7:
-;; @002b                               v25 = load.i64 notrap aligned readonly v14+16
-;; @002b                               v26 = load.i64 notrap aligned readonly v14+32
-;; @002b                               v27 = call_indirect sig0, v25(v26, v0)
+;; @002b                               v23 = load.i64 notrap aligned readonly v14+16
+;; @002b                               v24 = load.i64 notrap aligned readonly v14+32
+;; @002b                               v25 = call_indirect sig0, v23(v24, v0)
 ;; @002e                               jump block2
 ;; }
 ;;
@@ -90,48 +89,47 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64) -> i32 fast
 ;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
+;;     fn0 = colocated u1:9 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
 ;;                                     v15 -> v0
-;;                                     v19 -> v0
-;;                                     v27 -> v0
-;;                                     v38 = iconst.i64 8
-;;                                     v29 = iconst.i64 -2
-;; @0038                               v16 = load.i64 notrap aligned readonly v0+56
-;; @0038                               v17 = load.i64 notrap aligned readonly v16+72
+;;                                     v17 -> v0
+;;                                     v25 -> v0
+;;                                     v36 = iconst.i64 8
+;;                                     v27 = iconst.i64 -2
 ;; @0038                               v14 = iconst.i32 0
 ;; @0036                               v2 = iconst.i32 1
-;; @0038                               v20 = load.i64 notrap aligned readonly v0+64
-;; @0038                               v21 = load.i32 notrap aligned readonly v20
+;; @0038                               v18 = load.i64 notrap aligned readonly v0+64
+;; @0038                               v19 = load.i32 notrap aligned readonly v18
 ;; @0034                               jump block2
 ;;
 ;;                                 block2:
 ;; @0038                               v6 = load.i64 notrap aligned v0+72
-;;                                     v39 = iconst.i64 8
-;;                                     v40 = iadd v6, v39  ; v39 = 8
-;; @0038                               v11 = load.i64 table_oob aligned table v40
-;;                                     v41 = iconst.i64 -2
-;;                                     v42 = band v11, v41  ; v41 = -2
-;; @0038                               brif v11, block5(v42), block4
+;;                                     v37 = iconst.i64 8
+;;                                     v38 = iadd v6, v37  ; v37 = 8
+;; @0038                               v11 = load.i64 table_oob aligned table v38
+;;                                     v39 = iconst.i64 -2
+;;                                     v40 = band v11, v39  ; v39 = -2
+;; @0038                               brif v11, block5(v40), block4
 ;;
 ;;                                 block4 cold:
-;;                                     v43 = iconst.i32 0
-;;                                     v44 = iconst.i32 1
-;; @0038                               v18 = call_indirect.i64 sig1, v17(v0, v43, v44)  ; v43 = 0, v44 = 1
-;; @0038                               jump block5(v18)
+;;                                     v41 = iconst.i32 0
+;;                                     v42 = iconst.i32 1
+;; @0038                               v16 = call fn0(v0, v41, v42)  ; v41 = 0, v42 = 1
+;; @0038                               jump block5(v16)
 ;;
 ;;                                 block5(v13: i64):
-;; @0038                               v22 = load.i32 icall_null aligned readonly v13+24
-;; @0038                               v23 = icmp eq v22, v21
-;; @0038                               brif v23, block7, block6
+;; @0038                               v20 = load.i32 icall_null aligned readonly v13+24
+;; @0038                               v21 = icmp eq v20, v19
+;; @0038                               brif v21, block7, block6
 ;;
 ;;                                 block6 cold:
 ;; @0038                               trap bad_sig
 ;;
 ;;                                 block7:
-;; @0038                               v24 = load.i64 notrap aligned readonly v13+16
-;; @0038                               v25 = load.i64 notrap aligned readonly v13+32
-;; @0038                               v26 = call_indirect sig0, v24(v25, v0)
+;; @0038                               v22 = load.i64 notrap aligned readonly v13+16
+;; @0038                               v23 = load.i64 notrap aligned readonly v13+32
+;; @0038                               v24 = call_indirect sig0, v22(v23, v0)
 ;; @003b                               jump block2
 ;; }

tests/disas/icall-simd.wat

@@ -16,6 +16,7 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64, i8x16) -> i8x16 fast
 ;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
+;;     fn0 = colocated u1:9 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i8x16):
@@ -34,22 +35,20 @@
 ;;                                 block2 cold:
 ;; @0033                               v16 = iconst.i32 0
 ;; @0033                               v17 = global_value.i64 gv3
-;; @0033                               v18 = load.i64 notrap aligned readonly v17+56
-;; @0033                               v19 = load.i64 notrap aligned readonly v18+72
-;; @0033                               v20 = call_indirect sig1, v19(v17, v16, v2)  ; v16 = 0
-;; @0033                               jump block3(v20)
+;; @0033                               v18 = call fn0(v17, v16, v2)  ; v16 = 0
+;; @0033                               jump block3(v18)
 ;;
 ;;                                 block3(v15: i64):
-;; @0033                               v21 = global_value.i64 gv3
-;; @0033                               v22 = load.i64 notrap aligned readonly v21+64
-;; @0033                               v23 = load.i32 notrap aligned readonly v22
-;; @0033                               v24 = load.i32 icall_null aligned readonly v15+24
-;; @0033                               v25 = icmp eq v24, v23
-;; @0033                               trapz v25, bad_sig
-;; @0033                               v26 = load.i64 notrap aligned readonly v15+16
-;; @0033                               v27 = load.i64 notrap aligned readonly v15+32
-;; @0033                               v28 = call_indirect sig0, v26(v27, v0, v3)
-;; @0036                               jump block1(v28)
+;; @0033                               v19 = global_value.i64 gv3
+;; @0033                               v20 = load.i64 notrap aligned readonly v19+64
+;; @0033                               v21 = load.i32 notrap aligned readonly v20
+;; @0033                               v22 = load.i32 icall_null aligned readonly v15+24
+;; @0033                               v23 = icmp eq v22, v21
+;; @0033                               trapz v23, bad_sig
+;; @0033                               v24 = load.i64 notrap aligned readonly v15+16
+;; @0033                               v25 = load.i64 notrap aligned readonly v15+32
+;; @0033                               v26 = call_indirect sig0, v24(v25, v0, v3)
+;; @0036                               jump block1(v26)
 ;;
 ;;                                 block1(v4: i8x16):
 ;; @0036                               return v4

tests/disas/icall.wat

@@ -16,6 +16,7 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64, f32) -> i32 fast
 ;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
+;;     fn0 = colocated u1:9 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: f32):
@@ -34,22 +35,20 @@
 ;;                                 block2 cold:
 ;; @0033                               v16 = iconst.i32 0
 ;; @0033                               v17 = global_value.i64 gv3
-;; @0033                               v18 = load.i64 notrap aligned readonly v17+56
-;; @0033                               v19 = load.i64 notrap aligned readonly v18+72
-;; @0033                               v20 = call_indirect sig1, v19(v17, v16, v2)  ; v16 = 0
-;; @0033                               jump block3(v20)
+;; @0033                               v18 = call fn0(v17, v16, v2)  ; v16 = 0
+;; @0033                               jump block3(v18)
 ;;
 ;;                                 block3(v15: i64):
-;; @0033                               v21 = global_value.i64 gv3
-;; @0033                               v22 = load.i64 notrap aligned readonly v21+64
-;; @0033                               v23 = load.i32 notrap aligned readonly v22
-;; @0033                               v24 = load.i32 icall_null aligned readonly v15+24
-;; @0033                               v25 = icmp eq v24, v23
-;; @0033                               trapz v25, bad_sig
-;; @0033                               v26 = load.i64 notrap aligned readonly v15+16
-;; @0033                               v27 = load.i64 notrap aligned readonly v15+32
-;; @0033                               v28 = call_indirect sig0, v26(v27, v0, v3)
-;; @0036                               jump block1(v28)
+;; @0033                               v19 = global_value.i64 gv3
+;; @0033                               v20 = load.i64 notrap aligned readonly v19+64
+;; @0033                               v21 = load.i32 notrap aligned readonly v20
+;; @0033                               v22 = load.i32 icall_null aligned readonly v15+24
+;; @0033                               v23 = icmp eq v22, v21
+;; @0033                               trapz v23, bad_sig
+;; @0033                               v24 = load.i64 notrap aligned readonly v15+16
+;; @0033                               v25 = load.i64 notrap aligned readonly v15+32
+;; @0033                               v26 = call_indirect sig0, v24(v25, v0, v3)
+;; @0036                               jump block1(v26)
 ;;
 ;;                                 block1(v4: i32):
 ;; @0036                               return v4

tests/disas/passive-data.wat

@@ -20,16 +20,15 @@
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned readonly checked gv3+80
 ;;     sig0 = (i64 vmctx, i32 uext, i32 uext, i64, i32 uext, i32 uext) system_v
+;;     fn0 = colocated u1:6 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
 ;; @003d                               v5 = iconst.i32 0
 ;; @003d                               v6 = iconst.i32 0
 ;; @003d                               v7 = global_value.i64 gv3
-;; @003d                               v8 = load.i64 notrap aligned readonly v7+56
-;; @003d                               v9 = load.i64 notrap aligned readonly v8+48
-;; @003d                               v10 = uextend.i64 v2
-;; @003d                               call_indirect sig0, v9(v7, v5, v6, v10, v3, v4)  ; v5 = 0, v6 = 0
+;; @003d                               v8 = uextend.i64 v2
+;; @003d                               call fn0(v7, v5, v6, v8, v3, v4)  ; v5 = 0, v6 = 0
 ;; @0041                               jump block1
 ;;
 ;;                                 block1:
@@ -42,14 +41,13 @@
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     sig0 = (i64 vmctx, i32 uext) system_v
+;;     fn0 = colocated u1:8 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
 ;; @0044                               v2 = iconst.i32 0
 ;; @0044                               v3 = global_value.i64 gv3
-;; @0044                               v4 = load.i64 notrap aligned readonly v3+56
-;; @0044                               v5 = load.i64 notrap aligned readonly v4+64
-;; @0044                               call_indirect sig0, v5(v3, v2)  ; v2 = 0
+;; @0044                               call fn0(v3, v2)  ; v2 = 0
 ;; @0047                               jump block1
 ;;
 ;;                                 block1:

tests/disas/readonly-funcrefs.wat

@@ -39,47 +39,46 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64) fast
 ;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
+;;     fn0 = colocated u1:9 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;;                                     v15 -> v0
-;;                                     v19 -> v0
-;;                                     v26 -> v0
+;;                                     v17 -> v0
+;;                                     v24 -> v0
 ;; @0031                               v6 = load.i64 notrap aligned v0+72
 ;; @0031                               v3 = iconst.i32 2
 ;; @0031                               v4 = icmp uge v2, v3  ; v3 = 2
 ;; @0031                               v9 = iconst.i64 0
 ;; @0031                               v5 = uextend.i64 v2
-;;                                     v27 = iconst.i64 3
-;; @0031                               v7 = ishl v5, v27  ; v27 = 3
+;;                                     v25 = iconst.i64 3
+;; @0031                               v7 = ishl v5, v25  ; v25 = 3
 ;; @0031                               v8 = iadd v6, v7
 ;; @0031                               v10 = select_spectre_guard v4, v9, v8  ; v9 = 0
 ;; @0031                               v11 = load.i64 table_oob aligned table v10
-;;                                     v28 = iconst.i64 -2
-;; @0031                               v12 = band v11, v28  ; v28 = -2
+;;                                     v26 = iconst.i64 -2
+;; @0031                               v12 = band v11, v26  ; v26 = -2
 ;; @0031                               brif v11, block3(v12), block2
 ;;
 ;;                                 block2 cold:
-;; @0031                               v16 = load.i64 notrap aligned readonly v0+56
-;; @0031                               v17 = load.i64 notrap aligned readonly v16+72
 ;; @0031                               v14 = iconst.i32 0
-;; @0031                               v18 = call_indirect sig1, v17(v0, v14, v2)  ; v14 = 0
-;; @0031                               jump block3(v18)
+;; @0031                               v16 = call fn0(v0, v14, v2)  ; v14 = 0
+;; @0031                               jump block3(v16)
 ;;
 ;;                                 block3(v13: i64):
-;; @0031                               v22 = load.i32 icall_null aligned readonly v13+24
-;; @0031                               v20 = load.i64 notrap aligned readonly v0+64
-;; @0031                               v21 = load.i32 notrap aligned readonly v20
-;; @0031                               v23 = icmp eq v22, v21
-;; @0031                               brif v23, block5, block4
+;; @0031                               v20 = load.i32 icall_null aligned readonly v13+24
+;; @0031                               v18 = load.i64 notrap aligned readonly v0+64
+;; @0031                               v19 = load.i32 notrap aligned readonly v18
+;; @0031                               v21 = icmp eq v20, v19
+;; @0031                               brif v21, block5, block4
 ;;
 ;;                                 block4 cold:
 ;; @0031                               trap bad_sig
 ;;
 ;;                                 block5:
-;; @0031                               v24 = load.i64 notrap aligned readonly v13+16
-;; @0031                               v25 = load.i64 notrap aligned readonly v13+32
-;; @0031                               call_indirect sig0, v24(v25, v0)
+;; @0031                               v22 = load.i64 notrap aligned readonly v13+16
+;; @0031                               v23 = load.i64 notrap aligned readonly v13+32
+;; @0031                               call_indirect sig0, v22(v23, v0)
 ;; @0034                               jump block1
 ;;
 ;;                                 block1:

tests/disas/ref-func-0.wat

@@ -19,24 +19,21 @@
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     sig0 = (i64 vmctx, i32 uext) -> r64 system_v
+;;     fn0 = colocated u1:27 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
 ;; @008f                               v6 = global_value.i64 gv3
-;; @008f                               v7 = load.i64 notrap aligned readonly v6+56
-;; @008f                               v8 = load.i64 notrap aligned readonly v7+216
-;; @008f                               v9 = iconst.i32 0
-;; @008f                               v10 = call_indirect sig0, v8(v6, v9)  ; v9 = 0
-;; @0091                               v11 = global_value.i64 gv3
-;; @0091                               v12 = load.i64 notrap aligned readonly v11+56
-;; @0091                               v13 = load.i64 notrap aligned readonly v12+216
-;; @0091                               v14 = iconst.i32 1
-;; @0091                               v15 = call_indirect sig0, v13(v11, v14)  ; v14 = 1
-;; @0093                               v16 = global_value.i64 gv3
-;; @0093                               v17 = load.i64 notrap aligned table v16+144
-;; @0095                               v18 = global_value.i64 gv3
-;; @0095                               v19 = load.i64 notrap aligned table v18+160
-;; @0097                               jump block1(v10, v15, v17, v19)
+;; @008f                               v7 = iconst.i32 0
+;; @008f                               v8 = call fn0(v6, v7)  ; v7 = 0
+;; @0091                               v9 = global_value.i64 gv3
+;; @0091                               v10 = iconst.i32 1
+;; @0091                               v11 = call fn0(v9, v10)  ; v10 = 1
+;; @0093                               v12 = global_value.i64 gv3
+;; @0093                               v13 = load.i64 notrap aligned table v12+144
+;; @0095                               v14 = global_value.i64 gv3
+;; @0095                               v15 = load.i64 notrap aligned table v14+160
+;; @0097                               jump block1(v8, v11, v13, v15)
 ;;
 ;;                                 block1(v2: r64, v3: r64, v4: i64, v5: i64):
 ;; @0097                               return v2, v3, v4, v5

tests/disas/table-copy.wat

@@ -68,15 +68,14 @@
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     sig0 = (i64 vmctx, i32 uext, i32 uext, i32 uext, i32 uext, i32 uext) system_v
+;;     fn0 = colocated u1:1 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):
 ;; @0090                               v7 = iconst.i32 0
 ;; @0090                               v8 = iconst.i32 1
 ;; @0090                               v9 = global_value.i64 gv3
-;; @0090                               v10 = load.i64 notrap aligned readonly v9+56
-;; @0090                               v11 = load.i64 notrap aligned readonly v10+8
-;; @0090                               call_indirect sig0, v11(v9, v7, v8, v3, v4, v5)  ; v7 = 0, v8 = 1
+;; @0090                               call fn0(v9, v7, v8, v3, v4, v5)  ; v7 = 0, v8 = 1
 ;; @0094                               jump block1(v2)
 ;;
 ;;                                 block1(v6: i32):
@@ -89,15 +88,14 @@
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     sig0 = (i64 vmctx, i32 uext, i32 uext, i32 uext, i32 uext, i32 uext) system_v
+;;     fn0 = colocated u1:1 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):
 ;; @009f                               v7 = iconst.i32 1
 ;; @009f                               v8 = iconst.i32 0
 ;; @009f                               v9 = global_value.i64 gv3
-;; @009f                               v10 = load.i64 notrap aligned readonly v9+56
-;; @009f                               v11 = load.i64 notrap aligned readonly v10+8
-;; @009f                               call_indirect sig0, v11(v9, v7, v8, v3, v4, v5)  ; v7 = 1, v8 = 0
+;; @009f                               call fn0(v9, v7, v8, v3, v4, v5)  ; v7 = 1, v8 = 0
 ;; @00a3                               jump block1(v2)
 ;;
 ;;                                 block1(v6: i32):

tests/disas/table-get-fixed-size.wat

@@ -22,19 +22,20 @@
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, r64) system_v
+;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
 ;;                                     v14 -> v0
 ;;                                     v19 -> v0
-;;                                     v25 -> v0
+;;                                     v23 -> v0
 ;; @0052                               v3 = iconst.i32 0
 ;; @0054                               v4 = iconst.i32 7
 ;; @0054                               v5 = icmp uge v3, v4  ; v3 = 0, v4 = 7
 ;; @0054                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0054                               v7 = load.i64 notrap aligned v0+72
-;;                                     v26 = iconst.i64 3
-;; @0054                               v8 = ishl v6, v26  ; v26 = 3
+;;                                     v24 = iconst.i64 3
+;; @0054                               v8 = ishl v6, v24  ; v24 = 3
 ;; @0054                               v9 = iadd v7, v8
 ;; @0054                               v10 = iconst.i64 0
 ;; @0054                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -44,27 +45,25 @@
 ;; @0054                               brif v13, block2, block3
 ;;
 ;;                                 block3:
-;; @0054                               v15 = load.i64 notrap aligned v0+32
+;; @0054                               v15 = load.i64 notrap aligned v0+40
 ;; @0054                               v16 = load.i64 notrap aligned v15
 ;; @0054                               v17 = load.i64 notrap aligned v15+8
 ;; @0054                               v18 = icmp eq v16, v17
 ;; @0054                               brif v18, block4, block5
 ;;
 ;;                                 block5:
-;; @0054                               v22 = load.i64 notrap aligned v12
-;;                                     v27 = iconst.i64 1
-;; @0054                               v23 = iadd v22, v27  ; v27 = 1
-;; @0054                               store notrap aligned v23, v12
+;; @0054                               v20 = load.i64 notrap aligned v12
+;;                                     v25 = iconst.i64 1
+;; @0054                               v21 = iadd v20, v25  ; v25 = 1
+;; @0054                               store notrap aligned v21, v12
 ;; @0054                               store.r64 notrap aligned v12, v16
-;;                                     v28 = iconst.i64 8
-;; @0054                               v24 = iadd.i64 v16, v28  ; v28 = 8
-;; @0054                               store notrap aligned v24, v15
+;;                                     v26 = iconst.i64 8
+;; @0054                               v22 = iadd.i64 v16, v26  ; v26 = 8
+;; @0054                               store notrap aligned v22, v15
 ;; @0054                               jump block2
 ;;
 ;;                                 block4:
-;; @0054                               v20 = load.i64 notrap aligned readonly v0+56
-;; @0054                               v21 = load.i64 notrap aligned readonly v20+208
-;; @0054                               call_indirect sig0, v21(v0, v12)
+;; @0054                               call fn0(v0, v12)
 ;; @0054                               jump block2
 ;;
 ;;                                 block2:
@@ -81,18 +80,19 @@
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, r64) system_v
+;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;;                                     v14 -> v0
 ;;                                     v19 -> v0
-;;                                     v25 -> v0
+;;                                     v23 -> v0
 ;; @005b                               v4 = iconst.i32 7
 ;; @005b                               v5 = icmp uge v2, v4  ; v4 = 7
 ;; @005b                               v6 = uextend.i64 v2
 ;; @005b                               v7 = load.i64 notrap aligned v0+72
-;;                                     v26 = iconst.i64 3
-;; @005b                               v8 = ishl v6, v26  ; v26 = 3
+;;                                     v24 = iconst.i64 3
+;; @005b                               v8 = ishl v6, v24  ; v24 = 3
 ;; @005b                               v9 = iadd v7, v8
 ;; @005b                               v10 = iconst.i64 0
 ;; @005b                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -102,27 +102,25 @@
 ;; @005b                               brif v13, block2, block3
 ;;
 ;;                                 block3:
-;; @005b                               v15 = load.i64 notrap aligned v0+32
+;; @005b                               v15 = load.i64 notrap aligned v0+40
 ;; @005b                               v16 = load.i64 notrap aligned v15
 ;; @005b                               v17 = load.i64 notrap aligned v15+8
 ;; @005b                               v18 = icmp eq v16, v17
 ;; @005b                               brif v18, block4, block5
 ;;
 ;;                                 block5:
-;; @005b                               v22 = load.i64 notrap aligned v12
-;;                                     v27 = iconst.i64 1
-;; @005b                               v23 = iadd v22, v27  ; v27 = 1
-;; @005b                               store notrap aligned v23, v12
+;; @005b                               v20 = load.i64 notrap aligned v12
+;;                                     v25 = iconst.i64 1
+;; @005b                               v21 = iadd v20, v25  ; v25 = 1
+;; @005b                               store notrap aligned v21, v12
 ;; @005b                               store.r64 notrap aligned v12, v16
-;;                                     v28 = iconst.i64 8
-;; @005b                               v24 = iadd.i64 v16, v28  ; v28 = 8
-;; @005b                               store notrap aligned v24, v15
+;;                                     v26 = iconst.i64 8
+;; @005b                               v22 = iadd.i64 v16, v26  ; v26 = 8
+;; @005b                               store notrap aligned v22, v15
 ;; @005b                               jump block2
 ;;
 ;;                                 block4:
-;; @005b                               v20 = load.i64 notrap aligned readonly v0+56
-;; @005b                               v21 = load.i64 notrap aligned readonly v20+208
-;; @005b                               call_indirect sig0, v21(v0, v12)
+;; @005b                               call fn0(v0, v12)
 ;; @005b                               jump block2
 ;;
 ;;                                 block2:

tests/disas/table-get.wat

@@ -22,20 +22,21 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     gv5 = load.i32 notrap aligned gv3+80
 ;;     sig0 = (i64 vmctx, r64) system_v
+;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
 ;;                                     v14 -> v0
 ;;                                     v19 -> v0
-;;                                     v25 -> v0
-;;                                     v26 -> v0
+;;                                     v23 -> v0
+;;                                     v24 -> v0
 ;; @0051                               v3 = iconst.i32 0
 ;; @0053                               v4 = load.i32 notrap aligned v0+80
 ;; @0053                               v5 = icmp uge v3, v4  ; v3 = 0
 ;; @0053                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0053                               v7 = load.i64 notrap aligned v0+72
-;;                                     v27 = iconst.i64 3
-;; @0053                               v8 = ishl v6, v27  ; v27 = 3
+;;                                     v25 = iconst.i64 3
+;; @0053                               v8 = ishl v6, v25  ; v25 = 3
 ;; @0053                               v9 = iadd v7, v8
 ;; @0053                               v10 = iconst.i64 0
 ;; @0053                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -45,27 +46,25 @@
 ;; @0053                               brif v13, block2, block3
 ;;
 ;;                                 block3:
-;; @0053                               v15 = load.i64 notrap aligned v0+32
+;; @0053                               v15 = load.i64 notrap aligned v0+40
 ;; @0053                               v16 = load.i64 notrap aligned v15
 ;; @0053                               v17 = load.i64 notrap aligned v15+8
 ;; @0053                               v18 = icmp eq v16, v17
 ;; @0053                               brif v18, block4, block5
 ;;
 ;;                                 block5:
-;; @0053                               v22 = load.i64 notrap aligned v12
-;;                                     v28 = iconst.i64 1
-;; @0053                               v23 = iadd v22, v28  ; v28 = 1
-;; @0053                               store notrap aligned v23, v12
+;; @0053                               v20 = load.i64 notrap aligned v12
+;;                                     v26 = iconst.i64 1
+;; @0053                               v21 = iadd v20, v26  ; v26 = 1
+;; @0053                               store notrap aligned v21, v12
 ;; @0053                               store.r64 notrap aligned v12, v16
-;;                                     v29 = iconst.i64 8
-;; @0053                               v24 = iadd.i64 v16, v29  ; v29 = 8
-;; @0053                               store notrap aligned v24, v15
+;;                                     v27 = iconst.i64 8
+;; @0053                               v22 = iadd.i64 v16, v27  ; v27 = 8
+;; @0053                               store notrap aligned v22, v15
 ;; @0053                               jump block2
 ;;
 ;;                                 block4:
-;; @0053                               v20 = load.i64 notrap aligned readonly v0+56
-;; @0053                               v21 = load.i64 notrap aligned readonly v20+208
-;; @0053                               call_indirect sig0, v21(v0, v12)
+;; @0053                               call fn0(v0, v12)
 ;; @0053                               jump block2
 ;;
 ;;                                 block2:
@@ -83,19 +82,20 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     gv5 = load.i32 notrap aligned gv3+80
 ;;     sig0 = (i64 vmctx, r64) system_v
+;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;;                                     v14 -> v0
 ;;                                     v19 -> v0
-;;                                     v25 -> v0
-;;                                     v26 -> v0
+;;                                     v23 -> v0
+;;                                     v24 -> v0
 ;; @005a                               v4 = load.i32 notrap aligned v0+80
 ;; @005a                               v5 = icmp uge v2, v4
 ;; @005a                               v6 = uextend.i64 v2
 ;; @005a                               v7 = load.i64 notrap aligned v0+72
-;;                                     v27 = iconst.i64 3
-;; @005a                               v8 = ishl v6, v27  ; v27 = 3
+;;                                     v25 = iconst.i64 3
+;; @005a                               v8 = ishl v6, v25  ; v25 = 3
 ;; @005a                               v9 = iadd v7, v8
 ;; @005a                               v10 = iconst.i64 0
 ;; @005a                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -105,27 +105,25 @@
 ;; @005a                               brif v13, block2, block3
 ;;
 ;;                                 block3:
-;; @005a                               v15 = load.i64 notrap aligned v0+32
+;; @005a                               v15 = load.i64 notrap aligned v0+40
 ;; @005a                               v16 = load.i64 notrap aligned v15
 ;; @005a                               v17 = load.i64 notrap aligned v15+8
 ;; @005a                               v18 = icmp eq v16, v17
 ;; @005a                               brif v18, block4, block5
 ;;
 ;;                                 block5:
-;; @005a                               v22 = load.i64 notrap aligned v12
-;;                                     v28 = iconst.i64 1
-;; @005a                               v23 = iadd v22, v28  ; v28 = 1
-;; @005a                               store notrap aligned v23, v12
+;; @005a                               v20 = load.i64 notrap aligned v12
+;;                                     v26 = iconst.i64 1
+;; @005a                               v21 = iadd v20, v26  ; v26 = 1
+;; @005a                               store notrap aligned v21, v12
 ;; @005a                               store.r64 notrap aligned v12, v16
-;;                                     v29 = iconst.i64 8
-;; @005a                               v24 = iadd.i64 v16, v29  ; v29 = 8
-;; @005a                               store notrap aligned v24, v15
+;;                                     v27 = iconst.i64 8
+;; @005a                               v22 = iadd.i64 v16, v27  ; v27 = 8
+;; @005a                               store notrap aligned v22, v15
 ;; @005a                               jump block2
 ;;
 ;;                                 block4:
-;; @005a                               v20 = load.i64 notrap aligned readonly v0+56
-;; @005a                               v21 = load.i64 notrap aligned readonly v20+208
-;; @005a                               call_indirect sig0, v21(v0, v12)
+;; @005a                               call fn0(v0, v12)
 ;; @005a                               jump block2
 ;;
 ;;                                 block2:

tests/disas/table-set-fixed-size.wat

@@ -16,7 +16,6 @@
     local.get 0
     local.get 1
     table.set 0))
-
 ;; function u0:0(i64 vmctx, i64, r64) fast {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
@@ -24,18 +23,19 @@
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64) system_v
+;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: r64):
 ;;                                     v20 -> v0
-;;                                     v23 -> v0
+;;                                     v21 -> v0
 ;; @0052                               v3 = iconst.i32 0
 ;; @0056                               v4 = iconst.i32 7
 ;; @0056                               v5 = icmp uge v3, v4  ; v3 = 0, v4 = 7
 ;; @0056                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0056                               v7 = load.i64 notrap aligned v0+72
-;;                                     v24 = iconst.i64 3
-;; @0056                               v8 = ishl v6, v24  ; v24 = 3
+;;                                     v22 = iconst.i64 3
+;; @0056                               v8 = ishl v6, v22  ; v22 = 3
 ;; @0056                               v9 = iadd v7, v8
 ;; @0056                               v10 = iconst.i64 0
 ;; @0056                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -46,29 +46,27 @@
 ;;
 ;;                                 block2:
 ;; @0056                               v14 = load.i64 notrap aligned v2
-;;                                     v25 = iconst.i64 1
-;; @0056                               v15 = iadd v14, v25  ; v25 = 1
+;;                                     v23 = iconst.i64 1
+;; @0056                               v15 = iadd v14, v23  ; v23 = 1
 ;; @0056                               store notrap aligned v15, v2
 ;; @0056                               jump block3
 ;;
 ;;                                 block3:
-;;                                     v26 = iconst.i64 0
-;; @0056                               v16 = icmp.i64 eq v12, v26  ; v26 = 0
+;;                                     v24 = iconst.i64 0
+;; @0056                               v16 = icmp.i64 eq v12, v24  ; v24 = 0
 ;; @0056                               brif v16, block6, block4
 ;;
 ;;                                 block4:
 ;; @0056                               v17 = load.i64 notrap aligned v12
-;;                                     v27 = iconst.i64 -1
-;; @0056                               v18 = iadd v17, v27  ; v27 = -1
+;;                                     v25 = iconst.i64 -1
+;; @0056                               v18 = iadd v17, v25  ; v25 = -1
 ;; @0056                               store notrap aligned v18, v12
-;;                                     v28 = iconst.i64 0
-;; @0056                               v19 = icmp eq v18, v28  ; v28 = 0
+;;                                     v26 = iconst.i64 0
+;; @0056                               v19 = icmp eq v18, v26  ; v26 = 0
 ;; @0056                               brif v19, block5, block6
 ;;
 ;;                                 block5:
-;; @0056                               v21 = load.i64 notrap aligned readonly v0+56
-;; @0056                               v22 = load.i64 notrap aligned readonly v21+200
-;; @0056                               call_indirect sig0, v22(v0, v12)
+;; @0056                               call fn0(v0, v12)
 ;; @0056                               jump block6
 ;;
 ;;                                 block6:
@@ -85,17 +83,18 @@
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64) system_v
+;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: r64):
 ;;                                     v20 -> v0
-;;                                     v23 -> v0
+;;                                     v21 -> v0
 ;; @005f                               v4 = iconst.i32 7
 ;; @005f                               v5 = icmp uge v2, v4  ; v4 = 7
 ;; @005f                               v6 = uextend.i64 v2
 ;; @005f                               v7 = load.i64 notrap aligned v0+72
-;;                                     v24 = iconst.i64 3
-;; @005f                               v8 = ishl v6, v24  ; v24 = 3
+;;                                     v22 = iconst.i64 3
+;; @005f                               v8 = ishl v6, v22  ; v22 = 3
 ;; @005f                               v9 = iadd v7, v8
 ;; @005f                               v10 = iconst.i64 0
 ;; @005f                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -106,29 +105,27 @@
 ;;
 ;;                                 block2:
 ;; @005f                               v14 = load.i64 notrap aligned v3
-;;                                     v25 = iconst.i64 1
-;; @005f                               v15 = iadd v14, v25  ; v25 = 1
+;;                                     v23 = iconst.i64 1
+;; @005f                               v15 = iadd v14, v23  ; v23 = 1
 ;; @005f                               store notrap aligned v15, v3
 ;; @005f                               jump block3
 ;;
 ;;                                 block3:
-;;                                     v26 = iconst.i64 0
-;; @005f                               v16 = icmp.i64 eq v12, v26  ; v26 = 0
+;;                                     v24 = iconst.i64 0
+;; @005f                               v16 = icmp.i64 eq v12, v24  ; v24 = 0
 ;; @005f                               brif v16, block6, block4
 ;;
 ;;                                 block4:
 ;; @005f                               v17 = load.i64 notrap aligned v12
-;;                                     v27 = iconst.i64 -1
-;; @005f                               v18 = iadd v17, v27  ; v27 = -1
+;;                                     v25 = iconst.i64 -1
+;; @005f                               v18 = iadd v17, v25  ; v25 = -1
 ;; @005f                               store notrap aligned v18, v12
-;;                                     v28 = iconst.i64 0
-;; @005f                               v19 = icmp eq v18, v28  ; v28 = 0
+;;                                     v26 = iconst.i64 0
+;; @005f                               v19 = icmp eq v18, v26  ; v26 = 0
 ;; @005f                               brif v19, block5, block6
 ;;
 ;;                                 block5:
-;; @005f                               v21 = load.i64 notrap aligned readonly v0+56
-;; @005f                               v22 = load.i64 notrap aligned readonly v21+200
-;; @005f                               call_indirect sig0, v22(v0, v12)
+;; @005f                               call fn0(v0, v12)
 ;; @005f                               jump block6
 ;;
 ;;                                 block6:

tests/disas/table-set.wat

@@ -24,19 +24,20 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     gv5 = load.i32 notrap aligned gv3+80
 ;;     sig0 = (i64 vmctx, i64) system_v
+;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: r64):
 ;;                                     v20 -> v0
-;;                                     v23 -> v0
-;;                                     v24 -> v0
+;;                                     v21 -> v0
+;;                                     v22 -> v0
 ;; @0051                               v3 = iconst.i32 0
 ;; @0055                               v4 = load.i32 notrap aligned v0+80
 ;; @0055                               v5 = icmp uge v3, v4  ; v3 = 0
 ;; @0055                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0055                               v7 = load.i64 notrap aligned v0+72
-;;                                     v25 = iconst.i64 3
-;; @0055                               v8 = ishl v6, v25  ; v25 = 3
+;;                                     v23 = iconst.i64 3
+;; @0055                               v8 = ishl v6, v23  ; v23 = 3
 ;; @0055                               v9 = iadd v7, v8
 ;; @0055                               v10 = iconst.i64 0
 ;; @0055                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -47,29 +48,27 @@
 ;;
 ;;                                 block2:
 ;; @0055                               v14 = load.i64 notrap aligned v2
-;;                                     v26 = iconst.i64 1
-;; @0055                               v15 = iadd v14, v26  ; v26 = 1
+;;                                     v24 = iconst.i64 1
+;; @0055                               v15 = iadd v14, v24  ; v24 = 1
 ;; @0055                               store notrap aligned v15, v2
 ;; @0055                               jump block3
 ;;
 ;;                                 block3:
-;;                                     v27 = iconst.i64 0
-;; @0055                               v16 = icmp.i64 eq v12, v27  ; v27 = 0
+;;                                     v25 = iconst.i64 0
+;; @0055                               v16 = icmp.i64 eq v12, v25  ; v25 = 0
 ;; @0055                               brif v16, block6, block4
 ;;
 ;;                                 block4:
 ;; @0055                               v17 = load.i64 notrap aligned v12
-;;                                     v28 = iconst.i64 -1
-;; @0055                               v18 = iadd v17, v28  ; v28 = -1
+;;                                     v26 = iconst.i64 -1
+;; @0055                               v18 = iadd v17, v26  ; v26 = -1
 ;; @0055                               store notrap aligned v18, v12
-;;                                     v29 = iconst.i64 0
-;; @0055                               v19 = icmp eq v18, v29  ; v29 = 0
+;;                                     v27 = iconst.i64 0
+;; @0055                               v19 = icmp eq v18, v27  ; v27 = 0
 ;; @0055                               brif v19, block5, block6
 ;;
 ;;                                 block5:
-;; @0055                               v21 = load.i64 notrap aligned readonly v0+56
-;; @0055                               v22 = load.i64 notrap aligned readonly v21+200
-;; @0055                               call_indirect sig0, v22(v0, v12)
+;; @0055                               call fn0(v0, v12)
 ;; @0055                               jump block6
 ;;
 ;;                                 block6:
@@ -87,18 +86,19 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     gv5 = load.i32 notrap aligned gv3+80
 ;;     sig0 = (i64 vmctx, i64) system_v
+;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: r64):
 ;;                                     v20 -> v0
-;;                                     v23 -> v0
-;;                                     v24 -> v0
+;;                                     v21 -> v0
+;;                                     v22 -> v0
 ;; @005e                               v4 = load.i32 notrap aligned v0+80
 ;; @005e                               v5 = icmp uge v2, v4
 ;; @005e                               v6 = uextend.i64 v2
 ;; @005e                               v7 = load.i64 notrap aligned v0+72
-;;                                     v25 = iconst.i64 3
-;; @005e                               v8 = ishl v6, v25  ; v25 = 3
+;;                                     v23 = iconst.i64 3
+;; @005e                               v8 = ishl v6, v23  ; v23 = 3
 ;; @005e                               v9 = iadd v7, v8
 ;; @005e                               v10 = iconst.i64 0
 ;; @005e                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
@@ -109,29 +109,27 @@
 ;;
 ;;                                 block2:
 ;; @005e                               v14 = load.i64 notrap aligned v3
-;;                                     v26 = iconst.i64 1
-;; @005e                               v15 = iadd v14, v26  ; v26 = 1
+;;                                     v24 = iconst.i64 1
+;; @005e                               v15 = iadd v14, v24  ; v24 = 1
 ;; @005e                               store notrap aligned v15, v3
 ;; @005e                               jump block3
 ;;
 ;;                                 block3:
-;;                                     v27 = iconst.i64 0
-;; @005e                               v16 = icmp.i64 eq v12, v27  ; v27 = 0
+;;                                     v25 = iconst.i64 0
+;; @005e                               v16 = icmp.i64 eq v12, v25  ; v25 = 0
 ;; @005e                               brif v16, block6, block4
 ;;
 ;;                                 block4:
 ;; @005e                               v17 = load.i64 notrap aligned v12
-;;                                     v28 = iconst.i64 -1
-;; @005e                               v18 = iadd v17, v28  ; v28 = -1
+;;                                     v26 = iconst.i64 -1
+;; @005e                               v18 = iadd v17, v26  ; v26 = -1
 ;; @005e                               store notrap aligned v18, v12
-;;                                     v29 = iconst.i64 0
-;; @005e                               v19 = icmp eq v18, v29  ; v29 = 0
+;;                                     v27 = iconst.i64 0
+;; @005e                               v19 = icmp eq v18, v27  ; v27 = 0
 ;; @005e                               brif v19, block5, block6
 ;;
 ;;                                 block5:
-;; @005e                               v21 = load.i64 notrap aligned readonly v0+56
-;; @005e                               v22 = load.i64 notrap aligned readonly v21+200
-;; @005e                               call_indirect sig0, v22(v0, v12)
+;; @005e                               call fn0(v0, v12)
 ;; @005e                               jump block6
 ;;
 ;;                                 block6:

tests/disas/typed-funcrefs.wat

@@ -135,64 +135,61 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
 ;;     sig1 = (i64 vmctx, i64, i32, i32, i32, i32) -> i32 fast
+;;     fn0 = colocated u1:9 sig0
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):
 ;;                                     v21 -> v0
-;;                                     v47 -> v0
-;;                                     v56 -> v0
-;;                                     v59 -> v0
-;;                                     v30 -> v2
-;;                                     v31 -> v3
-;;                                     v32 -> v4
-;;                                     v33 -> v5
+;;                                     v45 -> v0
+;;                                     v52 -> v0
+;;                                     v55 -> v0
+;;                                     v28 -> v2
+;;                                     v29 -> v3
+;;                                     v30 -> v4
+;;                                     v31 -> v5
 ;; @0048                               v12 = load.i64 notrap aligned v0+72
-;;                                     v70 = iconst.i64 8
-;; @0048                               v14 = iadd v12, v70  ; v70 = 8
+;;                                     v66 = iconst.i64 8
+;; @0048                               v14 = iadd v12, v66  ; v66 = 8
 ;; @0048                               v17 = load.i64 table_oob aligned table v14
-;;                                     v58 = iconst.i64 -2
-;; @0048                               v18 = band v17, v58  ; v58 = -2
+;;                                     v54 = iconst.i64 -2
+;; @0048                               v18 = band v17, v54  ; v54 = -2
 ;; @0048                               brif v17, block3(v18), block2
 ;;
 ;;                                 block2 cold:
-;; @005b                               v48 = load.i64 notrap aligned readonly v0+56
-;; @005b                               v49 = load.i64 notrap aligned readonly v48+72
 ;; @003c                               v7 = iconst.i32 0
-;;                                     v28 -> v7
+;;                                     v26 -> v7
 ;; @0046                               v8 = iconst.i32 1
-;; @0048                               v24 = call_indirect sig0, v49(v0, v7, v8)  ; v7 = 0, v8 = 1
-;; @0048                               jump block3(v24)
+;; @0048                               v22 = call fn0(v0, v7, v8)  ; v7 = 0, v8 = 1
+;; @0048                               jump block3(v22)
 ;;
 ;;                                 block3(v19: i64):
-;; @004a                               v25 = load.i64 null_reference aligned readonly v19+16
-;; @004a                               v26 = load.i64 notrap aligned readonly v19+32
-;; @004a                               v27 = call_indirect sig1, v25(v26, v0, v2, v3, v4, v5)
-;; @005b                               v38 = load.i64 notrap aligned v0+72
-;;                                     v78 = iconst.i64 16
-;; @005b                               v40 = iadd v38, v78  ; v78 = 16
-;; @005b                               v43 = load.i64 table_oob aligned table v40
-;;                                     v79 = iconst.i64 -2
-;;                                     v80 = band v43, v79  ; v79 = -2
-;; @005b                               brif v43, block5(v80), block4
+;; @004a                               v23 = load.i64 null_reference aligned readonly v19+16
+;; @004a                               v24 = load.i64 notrap aligned readonly v19+32
+;; @004a                               v25 = call_indirect sig1, v23(v24, v0, v2, v3, v4, v5)
+;; @005b                               v36 = load.i64 notrap aligned v0+72
+;;                                     v74 = iconst.i64 16
+;; @005b                               v38 = iadd v36, v74  ; v74 = 16
+;; @005b                               v41 = load.i64 table_oob aligned table v38
+;;                                     v75 = iconst.i64 -2
+;;                                     v76 = band v41, v75  ; v75 = -2
+;; @005b                               brif v41, block5(v76), block4
 ;;
 ;;                                 block4 cold:
-;;                                     v81 = load.i64 notrap aligned readonly v0+56
-;;                                     v82 = load.i64 notrap aligned readonly v81+72
-;;                                     v83 = iconst.i32 0
-;; @0059                               v34 = iconst.i32 2
-;; @005b                               v50 = call_indirect sig0, v82(v0, v83, v34)  ; v83 = 0, v34 = 2
-;; @005b                               jump block5(v50)
+;;                                     v77 = iconst.i32 0
+;; @0059                               v32 = iconst.i32 2
+;; @005b                               v46 = call fn0(v0, v77, v32)  ; v77 = 0, v32 = 2
+;; @005b                               jump block5(v46)
 ;;
-;;                                 block5(v45: i64):
-;; @005d                               v51 = load.i64 null_reference aligned readonly v45+16
-;; @005d                               v52 = load.i64 notrap aligned readonly v45+32
-;; @005d                               v53 = call_indirect sig1, v51(v52, v0, v2, v3, v4, v5)
+;;                                 block5(v43: i64):
+;; @005d                               v47 = load.i64 null_reference aligned readonly v43+16
+;; @005d                               v48 = load.i64 notrap aligned readonly v43+32
+;; @005d                               v49 = call_indirect sig1, v47(v48, v0, v2, v3, v4, v5)
 ;; @0066                               jump block1
 ;;
 ;;                                 block1:
-;; @0061                               v55 = iadd.i32 v53, v27
-;;                                     v6 -> v55
-;; @0066                               return v55
+;; @0061                               v51 = iadd.i32 v49, v25
+;;                                     v6 -> v51
+;; @0066                               return v51
 ;; }
 ;;
 ;; function u0:2(i64 vmctx, i64, i32, i32, i32, i32) -> i32 fast {
@@ -203,64 +200,61 @@
 ;;     gv4 = load.i64 notrap aligned gv3+72
 ;;     sig0 = (i64 vmctx, i64, i32, i32, i32, i32) -> i32 fast
 ;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
+;;     fn0 = colocated u1:9 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):
 ;;                                     v21 -> v0
-;;                                     v47 -> v0
-;;                                     v56 -> v0
-;;                                     v59 -> v0
-;;                                     v30 -> v2
-;;                                     v31 -> v3
-;;                                     v32 -> v4
-;;                                     v33 -> v5
+;;                                     v45 -> v0
+;;                                     v52 -> v0
+;;                                     v55 -> v0
+;;                                     v28 -> v2
+;;                                     v29 -> v3
+;;                                     v30 -> v4
+;;                                     v31 -> v5
 ;; @0075                               v12 = load.i64 notrap aligned v0+72
-;;                                     v70 = iconst.i64 8
-;; @0075                               v14 = iadd v12, v70  ; v70 = 8
+;;                                     v66 = iconst.i64 8
+;; @0075                               v14 = iadd v12, v66  ; v66 = 8
 ;; @0075                               v17 = load.i64 table_oob aligned table v14
-;;                                     v58 = iconst.i64 -2
-;; @0075                               v18 = band v17, v58  ; v58 = -2
+;;                                     v54 = iconst.i64 -2
+;; @0075                               v18 = band v17, v54  ; v54 = -2
 ;; @0075                               brif v17, block3(v18), block2
 ;;
 ;;                                 block2 cold:
-;; @0087                               v48 = load.i64 notrap aligned readonly v0+56
-;; @0087                               v49 = load.i64 notrap aligned readonly v48+72
 ;; @0069                               v7 = iconst.i32 0
-;;                                     v28 -> v7
+;;                                     v26 -> v7
 ;; @0073                               v8 = iconst.i32 1
-;; @0075                               v24 = call_indirect sig1, v49(v0, v7, v8)  ; v7 = 0, v8 = 1
-;; @0075                               jump block3(v24)
+;; @0075                               v22 = call fn0(v0, v7, v8)  ; v7 = 0, v8 = 1
+;; @0075                               jump block3(v22)
 ;;
 ;;                                 block3(v19: i64):
-;; @0075                               v25 = load.i64 icall_null aligned readonly v19+16
-;; @0075                               v26 = load.i64 notrap aligned readonly v19+32
-;; @0075                               v27 = call_indirect sig0, v25(v26, v0, v2, v3, v4, v5)
-;; @0087                               v38 = load.i64 notrap aligned v0+72
-;;                                     v78 = iconst.i64 16
-;; @0087                               v40 = iadd v38, v78  ; v78 = 16
-;; @0087                               v43 = load.i64 table_oob aligned table v40
-;;                                     v79 = iconst.i64 -2
-;;                                     v80 = band v43, v79  ; v79 = -2
-;; @0087                               brif v43, block5(v80), block4
+;; @0075                               v23 = load.i64 icall_null aligned readonly v19+16
+;; @0075                               v24 = load.i64 notrap aligned readonly v19+32
+;; @0075                               v25 = call_indirect sig0, v23(v24, v0, v2, v3, v4, v5)
+;; @0087                               v36 = load.i64 notrap aligned v0+72
+;;                                     v74 = iconst.i64 16
+;; @0087                               v38 = iadd v36, v74  ; v74 = 16
+;; @0087                               v41 = load.i64 table_oob aligned table v38
+;;                                     v75 = iconst.i64 -2
+;;                                     v76 = band v41, v75  ; v75 = -2
+;; @0087                               brif v41, block5(v76), block4
 ;;
 ;;                                 block4 cold:
-;;                                     v81 = load.i64 notrap aligned readonly v0+56
-;;                                     v82 = load.i64 notrap aligned readonly v81+72
-;;                                     v83 = iconst.i32 0
-;; @0085                               v34 = iconst.i32 2
-;; @0087                               v50 = call_indirect sig1, v82(v0, v83, v34)  ; v83 = 0, v34 = 2
-;; @0087                               jump block5(v50)
+;;                                     v77 = iconst.i32 0
+;; @0085                               v32 = iconst.i32 2
+;; @0087                               v46 = call fn0(v0, v77, v32)  ; v77 = 0, v32 = 2
+;; @0087                               jump block5(v46)
 ;;
-;;                                 block5(v45: i64):
-;; @0087                               v51 = load.i64 icall_null aligned readonly v45+16
-;; @0087                               v52 = load.i64 notrap aligned readonly v45+32
-;; @0087                               v53 = call_indirect sig0, v51(v52, v0, v2, v3, v4, v5)
+;;                                 block5(v43: i64):
+;; @0087                               v47 = load.i64 icall_null aligned readonly v43+16
+;; @0087                               v48 = load.i64 notrap aligned readonly v43+32
+;; @0087                               v49 = call_indirect sig0, v47(v48, v0, v2, v3, v4, v5)
 ;; @0091                               jump block1
 ;;
 ;;                                 block1:
-;; @008c                               v55 = iadd.i32 v53, v27
-;;                                     v6 -> v55
-;; @0091                               return v55
+;; @008c                               v51 = iadd.i32 v49, v25
+;;                                     v6 -> v51
+;; @0091                               return v51
 ;; }
 ;;
 ;; function u0:3(i64 vmctx, i64, i32, i32, i32, i32) -> i32 fast {

winch/codegen/Cargo.toml

@@ -23,6 +23,7 @@ cranelift-codegen = { workspace = true, features = ["unwind"] }
 regalloc2 = { workspace = true }
 gimli = { workspace = true }
 wasmtime-environ = { workspace = true }
+wasmtime-cranelift = { workspace = true }
 
 [features]
 x64 = ["cranelift-codegen/x86"]

winch/codegen/src/codegen/builtin.rs

@@ -12,26 +12,23 @@ use wasmtime_environ::{BuiltinFunctionIndex, PtrSize, VMOffsets, WasmValType};
 #[derive(Copy, Clone)]
 pub(crate) enum BuiltinType {
     /// Dynamic built-in function, derived from the VMContext.
-    Dynamic {
-        /// The index of the built-in function.
-        index: u32,
-        /// The built-in function base, relative to the VMContext.
-        base: u32,
-    },
+    Builtin(BuiltinFunctionIndex),
     /// A known libcall.
     /// See [`cranelift_codegen::ir::LibCall`] for more details.
-    Known(LibCall),
+    LibCall(LibCall),
 }
 
 impl BuiltinType {
-    /// Create a new dynamic built-in function type.
-    pub fn dynamic(index: u32, base: u32) -> Self {
-        Self::Dynamic { index, base }
+    /// Creates a new builtin from a Wasmtime-defined builtin function
+    /// enumerated with a [`BuiltinFunctionIndex`].
+    pub fn builtin(idx: BuiltinFunctionIndex) -> Self {
+        Self::Builtin(idx)
     }
 
-    /// Create a new known built-in function type.
-    pub fn known(libcall: LibCall) -> Self {
-        Self::Known(libcall)
+    /// Creates a new builtin from a Compiler-defined [`LibCall`] typically used
+    /// late in lowering.
+    pub fn libcall(libcall: LibCall) -> Self {
+        Self::LibCall(libcall)
     }
 }
 
@@ -70,12 +67,8 @@ macro_rules! declare_function_sig {
         pub struct BuiltinFunctions {
             /// The target calling convention.
             call_conv: CallingConvention,
-            /// The target pointer size.
-            ptr_size: u8,
             /// The target pointer type, as a WebAssembly type.
             ptr_type: WasmValType,
-            /// The builtin functions base relative to the VMContext.
-            base: u32,
             /// F32 Ceil.
             ceil_f32: Option<BuiltinFunction>,
             /// F64 Ceil.
@@ -105,9 +98,7 @@ macro_rules! declare_function_sig {
                 let size = vmoffsets.ptr.size();
                 #[allow(unused_doc_comments)]
                 Self {
-                    ptr_size: size,
                     call_conv,
-                    base: vmoffsets.vmctx_builtin_functions(),
                     ptr_type: ptr_type_from_ptr_size(size),
                     ceil_f32: None,
                     ceil_f64: None,
@@ -163,7 +154,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn ceil_f32<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.ceil_f32.is_none() {
                     let sig = self.over_f32::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::CeilF32) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::CeilF32) });
                     self.ceil_f32 = Some(BuiltinFunction {
                         inner,
                     });
@@ -174,7 +165,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn ceil_f64<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.ceil_f64.is_none() {
                     let sig = self.over_f64::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::CeilF64) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::CeilF64) });
                     self.ceil_f64 = Some(BuiltinFunction {
                         inner,
                     });
@@ -185,7 +176,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn floor_f32<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.floor_f32.is_none() {
                     let sig = self.over_f32::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::FloorF32) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::FloorF32) });
                     self.floor_f32 = Some(BuiltinFunction {
                         inner,
                     });
@@ -196,7 +187,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn floor_f64<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.floor_f64.is_none() {
                     let sig = self.over_f64::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::FloorF64) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::FloorF64) });
                     self.floor_f64 = Some(BuiltinFunction {
                         inner,
                     });
@@ -207,7 +198,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn trunc_f32<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.trunc_f32.is_none() {
                     let sig = self.over_f32::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::TruncF32) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::TruncF32) });
                     self.trunc_f32 = Some(BuiltinFunction {
                         inner,
                     });
@@ -218,7 +209,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn trunc_f64<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.trunc_f64.is_none() {
                     let sig = self.over_f64::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::TruncF64) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::TruncF64) });
                     self.trunc_f64 = Some(BuiltinFunction {
                         inner,
                     });
@@ -229,7 +220,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn nearest_f32<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.nearest_f32.is_none() {
                     let sig = self.over_f32::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::NearestF32) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::NearestF32) });
                     self.nearest_f32 = Some(BuiltinFunction {
                         inner,
                     });
@@ -240,7 +231,7 @@ macro_rules! declare_function_sig {
             pub(crate) fn nearest_f64<A: ABI>(&mut self) -> BuiltinFunction {
                 if self.nearest_f64.is_none() {
                     let sig = self.over_f64::<A>();
-                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::known(LibCall::NearestF64) });
+                    let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::libcall(LibCall::NearestF64) });
                     self.nearest_f64 = Some(BuiltinFunction {
                         inner,
                     });
@@ -256,7 +247,7 @@ macro_rules! declare_function_sig {
                         let result = vec![ $(self.$result() )?];
                         let sig = A::sig_from(&params, &result, &self.call_conv);
                         let index = BuiltinFunctionIndex::$name();
-                        let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::dynamic(index.index() * (self.ptr_size as u32), self.base) });
+                        let inner = Arc::new(BuiltinFunctionInner { sig, ty: BuiltinType::builtin(index) });
                         self.$name = Some(BuiltinFunction {
                             inner,
                         });

winch/codegen/src/codegen/call.rs

@@ -58,15 +58,16 @@
 
 use crate::{
     abi::{vmctx, ABIOperand, ABISig, RetArea, ABI},
-    codegen::{BuiltinFunction, BuiltinType, Callee, CalleeInfo, CodeGenContext},
+    codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext},
     masm::{
         CalleeKind, ContextArgs, MacroAssembler, MemMoveDirection, OperandSize, SPOffset,
         VMContextLoc,
     },
     reg::Reg,
     stack::Val,
+    FuncEnv,
 };
-use wasmtime_environ::{PtrSize, VMOffsets};
+use wasmtime_environ::{FuncIndex, PtrSize, VMOffsets};
 
 /// All the information needed to emit a function call.
 #[derive(Copy, Clone)]
@@ -80,14 +81,15 @@ impl FnCall {
     /// 4. Creates the stack space needed for the return area.
     /// 5. Emits the call.
     /// 6. Cleans up the stack space.
-    pub fn emit<'a, M: MacroAssembler, P: PtrSize>(
+    pub fn emit<M: MacroAssembler>(
+        env: &mut FuncEnv<M::Ptr>,
         masm: &mut M,
         context: &mut CodeGenContext,
-        callee: Callee<'a>,
+        callee: Callee,
     ) {
-        let sig = callee.sig();
-        let (kind, callee_context) = Self::lower(&context.vmoffsets, &callee, sig, context, masm);
+        let (kind, callee_context) = Self::lower(env, context.vmoffsets, &callee, context, masm);
 
+        let sig = env.callee_sig::<M::ABI>(&callee);
         context.spill(masm);
         let ret_area = Self::make_ret_area(&sig, masm);
         let arg_stack_space = sig.params_stack_size();
@@ -122,51 +124,49 @@ impl FnCall {
     /// Lowers the high-level [`Callee`] to a [`CalleeKind`] and
     /// [ContextArgs] pair which contains all the metadata needed for
     /// emission.
-    fn lower<P: PtrSize, M: MacroAssembler>(
-        vmoffsets: &VMOffsets<P>,
+    fn lower<M: MacroAssembler>(
+        env: &mut FuncEnv<M::Ptr>,
+        vmoffsets: &VMOffsets<u8>,
         callee: &Callee,
-        sig: &ABISig,
         context: &mut CodeGenContext,
         masm: &mut M,
     ) -> (CalleeKind, ContextArgs) {
         let ptr = vmoffsets.ptr.size();
         match callee {
-            Callee::Builtin(b) => Self::lower_builtin(b, context, masm),
-            Callee::FuncRef(_) => Self::lower_funcref(sig, ptr, context, masm),
-            Callee::Local(i) => Self::lower_local::<M>(i),
-            Callee::Import(i) => Self::lower_import(i, sig, context, masm, vmoffsets),
+            Callee::Builtin(b) => Self::lower_builtin(env, b),
+            Callee::FuncRef(_) => {
+                Self::lower_funcref(env.callee_sig::<M::ABI>(callee), ptr, context, masm)
+            }
+            Callee::Local(i) => Self::lower_local(env, *i),
+            Callee::Import(i) => {
+                let sig = env.callee_sig::<M::ABI>(callee);
+                Self::lower_import(*i, sig, context, masm, vmoffsets)
+            }
         }
     }
 
     /// Lowers a builtin function by loading its address to the next available
     /// register.
-    fn lower_builtin<M: MacroAssembler>(
+    fn lower_builtin<P: PtrSize>(
+        env: &mut FuncEnv<P>,
         builtin: &BuiltinFunction,
-        context: &mut CodeGenContext,
-        masm: &mut M,
     ) -> (CalleeKind, ContextArgs) {
         match builtin.ty() {
-            BuiltinType::Dynamic { index, base } => {
-                let sig = builtin.sig();
-                let callee = context.without::<Reg, _, _>(&sig.regs, masm, |cx, masm| {
-                    let scratch = <M::ABI as ABI>::scratch_reg();
-                    let builtins_base = masm.address_at_vmctx(base);
-                    masm.load_ptr(builtins_base, scratch);
-                    let addr = masm.address_at_reg(scratch, index);
-                    let callee = cx.any_gpr(masm);
-                    masm.load_ptr(addr, callee);
-                    callee
-                });
-                (CalleeKind::indirect(callee), ContextArgs::pinned_vmctx())
-            }
-            BuiltinType::Known(c) => (CalleeKind::known(c), ContextArgs::none()),
+            BuiltinType::Builtin(idx) => (
+                CalleeKind::direct(env.name_builtin(idx)),
+                ContextArgs::pinned_vmctx(),
+            ),
+            BuiltinType::LibCall(c) => (CalleeKind::libcall(c), ContextArgs::none()),
         }
     }
 
     /// Lower  a local function to a [`CalleeKind`] and [ContextArgs] pair.
-    fn lower_local<M: MacroAssembler>(info: &CalleeInfo) -> (CalleeKind, ContextArgs) {
+    fn lower_local<P: PtrSize>(
+        env: &mut FuncEnv<P>,
+        index: FuncIndex,
+    ) -> (CalleeKind, ContextArgs) {
         (
-            CalleeKind::direct(info.index.as_u32()),
+            CalleeKind::direct(env.name_wasm(index)),
             ContextArgs::pinned_callee_and_caller_vmctx(),
         )
     }
@@ -174,7 +174,7 @@ impl FnCall {
     /// Lowers a function import by loading its address to the next available
     /// register.
     fn lower_import<M: MacroAssembler, P: PtrSize>(
-        info: &CalleeInfo,
+        index: FuncIndex,
         sig: &ABISig,
         context: &mut CodeGenContext,
         masm: &mut M,
@@ -184,11 +184,11 @@ impl FnCall {
             context.without::<(Reg, Reg), M, _>(&sig.regs, masm, |context, masm| {
                 (context.any_gpr(masm), context.any_gpr(masm))
             });
-        let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(info.index);
+        let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(index);
         let callee_vmctx_addr = masm.address_at_vmctx(callee_vmctx_offset);
         masm.load_ptr(callee_vmctx_addr, callee_vmctx);
 
-        let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(info.index);
+        let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(index);
         let callee_addr = masm.address_at_vmctx(callee_body_offset);
         masm.load_ptr(callee_addr, callee);
 

winch/codegen/src/codegen/env.rs

@@ -3,15 +3,17 @@ use crate::{
     codegen::{control, BlockSig, BuiltinFunction, BuiltinFunctions, OperandSize},
     isa::TargetIsa,
 };
+use cranelift_codegen::ir::{UserExternalName, UserExternalNameRef};
 use std::collections::{
     hash_map::Entry::{Occupied, Vacant},
     HashMap,
 };
+use std::mem;
 use wasmparser::BlockType;
 use wasmtime_environ::{
-    FuncIndex, GlobalIndex, MemoryIndex, MemoryPlan, MemoryStyle, ModuleTranslation,
-    ModuleTypesBuilder, PtrSize, TableIndex, TablePlan, TypeConvert, TypeIndex, VMOffsets,
-    WasmHeapType, WasmValType, WASM_PAGE_SIZE,
+    BuiltinFunctionIndex, FuncIndex, GlobalIndex, MemoryIndex, MemoryPlan, MemoryStyle,
+    ModuleTranslation, ModuleTypesBuilder, PrimaryMap, PtrSize, TableIndex, TablePlan, TypeConvert,
+    TypeIndex, VMOffsets, WasmHeapType, WasmValType, WASM_PAGE_SIZE,
 };
 
 #[derive(Debug, Clone, Copy)]
@@ -85,38 +87,17 @@ pub struct HeapData {
 /// It categorizes how the callee should be treated
 /// when performing the call.
 #[derive(Clone)]
-pub(crate) enum Callee<'a> {
+pub(crate) enum Callee {
     /// Locally defined function.
-    Local(&'a CalleeInfo),
+    Local(FuncIndex),
     /// Imported function.
-    Import(&'a CalleeInfo),
+    Import(FuncIndex),
     /// Function reference.
-    FuncRef(&'a ABISig),
+    FuncRef(TypeIndex),
     /// A built-in function.
     Builtin(BuiltinFunction),
 }
 
-impl Callee<'_> {
-    /// Returns the [ABISig] of the [Callee].
-    pub(crate) fn sig(&self) -> &ABISig {
-        match self {
-            Self::Local(info) | Self::Import(info) => &info.sig,
-            Self::FuncRef(sig) => sig,
-            Self::Builtin(b) => b.sig(),
-        }
-    }
-}
-
-/// Metadata about a function callee. Used by the code generation to
-/// emit function calls to local or imported functions.
-#[derive(Clone)]
-pub struct CalleeInfo {
-    /// The function's ABI signature.
-    pub(crate) sig: ABISig,
-    /// The callee index in the WebAssembly function index space.
-    pub index: FuncIndex,
-}
-
 /// The function environment.
 ///
 /// Contains all information about the module and runtime that is accessible to
@@ -134,9 +115,9 @@ pub struct FuncEnv<'a, 'translation: 'a, 'data: 'translation, P: PtrSize> {
     resolved_tables: HashMap<TableIndex, TableData>,
     /// Track resolved heap information.
     resolved_heaps: HashMap<MemoryIndex, HeapData>,
-    /// A map from [FunctionIndex] to [CalleeInfo], to keep track of the resolved
+    /// A map from [FunctionIndex] to [ABISig], to keep track of the resolved
     /// function callees.
-    resolved_callees: HashMap<FuncIndex, CalleeInfo>,
+    resolved_callees: HashMap<FuncIndex, ABISig>,
     /// A map from [TypeIndex] to [ABISig], to keep track of the resolved
     /// indirect function signatures.
     resolved_sigs: HashMap<TypeIndex, ABISig>,
@@ -146,6 +127,8 @@ pub struct FuncEnv<'a, 'translation: 'a, 'data: 'translation, P: PtrSize> {
     ptr_type: WasmValType,
     /// Whether or not to enable Spectre mitigation on heap bounds checks.
     heap_access_spectre_mitigation: bool,
+    name_map: PrimaryMap<UserExternalNameRef, UserExternalName>,
+    name_intern: HashMap<UserExternalName, UserExternalNameRef>,
 }
 
 pub fn ptr_type_from_ptr_size(size: u8) -> WasmValType {
@@ -176,6 +159,8 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
             ptr_type,
             heap_access_spectre_mitigation: isa.flags().enable_heap_access_spectre_mitigation(),
             builtins,
+            name_map: Default::default(),
+            name_intern: Default::default(),
         }
     }
 
@@ -185,38 +170,17 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
     }
 
     /// Resolves a [`Callee::FuncRef`] from a type index.
-    pub(crate) fn funcref<A>(&mut self, idx: TypeIndex) -> Callee
-    where
-        A: ABI,
-    {
-        let val = || {
-            let sig_index = self.translation.module.types[idx].unwrap_function();
-            let ty = &self.types[sig_index];
-            let sig = wasm_sig::<A>(ty);
-            sig
-        };
-        Callee::FuncRef(self.resolved_sigs.entry(idx).or_insert_with(val))
+    pub(crate) fn funcref(&mut self, idx: TypeIndex) -> Callee {
+        Callee::FuncRef(idx)
     }
 
     /// Resolves a function [`Callee`] from an index.
-    pub(crate) fn callee_from_index<A>(&mut self, idx: FuncIndex) -> Callee
-    where
-        A: ABI,
-    {
-        let types = self.translation.get_types();
-        let ty = types[types.core_function_at(idx.as_u32())].unwrap_func();
+    pub(crate) fn callee_from_index(&mut self, idx: FuncIndex) -> Callee {
         let import = self.translation.module.is_imported_function(idx);
-        let val = || {
-            let converter = TypeConverter::new(self.translation, self.types);
-            let ty = converter.convert_func_type(&ty);
-            let sig = wasm_sig::<A>(&ty);
-            CalleeInfo { sig, index: idx }
-        };
-
         if import {
-            Callee::Import(self.resolved_callees.entry(idx).or_insert_with(val))
+            Callee::Import(idx)
         } else {
-            Callee::Local(self.resolved_callees.entry(idx).or_insert_with(val))
+            Callee::Local(idx)
         }
     }
 
@@ -349,6 +313,65 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
     pub fn heap_access_spectre_mitigation(&self) -> bool {
         self.heap_access_spectre_mitigation
     }
+
+    pub(crate) fn callee_sig<'b, A>(&'b mut self, callee: &'b Callee) -> &'b ABISig
+    where
+        A: ABI,
+    {
+        match callee {
+            Callee::Local(idx) | Callee::Import(idx) => {
+                let types = self.translation.get_types();
+                let ty = types[types.core_function_at(idx.as_u32())].unwrap_func();
+                let val = || {
+                    let converter = TypeConverter::new(self.translation, self.types);
+                    let ty = converter.convert_func_type(&ty);
+                    wasm_sig::<A>(&ty)
+                };
+                self.resolved_callees.entry(*idx).or_insert_with(val)
+            }
+            Callee::FuncRef(idx) => {
+                let val = || {
+                    let sig_index = self.translation.module.types[*idx].unwrap_function();
+                    let ty = &self.types[sig_index];
+                    let sig = wasm_sig::<A>(ty);
+                    sig
+                };
+                self.resolved_sigs.entry(*idx).or_insert_with(val)
+            }
+            Callee::Builtin(b) => b.sig(),
+        }
+    }
+
+    /// Creates a name to reference the `builtin` provided.
+    pub fn name_builtin(&mut self, builtin: BuiltinFunctionIndex) -> UserExternalNameRef {
+        self.intern_name(UserExternalName {
+            namespace: wasmtime_cranelift::NS_WASMTIME_BUILTIN,
+            index: builtin.index(),
+        })
+    }
+
+    /// Creates a name to reference the wasm function `index` provided.
+    pub fn name_wasm(&mut self, index: FuncIndex) -> UserExternalNameRef {
+        self.intern_name(UserExternalName {
+            namespace: wasmtime_cranelift::NS_WASM_FUNC,
+            index: index.as_u32(),
+        })
+    }
+
+    /// Interns `name` into a `UserExternalNameRef` and ensures that duplicate
+    /// instances of `name` are given a unique name ref index.
+    fn intern_name(&mut self, name: UserExternalName) -> UserExternalNameRef {
+        *self
+            .name_intern
+            .entry(name.clone())
+            .or_insert_with(|| self.name_map.push(name))
+    }
+
+    /// Extracts the name map that was created while translating this function.
+    pub fn take_name_map(&mut self) -> PrimaryMap<UserExternalNameRef, UserExternalName> {
+        self.name_intern.clear();
+        mem::take(&mut self.name_map)
+    }
 }
 
 /// A wrapper struct over a reference to a [ModuleTranslation] and

winch/codegen/src/codegen/mod.rs

@@ -5,7 +5,6 @@ use crate::{
     masm::{ExtendKind, IntCmpKind, MacroAssembler, OperandSize, RegImm, SPOffset, TrapCode},
     stack::TypedReg,
 };
-
 use anyhow::Result;
 use smallvec::SmallVec;
 use wasmparser::{
@@ -433,7 +432,8 @@ where
         // This is safe since the FnCall::emit call below, will ensure
         // that the result register is placed on the value stack.
         self.context.free_reg(elem_value);
-        FnCall::emit::<M, M::Ptr>(
+        FnCall::emit::<M>(
+            &mut self.env,
             self.masm,
             &mut self.context,
             Callee::Builtin(builtin.clone()),

winch/codegen/src/isa/aarch64/mod.rs

@@ -17,6 +17,7 @@ use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};
 use masm::MacroAssembler as Aarch64Masm;
 use target_lexicon::Triple;
 use wasmparser::{FuncValidator, FunctionBody, ValidatorResources};
+use wasmtime_cranelift::CompiledFunction;
 use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, VMOffsets, WasmFuncType};
 
 mod abi;
@@ -90,7 +91,7 @@ impl TargetIsa for Aarch64 {
         types: &ModuleTypesBuilder,
         builtins: &mut BuiltinFunctions,
         validator: &mut FuncValidator<ValidatorResources>,
-    ) -> Result<MachBufferFinalized<Final>> {
+    ) -> Result<CompiledFunction> {
         let pointer_bytes = self.pointer_bytes();
         let vmoffsets = VMOffsets::new(pointer_bytes, &translation.module);
         let mut body = body.get_binary_reader();
@@ -122,7 +123,12 @@ impl TargetIsa for Aarch64 {
         let mut codegen = CodeGen::new(&mut masm, codegen_context, env, abi_sig);
 
         codegen.emit(&mut body, validator)?;
-        Ok(masm.finalize())
+        let names = codegen.env.take_name_map();
+        Ok(CompiledFunction::new(
+            masm.finalize(),
+            names,
+            self.function_alignment(),
+        ))
     }
 
     fn text_section_builder(&self, num_funcs: usize) -> Box<dyn TextSectionBuilder> {

winch/codegen/src/isa/mod.rs

@@ -11,6 +11,7 @@ use std::{
 };
 use target_lexicon::{Architecture, Triple};
 use wasmparser::{FuncValidator, FunctionBody, ValidatorResources};
+use wasmtime_cranelift::CompiledFunction;
 use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, WasmFuncType};
 
 #[cfg(feature = "x64")]
@@ -162,7 +163,7 @@ pub trait TargetIsa: Send + Sync {
         types: &ModuleTypesBuilder,
         builtins: &mut BuiltinFunctions,
         validator: &mut FuncValidator<ValidatorResources>,
-    ) -> Result<MachBufferFinalized<Final>>;
+    ) -> Result<CompiledFunction>;
 
     /// Get the default calling convention of the underlying target triple.
     fn default_call_conv(&self) -> CallConv {

winch/codegen/src/isa/x64/asm.rs

@@ -5,7 +5,6 @@ use crate::{
     masm::{DivKind, ExtendKind, IntCmpKind, OperandSize, RemKind, RoundingMode, ShiftKind},
 };
 use cranelift_codegen::{
-    entity::EntityRef,
     ir::{
         types, ConstantPool, ExternalName, LibCall, MemFlags, Opcode, TrapCode, UserExternalNameRef,
     },
@@ -1252,8 +1251,8 @@ impl Assembler {
     }
 
     /// Emit a call to a locally defined function through an index.
-    pub fn call_with_index(&mut self, index: u32) {
-        let dest = ExternalName::user(UserExternalNameRef::new(index as usize));
+    pub fn call_with_name(&mut self, name: UserExternalNameRef) {
+        let dest = ExternalName::user(name);
         self.emit(Inst::CallKnown {
             dest,
             opcode: Opcode::Call,

winch/codegen/src/isa/x64/masm.rs

@@ -373,8 +373,8 @@ impl Masm for MacroAssembler {
         let callee = load_callee(self);
         match callee {
             CalleeKind::Indirect(reg) => self.asm.call_with_reg(reg),
-            CalleeKind::Direct(idx) => self.asm.call_with_index(idx),
-            CalleeKind::Known(lib) => self.asm.call_with_lib(lib, regs::scratch()),
+            CalleeKind::Direct(idx) => self.asm.call_with_name(idx),
+            CalleeKind::LibCall(lib) => self.asm.call_with_lib(lib, regs::scratch()),
         };
         total_stack
     }

winch/codegen/src/isa/x64/mod.rs

@@ -18,6 +18,7 @@ use cranelift_codegen::{isa::x64::settings as x64_settings, Final, MachBufferFin
 use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};
 use target_lexicon::Triple;
 use wasmparser::{FuncValidator, FunctionBody, ValidatorResources};
+use wasmtime_cranelift::CompiledFunction;
 use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, VMOffsets, WasmFuncType};
 
 use self::regs::{ALL_FPR, ALL_GPR, MAX_FPR, MAX_GPR, NON_ALLOCATABLE_FPR, NON_ALLOCATABLE_GPR};
@@ -93,7 +94,7 @@ impl TargetIsa for X64 {
         types: &ModuleTypesBuilder,
         builtins: &mut BuiltinFunctions,
         validator: &mut FuncValidator<ValidatorResources>,
-    ) -> Result<MachBufferFinalized<Final>> {
+    ) -> Result<CompiledFunction> {
         let pointer_bytes = self.pointer_bytes();
         let vmoffsets = VMOffsets::new(pointer_bytes, &translation.module);
 
@@ -136,7 +137,12 @@ impl TargetIsa for X64 {
 
         codegen.emit(&mut body, validator)?;
 
-        Ok(masm.finalize())
+        let names = codegen.env.take_name_map();
+        Ok(CompiledFunction::new(
+            masm.finalize(),
+            names,
+            self.function_alignment(),
+        ))
     }
 
     fn text_section_builder(&self, num_funcs: usize) -> Box<dyn TextSectionBuilder> {

winch/codegen/src/masm.rs

@@ -2,7 +2,7 @@ use crate::abi::{self, align_to, LocalSlot};
 use crate::codegen::{CodeGenContext, FuncEnv, HeapData, TableData};
 use crate::isa::reg::Reg;
 use cranelift_codegen::{
-    ir::{Endianness, LibCall, MemFlags},
+    ir::{Endianness, LibCall, MemFlags, UserExternalNameRef},
     Final, MachBufferFinalized, MachLabel,
 };
 use std::{fmt::Debug, ops::Range};
@@ -365,9 +365,9 @@ pub(crate) enum CalleeKind {
     /// A function call to a raw address.
     Indirect(Reg),
     /// A function call to a local function.
-    Direct(u32),
+    Direct(UserExternalNameRef),
     /// Call to a well known LibCall.
-    Known(LibCall),
+    LibCall(LibCall),
 }
 
 impl CalleeKind {
@@ -376,14 +376,14 @@ impl CalleeKind {
         Self::Indirect(reg)
     }
 
-    /// Creates a direct callee kind from a function index.
-    pub fn direct(index: u32) -> Self {
-        Self::Direct(index)
+    /// Creates a direct callee kind from a function name.
+    pub fn direct(name: UserExternalNameRef) -> Self {
+        Self::Direct(name)
     }
 
     /// Creates a known callee kind from a libcall.
-    pub fn known(call: LibCall) -> Self {
-        Self::Known(call)
+    pub fn libcall(call: LibCall) -> Self {
+        Self::LibCall(call)
     }
 }
 

winch/codegen/src/visitor.rs

@@ -461,7 +461,7 @@ where
             OperandSize::S32,
             |env, cx, masm| {
                 let builtin = env.builtins.floor_f32::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -474,7 +474,7 @@ where
             OperandSize::S64,
             |env, cx, masm| {
                 let builtin = env.builtins.floor_f64::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -487,7 +487,7 @@ where
             OperandSize::S32,
             |env, cx, masm| {
                 let builtin = env.builtins.ceil_f32::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -500,7 +500,7 @@ where
             OperandSize::S64,
             |env, cx, masm| {
                 let builtin = env.builtins.ceil_f64::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -513,7 +513,7 @@ where
             OperandSize::S32,
             |env, cx, masm| {
                 let builtin = env.builtins.nearest_f32::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin))
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin))
             },
         );
     }
@@ -526,7 +526,7 @@ where
             OperandSize::S64,
             |env, cx, masm| {
                 let builtin = env.builtins.nearest_f64::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -539,7 +539,7 @@ where
             OperandSize::S32,
             |env, cx, masm| {
                 let builtin = env.builtins.trunc_f32::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -552,7 +552,7 @@ where
             OperandSize::S64,
             |env, cx, masm| {
                 let builtin = env.builtins.trunc_f64::<M::ABI>();
-                FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin));
+                FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin));
             },
         );
     }
@@ -1353,12 +1353,8 @@ where
     }
 
     fn visit_call(&mut self, index: u32) {
-        FnCall::emit::<M, M::Ptr>(
-            self.masm,
-            &mut self.context,
-            self.env
-                .callee_from_index::<M::ABI>(FuncIndex::from_u32(index)),
-        )
+        let callee = self.env.callee_from_index(FuncIndex::from_u32(index));
+        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)
     }
 
     fn visit_call_indirect(&mut self, type_index: u32, table_index: u32, _: u8) {
@@ -1387,11 +1383,8 @@ where
             }
         }
 
-        FnCall::emit::<M, M::Ptr>(
-            self.masm,
-            &mut self.context,
-            self.env.funcref::<M::ABI>(type_index),
-        )
+        let callee = self.env.funcref(type_index);
+        FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)
     }
 
     fn visit_table_init(&mut self, elem: u32, table: u32) {
@@ -1403,7 +1396,8 @@ where
             .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]);
 
         let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>();
-        FnCall::emit::<M, M::Ptr>(
+        FnCall::emit::<M>(
+            &mut self.env,
             self.masm,
             &mut self.context,
             Callee::Builtin(builtin.clone()),
@@ -1418,7 +1412,12 @@ where
             .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]);
 
         let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>();
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin))
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(builtin),
+        )
     }
 
     fn visit_table_get(&mut self, table: u32) {
@@ -1459,7 +1458,8 @@ where
             .stack
             .insert_many(at, &[table.try_into().unwrap()]);
 
-        FnCall::emit::<M, M::Ptr>(
+        FnCall::emit::<M>(
+            &mut self.env,
             self.masm,
             &mut self.context,
             Callee::Builtin(builtin.clone()),
@@ -1486,7 +1486,8 @@ where
         self.context
             .stack
             .insert_many(at, &[table.try_into().unwrap()]);
-        FnCall::emit::<M, M::Ptr>(
+        FnCall::emit::<M>(
+            &mut self.env,
             self.masm,
             &mut self.context,
             Callee::Builtin(builtin.clone()),
@@ -1533,7 +1534,12 @@ where
     fn visit_elem_drop(&mut self, index: u32) {
         let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>();
         self.context.stack.extend([index.try_into().unwrap()]);
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(elem_drop))
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(elem_drop),
+        )
     }
 
     fn visit_memory_init(&mut self, data_index: u32, mem: u32) {
@@ -1544,7 +1550,12 @@ where
             &[mem.try_into().unwrap(), data_index.try_into().unwrap()],
         );
         let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>();
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin))
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(builtin),
+        )
     }
 
     fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) {
@@ -1567,7 +1578,12 @@ where
 
         let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>();
 
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin))
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(builtin),
+        )
     }
 
     fn visit_memory_fill(&mut self, mem: u32) {
@@ -1579,7 +1595,12 @@ where
             .insert_many(at, &[mem.try_into().unwrap()]);
 
         let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>();
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin))
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(builtin),
+        )
     }
 
     fn visit_memory_size(&mut self, mem: u32, _: u8) {
@@ -1596,7 +1617,12 @@ where
 
         let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem));
         let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>();
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin));
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(builtin),
+        );
 
         // The memory32_grow builtin returns a pointer type, therefore we must
         // ensure that the return type is representative of the address space of
@@ -1618,7 +1644,12 @@ where
         self.context.stack.extend([data_index.try_into().unwrap()]);
 
         let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>();
-        FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin))
+        FnCall::emit::<M>(
+            &mut self.env,
+            self.masm,
+            &mut self.context,
+            Callee::Builtin(builtin),
+        )
     }
 
     fn visit_nop(&mut self) {}

winch/filetests/filetests/x64/call_indirect/call_indirect.wat

@@ -35,7 +35,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c330000000       	add	r11, 0x30
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f87b7010000         	ja	0x1d2
+;;      	 0f87ad010000         	ja	0x1c8
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec18             	sub	rsp, 0x18
 ;;      	 48897c2410           	mov	qword ptr [rsp + 0x10], rdi
@@ -48,7 +48,7 @@
 ;;      	 85c0                 	test	eax, eax
 ;;      	 0f840a000000         	je	0x52
 ;;   48:	 b801000000           	mov	eax, 1
-;;      	 e97a010000           	jmp	0x1cc
+;;      	 e970010000           	jmp	0x1c2
 ;;   52:	 8b442404             	mov	eax, dword ptr [rsp + 4]
 ;;      	 83e802               	sub	eax, 2
 ;;      	 4883ec04             	sub	rsp, 4
@@ -57,7 +57,7 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f8361010000         	jae	0x1d4
+;;      	 0f8357010000         	jae	0x1ca
 ;;   73:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
@@ -67,27 +67,25 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f852a000000         	jne	0xc0
-;;   96:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f8525000000         	jne	0xbb
+;;   96:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b1424               	mov	edx, dword ptr [rsp]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0xad
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742414           	mov	r14, qword ptr [rsp + 0x14]
-;;      	 e904000000           	jmp	0xc4
-;;   c0:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0xbf
+;;   bb:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f8409010000         	je	0x1d6
-;;   cd:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
+;;      	 0f8404010000         	je	0x1cc
+;;   c8:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
 ;;      	 418b0b               	mov	ecx, dword ptr [r11]
 ;;      	 8b5018               	mov	edx, dword ptr [rax + 0x18]
 ;;      	 39d1                 	cmp	ecx, edx
-;;      	 0f85f9000000         	jne	0x1d8
-;;   df:	 50                   	push	rax
+;;      	 0f85f4000000         	jne	0x1ce
+;;   da:	 50                   	push	rax
 ;;      	 59                   	pop	rcx
 ;;      	 4c8b4120             	mov	r8, qword ptr [rcx + 0x20]
 ;;      	 488b5910             	mov	rbx, qword ptr [rcx + 0x10]
@@ -109,8 +107,8 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f83ac000000         	jae	0x1da
-;;  12e:	 4189cb               	mov	r11d, ecx
+;;      	 0f83a7000000         	jae	0x1d0
+;;  129:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
 ;;      	 4889d6               	mov	rsi, rdx
@@ -119,29 +117,27 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f8533000000         	jne	0x184
-;;  151:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f852e000000         	jne	0x17a
+;;  14c:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4883ec0c             	sub	rsp, 0xc
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b54240c             	mov	edx, dword ptr [rsp + 0xc]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0x168
 ;;      	 4883c40c             	add	rsp, 0xc
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742418           	mov	r14, qword ptr [rsp + 0x18]
-;;      	 e904000000           	jmp	0x188
-;;  184:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0x17e
+;;  17a:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f844b000000         	je	0x1dc
-;;  191:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
+;;      	 0f844b000000         	je	0x1d2
+;;  187:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
 ;;      	 418b0b               	mov	ecx, dword ptr [r11]
 ;;      	 8b5018               	mov	edx, dword ptr [rax + 0x18]
 ;;      	 39d1                 	cmp	ecx, edx
-;;      	 0f853b000000         	jne	0x1de
-;;  1a3:	 50                   	push	rax
+;;      	 0f853b000000         	jne	0x1d4
+;;  199:	 50                   	push	rax
 ;;      	 59                   	pop	rcx
 ;;      	 4c8b4120             	mov	r8, qword ptr [rcx + 0x20]
 ;;      	 488b5910             	mov	rbx, qword ptr [rcx + 0x10]
@@ -158,10 +154,10 @@
 ;;      	 4883c418             	add	rsp, 0x18
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
+;;  1c8:	 0f0b                 	ud2	
+;;  1ca:	 0f0b                 	ud2	
+;;  1cc:	 0f0b                 	ud2	
+;;  1ce:	 0f0b                 	ud2	
+;;  1d0:	 0f0b                 	ud2	
 ;;  1d2:	 0f0b                 	ud2	
 ;;  1d4:	 0f0b                 	ud2	
-;;  1d6:	 0f0b                 	ud2	
-;;  1d8:	 0f0b                 	ud2	
-;;  1da:	 0f0b                 	ud2	
-;;  1dc:	 0f0b                 	ud2	
-;;  1de:	 0f0b                 	ud2	

winch/filetests/filetests/x64/call_indirect/local_arg.wat

@@ -39,7 +39,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c320000000       	add	r11, 0x20
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f87d0000000         	ja	0xeb
+;;      	 0f87cb000000         	ja	0xe6
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec18             	sub	rsp, 0x18
 ;;      	 48897c2410           	mov	qword ptr [rsp + 0x10], rdi
@@ -52,7 +52,7 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f8399000000         	jae	0xed
+;;      	 0f8394000000         	jae	0xe8
 ;;   54:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
@@ -62,27 +62,25 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f852a000000         	jne	0xa1
-;;   77:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f8525000000         	jne	0x9c
+;;   77:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b1424               	mov	edx, dword ptr [rsp]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0x8e
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742414           	mov	r14, qword ptr [rsp + 0x14]
-;;      	 e904000000           	jmp	0xa5
-;;   a1:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0xa0
+;;   9c:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f8441000000         	je	0xef
-;;   ae:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
+;;      	 0f8441000000         	je	0xea
+;;   a9:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
 ;;      	 418b0b               	mov	ecx, dword ptr [r11]
 ;;      	 8b5018               	mov	edx, dword ptr [rax + 0x18]
 ;;      	 39d1                 	cmp	ecx, edx
-;;      	 0f8531000000         	jne	0xf1
-;;   c0:	 488b5820             	mov	rbx, qword ptr [rax + 0x20]
+;;      	 0f8531000000         	jne	0xec
+;;   bb:	 488b5820             	mov	rbx, qword ptr [rax + 0x20]
 ;;      	 488b4810             	mov	rcx, qword ptr [rax + 0x10]
 ;;      	 4883ec04             	sub	rsp, 4
 ;;      	 4889df               	mov	rdi, rbx
@@ -95,7 +93,7 @@
 ;;      	 4883c418             	add	rsp, 0x18
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;   eb:	 0f0b                 	ud2	
-;;   ed:	 0f0b                 	ud2	
-;;   ef:	 0f0b                 	ud2	
-;;   f1:	 0f0b                 	ud2	
+;;   e6:	 0f0b                 	ud2	
+;;   e8:	 0f0b                 	ud2	
+;;   ea:	 0f0b                 	ud2	
+;;   ec:	 0f0b                 	ud2	

winch/filetests/filetests/x64/load/grow_load.wat

@@ -32,7 +32,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c360000000       	add	r11, 0x60
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f87f1000000         	ja	0x10c
+;;      	 0f87ed000000         	ja	0x108
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec50             	sub	rsp, 0x50
 ;;      	 48897c2448           	mov	qword ptr [rsp + 0x48], rdi
@@ -59,13 +59,11 @@
 ;;      	 c1e810               	shr	eax, 0x10
 ;;      	 4883ec04             	sub	rsp, 4
 ;;      	 890424               	mov	dword ptr [rsp], eax
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b0b               	mov	rcx, qword ptr [r11]
 ;;      	 4883ec0c             	sub	rsp, 0xc
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 8b74240c             	mov	esi, dword ptr [rsp + 0xc]
 ;;      	 ba00000000           	mov	edx, 0
-;;      	 ffd1                 	call	rcx
+;;      	 e800000000           	call	0xa0
 ;;      	 4883c40c             	add	rsp, 0xc
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742448           	mov	r14, qword ptr [rsp + 0x48]
@@ -74,13 +72,13 @@
 ;;      	 4801c1               	add	rcx, rax
 ;;      	 4881c124300200       	add	rcx, 0x23024
 ;;      	 480fbe01             	movsx	rax, byte ptr [rcx]
-;;      	 f30f100543000000     	movss	xmm0, dword ptr [rip + 0x43]
+;;      	 f30f100547000000     	movss	xmm0, dword ptr [rip + 0x47]
 ;;      	 4883ec0c             	sub	rsp, 0xc
-;;      	 f2440f103d3e000000   	
-;; 				movsd	xmm15, qword ptr [rip + 0x3e]
+;;      	 f2440f103d42000000   	
+;; 				movsd	xmm15, qword ptr [rip + 0x42]
 ;;      	 f2440f113c24         	movsd	qword ptr [rsp], xmm15
-;;      	 f3440f103d27000000   	
-;; 				movss	xmm15, dword ptr [rip + 0x27]
+;;      	 f3440f103d2b000000   	
+;; 				movss	xmm15, dword ptr [rip + 0x2b]
 ;;      	 f3440f117c2408       	movss	dword ptr [rsp + 8], xmm15
 ;;      	 488b44240c           	mov	rax, qword ptr [rsp + 0xc]
 ;;      	 415b                 	pop	r11
@@ -91,7 +89,9 @@
 ;;      	 4883c450             	add	rsp, 0x50
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;  10c:	 0f0b                 	ud2	
+;;  108:	 0f0b                 	ud2	
+;;  10a:	 0000                 	add	byte ptr [rax], al
+;;  10c:	 0000                 	add	byte ptr [rax], al
 ;;  10e:	 0000                 	add	byte ptr [rax], al
 ;;  110:	 0000                 	add	byte ptr [rax], al
 ;;  112:	 0000                 	add	byte ptr [rax], al

winch/filetests/filetests/x64/table/fill.wat

@@ -73,7 +73,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c340000000       	add	r11, 0x40
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f8707010000         	ja	0x122
+;;      	 0f87fd000000         	ja	0x118
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec28             	sub	rsp, 0x28
 ;;      	 48897c2420           	mov	qword ptr [rsp + 0x20], rdi
@@ -91,7 +91,7 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f83b9000000         	jae	0x124
+;;      	 0f83af000000         	jae	0x11a
 ;;   6b:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
@@ -101,24 +101,20 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f8533000000         	jne	0xc1
-;;   8e:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f852e000000         	jne	0xbc
+;;   8e:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4883ec04             	sub	rsp, 4
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b542404             	mov	edx, dword ptr [rsp + 4]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0xaa
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742420           	mov	r14, qword ptr [rsp + 0x20]
-;;      	 e904000000           	jmp	0xc5
-;;   c1:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0xc0
+;;   bc:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 4889442404           	mov	qword ptr [rsp + 4], rax
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4358             	mov	rax, qword ptr [r11 + 0x58]
 ;;      	 448b5c2414           	mov	r11d, dword ptr [rsp + 0x14]
 ;;      	 4883ec04             	sub	rsp, 4
 ;;      	 44891c24             	mov	dword ptr [rsp], r11d
@@ -133,12 +129,12 @@
 ;;      	 8b542414             	mov	edx, dword ptr [rsp + 0x14]
 ;;      	 488b4c240c           	mov	rcx, qword ptr [rsp + 0xc]
 ;;      	 448b442408           	mov	r8d, dword ptr [rsp + 8]
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x105
 ;;      	 4883c408             	add	rsp, 8
 ;;      	 4883c410             	add	rsp, 0x10
 ;;      	 4c8b742420           	mov	r14, qword ptr [rsp + 0x20]
 ;;      	 4883c428             	add	rsp, 0x28
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;  122:	 0f0b                 	ud2	
-;;  124:	 0f0b                 	ud2	
+;;  118:	 0f0b                 	ud2	
+;;  11a:	 0f0b                 	ud2	

winch/filetests/filetests/x64/table/get.wat

@@ -31,7 +31,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c320000000       	add	r11, 0x20
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f8797000000         	ja	0xb2
+;;      	 0f8792000000         	ja	0xad
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec18             	sub	rsp, 0x18
 ;;      	 48897c2410           	mov	qword ptr [rsp + 0x10], rdi
@@ -45,7 +45,7 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f8362000000         	jae	0xb4
+;;      	 0f835d000000         	jae	0xaf
 ;;   52:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
@@ -55,23 +55,21 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f8533000000         	jne	0xa8
-;;   75:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f852e000000         	jne	0xa3
+;;   75:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4883ec04             	sub	rsp, 4
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b542404             	mov	edx, dword ptr [rsp + 4]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0x91
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742410           	mov	r14, qword ptr [rsp + 0x10]
-;;      	 e904000000           	jmp	0xac
-;;   a8:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0xa7
+;;   a3:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 4883c418             	add	rsp, 0x18
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;   b2:	 0f0b                 	ud2	
-;;   b4:	 0f0b                 	ud2	
+;;   ad:	 0f0b                 	ud2	
+;;   af:	 0f0b                 	ud2	

winch/filetests/filetests/x64/table/grow.wat

@@ -15,24 +15,22 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c320000000       	add	r11, 0x20
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f8745000000         	ja	0x60
+;;      	 0f8740000000         	ja	0x5b
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec18             	sub	rsp, 0x18
 ;;      	 48897c2410           	mov	qword ptr [rsp + 0x10], rdi
 ;;      	 4889742408           	mov	qword ptr [rsp + 8], rsi
 ;;      	 48891424             	mov	qword ptr [rsp], rdx
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b50             	mov	rbx, qword ptr [r11 + 0x50]
 ;;      	 4c8b1c24             	mov	r11, qword ptr [rsp]
 ;;      	 4153                 	push	r11
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba0a000000           	mov	edx, 0xa
 ;;      	 488b0c24             	mov	rcx, qword ptr [rsp]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0x4c
 ;;      	 4883c408             	add	rsp, 8
 ;;      	 4c8b742410           	mov	r14, qword ptr [rsp + 0x10]
 ;;      	 4883c418             	add	rsp, 0x18
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;   60:	 0f0b                 	ud2	
+;;   5b:	 0f0b                 	ud2	

winch/filetests/filetests/x64/table/init_copy_drop.wat

@@ -124,97 +124,79 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c310000000       	add	r11, 0x10
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f877f010000         	ja	0x19a
+;;      	 0f8752010000         	ja	0x16d
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec10             	sub	rsp, 0x10
 ;;      	 48897c2408           	mov	qword ptr [rsp + 8], rdi
 ;;      	 48893424             	mov	qword ptr [rsp], rsi
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4310             	mov	rax, qword ptr [r11 + 0x10]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba01000000           	mov	edx, 1
 ;;      	 b907000000           	mov	ecx, 7
 ;;      	 41b800000000         	mov	r8d, 0
 ;;      	 41b904000000         	mov	r9d, 4
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x4e
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4318             	mov	rax, qword ptr [r11 + 0x18]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be01000000           	mov	esi, 1
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x60
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4310             	mov	rax, qword ptr [r11 + 0x10]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba03000000           	mov	edx, 3
 ;;      	 b90f000000           	mov	ecx, 0xf
 ;;      	 41b801000000         	mov	r8d, 1
 ;;      	 41b903000000         	mov	r9d, 3
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x88
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4318             	mov	rax, qword ptr [r11 + 0x18]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be03000000           	mov	esi, 3
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x9a
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4308             	mov	rax, qword ptr [r11 + 8]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba00000000           	mov	edx, 0
 ;;      	 b914000000           	mov	ecx, 0x14
 ;;      	 41b80f000000         	mov	r8d, 0xf
 ;;      	 41b905000000         	mov	r9d, 5
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0xc2
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4308             	mov	rax, qword ptr [r11 + 8]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba00000000           	mov	edx, 0
 ;;      	 b915000000           	mov	ecx, 0x15
 ;;      	 41b81d000000         	mov	r8d, 0x1d
 ;;      	 41b901000000         	mov	r9d, 1
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0xea
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4308             	mov	rax, qword ptr [r11 + 8]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba00000000           	mov	edx, 0
 ;;      	 b918000000           	mov	ecx, 0x18
 ;;      	 41b80a000000         	mov	r8d, 0xa
 ;;      	 41b901000000         	mov	r9d, 1
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x112
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4308             	mov	rax, qword ptr [r11 + 8]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba00000000           	mov	edx, 0
 ;;      	 b90d000000           	mov	ecx, 0xd
 ;;      	 41b80b000000         	mov	r8d, 0xb
 ;;      	 41b904000000         	mov	r9d, 4
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x13a
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
-;;      	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b4308             	mov	rax, qword ptr [r11 + 8]
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 ba00000000           	mov	edx, 0
 ;;      	 b913000000           	mov	ecx, 0x13
 ;;      	 41b814000000         	mov	r8d, 0x14
 ;;      	 41b905000000         	mov	r9d, 5
-;;      	 ffd0                 	call	rax
+;;      	 e800000000           	call	0x162
 ;;      	 4c8b742408           	mov	r14, qword ptr [rsp + 8]
 ;;      	 4883c410             	add	rsp, 0x10
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;  19a:	 0f0b                 	ud2	
+;;  16d:	 0f0b                 	ud2	
 ;;
 ;;      	 55                   	push	rbp
 ;;      	 4889e5               	mov	rbp, rsp
@@ -222,7 +204,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c320000000       	add	r11, 0x20
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f87d7000000         	ja	0xf2
+;;      	 0f87d2000000         	ja	0xed
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec18             	sub	rsp, 0x18
 ;;      	 48897c2410           	mov	qword ptr [rsp + 0x10], rdi
@@ -236,7 +218,7 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b9af0000000         	mov	ebx, dword ptr [rdx + 0xf0]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f839f000000         	jae	0xf4
+;;      	 0f839a000000         	jae	0xef
 ;;   55:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b92e8000000       	mov	rdx, qword ptr [rdx + 0xe8]
@@ -246,29 +228,27 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f8533000000         	jne	0xae
-;;   7b:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f852e000000         	jne	0xa9
+;;   7b:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4883ec04             	sub	rsp, 4
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b542404             	mov	edx, dword ptr [rsp + 4]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0x97
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742410           	mov	r14, qword ptr [rsp + 0x10]
-;;      	 e904000000           	jmp	0xb2
-;;   ae:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0xad
+;;   a9:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f843b000000         	je	0xf6
-;;   bb:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
+;;      	 0f843b000000         	je	0xf1
+;;   b6:	 4d8b5e40             	mov	r11, qword ptr [r14 + 0x40]
 ;;      	 418b0b               	mov	ecx, dword ptr [r11]
 ;;      	 8b5018               	mov	edx, dword ptr [rax + 0x18]
 ;;      	 39d1                 	cmp	ecx, edx
-;;      	 0f852b000000         	jne	0xf8
-;;   cd:	 50                   	push	rax
+;;      	 0f852b000000         	jne	0xf3
+;;   c8:	 50                   	push	rax
 ;;      	 59                   	pop	rcx
 ;;      	 488b5920             	mov	rbx, qword ptr [rcx + 0x20]
 ;;      	 488b5110             	mov	rdx, qword ptr [rcx + 0x10]
@@ -281,7 +261,7 @@
 ;;      	 4883c418             	add	rsp, 0x18
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;   f2:	 0f0b                 	ud2	
-;;   f4:	 0f0b                 	ud2	
-;;   f6:	 0f0b                 	ud2	
-;;   f8:	 0f0b                 	ud2	
+;;   ed:	 0f0b                 	ud2	
+;;   ef:	 0f0b                 	ud2	
+;;   f1:	 0f0b                 	ud2	
+;;   f3:	 0f0b                 	ud2	

winch/filetests/filetests/x64/table/set.wat

@@ -70,7 +70,7 @@
 ;;      	 4d8b1b               	mov	r11, qword ptr [r11]
 ;;      	 4981c320000000       	add	r11, 0x20
 ;;      	 4939e3               	cmp	r11, rsp
-;;      	 0f87d1000000         	ja	0xec
+;;      	 0f87cc000000         	ja	0xe7
 ;;   1b:	 4989fe               	mov	r14, rdi
 ;;      	 4883ec18             	sub	rsp, 0x18
 ;;      	 48897c2410           	mov	qword ptr [rsp + 0x10], rdi
@@ -88,7 +88,7 @@
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f838c000000         	jae	0xee
+;;      	 0f8387000000         	jae	0xe9
 ;;   62:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
@@ -98,26 +98,24 @@
 ;;      	 480f43d6             	cmovae	rdx, rsi
 ;;      	 488b02               	mov	rax, qword ptr [rdx]
 ;;      	 4885c0               	test	rax, rax
-;;      	 0f852a000000         	jne	0xaf
-;;   85:	 4d8b5e38             	mov	r11, qword ptr [r14 + 0x38]
-;;      	 498b5b48             	mov	rbx, qword ptr [r11 + 0x48]
-;;      	 4883ec04             	sub	rsp, 4
+;;      	 0f8525000000         	jne	0xaa
+;;   85:	 4883ec04             	sub	rsp, 4
 ;;      	 890c24               	mov	dword ptr [rsp], ecx
 ;;      	 4c89f7               	mov	rdi, r14
 ;;      	 be00000000           	mov	esi, 0
 ;;      	 8b1424               	mov	edx, dword ptr [rsp]
-;;      	 ffd3                 	call	rbx
+;;      	 e800000000           	call	0x9c
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c8b742414           	mov	r14, qword ptr [rsp + 0x14]
-;;      	 e904000000           	jmp	0xb3
-;;   af:	 4883e0fe             	and	rax, 0xfffffffffffffffe
+;;      	 e904000000           	jmp	0xae
+;;   aa:	 4883e0fe             	and	rax, 0xfffffffffffffffe
 ;;      	 8b0c24               	mov	ecx, dword ptr [rsp]
 ;;      	 4883c404             	add	rsp, 4
 ;;      	 4c89f2               	mov	rdx, r14
 ;;      	 8b5a50               	mov	ebx, dword ptr [rdx + 0x50]
 ;;      	 39d9                 	cmp	ecx, ebx
-;;      	 0f8328000000         	jae	0xf0
-;;   c8:	 4189cb               	mov	r11d, ecx
+;;      	 0f8328000000         	jae	0xeb
+;;   c3:	 4189cb               	mov	r11d, ecx
 ;;      	 4d6bdb08             	imul	r11, r11, 8
 ;;      	 488b5248             	mov	rdx, qword ptr [rdx + 0x48]
 ;;      	 4889d6               	mov	rsi, rdx
@@ -129,6 +127,6 @@
 ;;      	 4883c418             	add	rsp, 0x18
 ;;      	 5d                   	pop	rbp
 ;;      	 c3                   	ret	
-;;   ec:	 0f0b                 	ud2	
-;;   ee:	 0f0b                 	ud2	
-;;   f0:	 0f0b                 	ud2	
+;;   e7:	 0f0b                 	ud2	
+;;   e9:	 0f0b                 	ud2	
+;;   eb:	 0f0b                 	ud2	

winch/filetests/src/lib.rs

@@ -173,7 +173,7 @@ mod test {
             )
             .expect("Couldn't compile function");
 
-        disasm(buffer.data(), isa, OffsetStyle::Minimal).unwrap()
+        disasm(buffer.buffer.data(), isa, OffsetStyle::Minimal).unwrap()
     }
 
     struct DummyConvert;

winch/src/compile.rs

@@ -71,7 +71,7 @@ fn compile(
         .expect("Couldn't compile function");
 
     println!("Disassembly for function: {}", index.as_u32());
-    disasm(buffer.data(), isa, OffsetStyle::Full)?
+    disasm(buffer.buffer.data(), isa, OffsetStyle::Full)?
         .iter()
         .for_each(|s| println!("{}", s));