Implement the `memory.copy` instruction from the bulk memory proposal

crates/environ/src/func_environ.rs

@@ -70,9 +70,17 @@ impl BuiltinFunctionIndex {
     pub const fn get_elem_drop_index() -> Self {
         Self(9)
     }
+    /// Returns an index for wasm's `memory.copy` for locally defined memories.
+    pub const fn get_memory_copy_index() -> Self {
+        Self(10)
+    }
+    /// Returns an index for wasm's `memory.copy` for imported memories.
+    pub const fn get_imported_memory_copy_index() -> Self {
+        Self(11)
+    }
     /// Returns the total number of builtin functions.
     pub const fn builtin_functions_total_number() -> u32 {
-        10
+        12
     }
 
     /// Return the index as an u32 number.
@@ -110,6 +118,10 @@ pub struct FuncEnvironment<'module_environment> {
     /// The external function signature for implementing wasm's `elem.drop`.
     elem_drop_sig: Option<ir::SigRef>,
 
+    /// The external function signature for implementing wasm's `memory.copy`
+    /// (it's the same for both local and imported memories).
+    memory_copy_sig: Option<ir::SigRef>,
+
     /// Offsets to struct fields accessed by JIT code.
     offsets: VMOffsets,
 }
@@ -128,6 +140,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             table_copy_sig: None,
             table_init_sig: None,
             elem_drop_sig: None,
+            memory_copy_sig: None,
             offsets: VMOffsets::new(target_config.pointer_bytes(), module),
         }
     }
@@ -371,6 +384,51 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         (sig, BuiltinFunctionIndex::get_elem_drop_index())
     }
 
+    fn get_memory_copy_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.memory_copy_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Memory index.
+                    AbiParam::new(I32),
+                    // Destination address.
+                    AbiParam::new(I32),
+                    // Source address.
+                    AbiParam::new(I32),
+                    // Length.
+                    AbiParam::new(I32),
+                    // Source location.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.memory_copy_sig = Some(sig);
+        sig
+    }
+
+    fn get_memory_copy_func(
+        &mut self,
+        func: &mut Function,
+        memory_index: MemoryIndex,
+    ) -> (ir::SigRef, usize, BuiltinFunctionIndex) {
+        let sig = self.get_memory_copy_sig(func);
+        if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) {
+            (
+                sig,
+                defined_memory_index.index(),
+                BuiltinFunctionIndex::get_memory_copy_index(),
+            )
+        } else {
+            (
+                sig,
+                memory_index.index(),
+                BuiltinFunctionIndex::get_imported_memory_copy_index(),
+            )
+        }
+    }
+
     /// Translates load of builtin function and returns a pair of values `vmctx`
     /// and address of the loaded function.
     fn translate_load_builtin_function_address(
@@ -968,16 +1026,30 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
     fn translate_memory_copy(
         &mut self,
-        _pos: FuncCursor,
-        _index: MemoryIndex,
+        mut pos: FuncCursor,
+        memory_index: MemoryIndex,
         _heap: ir::Heap,
-        _dst: ir::Value,
-        _src: ir::Value,
-        _len: ir::Value,
+        dst: ir::Value,
+        src: ir::Value,
+        len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported(
-            "bulk memory: `memory.copy`".to_string(),
-        ))
+        let (func_sig, memory_index, func_idx) =
+            self.get_memory_copy_func(&mut pos.func, memory_index);
+
+        let memory_index_arg = pos.ins().iconst(I32, memory_index as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        let src_loc = pos.srcloc();
+        let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64);
+
+        pos.ins().call_indirect(
+            func_sig,
+            func_addr,
+            &[vmctx, memory_index_arg, dst, src, len, src_loc_arg],
+        );
+
+        Ok(())
     }
 
     fn translate_memory_fill(
@@ -1005,7 +1077,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _len: ir::Value,
     ) -> WasmResult<()> {
         Err(WasmError::Unsupported(
-            "bulk memory: `memory.copy`".to_string(),
+            "bulk memory: `memory.init`".to_string(),
         ))
     }
 

crates/runtime/src/instance.rs

@@ -15,6 +15,7 @@ use crate::vmcontext::{
     VMTableDefinition, VMTableImport,
 };
 use crate::{TrapDescription, TrapRegistration};
+use backtrace::Backtrace;
 use memoffset::offset_of;
 use more_asserts::assert_lt;
 use std::alloc::{self, Layout};
@@ -601,7 +602,7 @@ impl Instance {
                     source_loc,
                     trap_code: ir::TrapCode::TableOutOfBounds,
                 },
-                backtrace: backtrace::Backtrace::new(),
+                backtrace: Backtrace::new(),
             });
         }
 
@@ -625,6 +626,71 @@ impl Instance {
         }
     }
 
+    /// Do a `memory.copy` for a locally defined memory.
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Trap` error when the source or destination ranges are out of
+    /// bounds.
+    pub(crate) fn defined_memory_copy(
+        &self,
+        memory_index: DefinedMemoryIndex,
+        dst: u32,
+        src: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy
+
+        let memory = self.memory(memory_index);
+        if src
+            .checked_add(len)
+            .map_or(true, |n| n as usize > memory.current_length)
+            || dst
+                .checked_add(len)
+                .map_or(true, |m| m as usize > memory.current_length)
+        {
+            return Err(Trap::Wasm {
+                desc: TrapDescription {
+                    source_loc,
+                    trap_code: ir::TrapCode::HeapOutOfBounds,
+                },
+                backtrace: Backtrace::new(),
+            });
+        }
+
+        let dst = isize::try_from(dst).unwrap();
+        let src = isize::try_from(src).unwrap();
+
+        // Bounds and casts are checked above, by this point we know that
+        // everything is safe.
+        unsafe {
+            let dst = memory.base.offset(dst);
+            let src = memory.base.offset(src);
+            ptr::copy(src, dst, len as usize);
+        }
+
+        Ok(())
+    }
+
+    /// Perform a `memory.copy` on an imported memory.
+    pub(crate) fn imported_memory_copy(
+        &self,
+        memory_index: MemoryIndex,
+        dst: u32,
+        src: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        let import = self.imported_memory(memory_index);
+        unsafe {
+            let foreign_instance = (&*import.vmctx).instance();
+            let foreign_memory = &*import.from;
+            let foreign_index = foreign_instance.memory_index(foreign_memory);
+            foreign_instance.defined_memory_copy(foreign_index, dst, src, len, source_loc)
+        }
+    }
+
     /// Get a table by index regardless of whether it is locally-defined or an
     /// imported, foreign table.
     pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table {

crates/runtime/src/libcalls.rs

@@ -262,3 +262,39 @@ pub unsafe extern "C" fn wasmtime_elem_drop(vmctx: *mut VMContext, elem_index: u
     let instance = (&mut *vmctx).instance();
     instance.elem_drop(elem_index);
 }
+
+/// Implementation of `memory.copy` for locally defined memories.
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_memory_copy(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let memory_index = DefinedMemoryIndex::from_u32(memory_index);
+    let source_loc = ir::SourceLoc::new(source_loc);
+    let instance = (&mut *vmctx).instance();
+    if let Err(trap) = instance.defined_memory_copy(memory_index, dst, src, len, source_loc) {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `memory.copy` for imported memories.
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_imported_memory_copy(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let memory_index = MemoryIndex::from_u32(memory_index);
+    let source_loc = ir::SourceLoc::new(source_loc);
+    let instance = (&mut *vmctx).instance();
+    if let Err(trap) = instance.imported_memory_copy(memory_index, dst, src, len, source_loc) {
+        raise_lib_trap(trap);
+    }
+}

crates/runtime/src/vmcontext.rs

@@ -563,6 +563,11 @@ impl VMBuiltinFunctionsArray {
         ptrs[BuiltinFunctionIndex::get_elem_drop_index().index() as usize] =
             wasmtime_elem_drop as usize;
 
+        ptrs[BuiltinFunctionIndex::get_memory_copy_index().index() as usize] =
+            wasmtime_memory_copy as usize;
+        ptrs[BuiltinFunctionIndex::get_imported_memory_copy_index().index() as usize] =
+            wasmtime_imported_memory_copy as usize;
+
         debug_assert!(ptrs.iter().cloned().all(|p| p != 0));
 
         Self { ptrs }

tests/misc_testsuite/imported-memory-copy.wast

@@ -0,0 +1,129 @@
+(module $foreign
+  (memory (export "mem") 1 1)
+  (data 0 (i32.const 1000) "hello")
+  (data 0 (i32.const 2000) "olleh"))
+
+(register "foreign" $foreign)
+
+(module
+  (memory (import "foreign" "mem") 1 1)
+
+  (func $is_char (param i32 i32) (result i32)
+    local.get 0
+    i32.load8_u
+    local.get 1
+    i32.eq)
+
+  (func (export "is hello?") (param i32) (result i32)
+    local.get 0
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "is olleh?") (param i32) (result i32)
+    local.get 0
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "memory.copy") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    memory.copy))
+
+;; Our memory has our initial data in the right places.
+(assert_return
+  (invoke "is hello?" (i32.const 1000))
+  (i32.const 1))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 500))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 1500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 1500))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 1998) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 1998))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 1998) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst = src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))

tests/misc_testsuite/memory-copy.wast

@@ -0,0 +1,124 @@
+(module
+  (memory 1 1)
+  (data 0 (i32.const 1000) "hello")
+  (data 0 (i32.const 2000) "olleh")
+
+  (func $is_char (param i32 i32) (result i32)
+    local.get 0
+    i32.load8_u
+    local.get 1
+    i32.eq)
+
+  (func (export "is hello?") (param i32) (result i32)
+    local.get 0
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "is olleh?") (param i32) (result i32)
+    local.get 0
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "memory.copy") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    memory.copy))
+
+;; Our memory has our initial data in the right places.
+(assert_return
+  (invoke "is hello?" (i32.const 1000))
+  (i32.const 1))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 500))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 1500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 1500))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 1998) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 1998))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 1998) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst = src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))

tests/wast_testsuites.rs

@@ -22,6 +22,8 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> {
         || wast.iter().any(|s| s == "table_copy.wast")
         || wast.iter().any(|s| s == "elem_drop.wast")
         || wast.iter().any(|s| s == "elem-ref-null.wast")
+        || wast.iter().any(|s| s == "memory-copy.wast")
+        || wast.iter().any(|s| s == "imported-memory-copy.wast")
         || wast
             .iter()
             .any(|s| s == "table_copy_on_imported_tables.wast");