diff --git a/crates/environ/src/func_environ.rs b/crates/environ/src/func_environ.rs index f3a11ff735..39e8d5f0b4 100644 --- a/crates/environ/src/func_environ.rs +++ b/crates/environ/src/func_environ.rs @@ -70,9 +70,17 @@ impl BuiltinFunctionIndex { pub const fn get_elem_drop_index() -> Self { Self(9) } + /// Returns an index for wasm's `memory.copy` for locally defined memories. + pub const fn get_memory_copy_index() -> Self { + Self(10) + } + /// Returns an index for wasm's `memory.copy` for imported memories. + pub const fn get_imported_memory_copy_index() -> Self { + Self(11) + } /// Returns the total number of builtin functions. pub const fn builtin_functions_total_number() -> u32 { - 10 + 12 } /// Return the index as an u32 number. @@ -110,6 +118,10 @@ pub struct FuncEnvironment<'module_environment> { /// The external function signature for implementing wasm's `elem.drop`. elem_drop_sig: Option, + /// The external function signature for implementing wasm's `memory.copy` + /// (it's the same for both local and imported memories). + memory_copy_sig: Option, + /// Offsets to struct fields accessed by JIT code. offsets: VMOffsets, } @@ -128,6 +140,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { table_copy_sig: None, table_init_sig: None, elem_drop_sig: None, + memory_copy_sig: None, offsets: VMOffsets::new(target_config.pointer_bytes(), module), } } @@ -371,6 +384,51 @@ impl<'module_environment> FuncEnvironment<'module_environment> { (sig, BuiltinFunctionIndex::get_elem_drop_index()) } + fn get_memory_copy_sig(&mut self, func: &mut Function) -> ir::SigRef { + let sig = self.memory_copy_sig.unwrap_or_else(|| { + func.import_signature(Signature { + params: vec![ + AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), + // Memory index. + AbiParam::new(I32), + // Destination address. + AbiParam::new(I32), + // Source address. + AbiParam::new(I32), + // Length. + AbiParam::new(I32), + // Source location. + AbiParam::new(I32), + ], + returns: vec![], + call_conv: self.target_config.default_call_conv, + }) + }); + self.memory_copy_sig = Some(sig); + sig + } + + fn get_memory_copy_func( + &mut self, + func: &mut Function, + memory_index: MemoryIndex, + ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { + let sig = self.get_memory_copy_sig(func); + if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) { + ( + sig, + defined_memory_index.index(), + BuiltinFunctionIndex::get_memory_copy_index(), + ) + } else { + ( + sig, + memory_index.index(), + BuiltinFunctionIndex::get_imported_memory_copy_index(), + ) + } + } + /// Translates load of builtin function and returns a pair of values `vmctx` /// and address of the loaded function. fn translate_load_builtin_function_address( @@ -968,16 +1026,30 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m fn translate_memory_copy( &mut self, - _pos: FuncCursor, - _index: MemoryIndex, + mut pos: FuncCursor, + memory_index: MemoryIndex, _heap: ir::Heap, - _dst: ir::Value, - _src: ir::Value, - _len: ir::Value, + dst: ir::Value, + src: ir::Value, + len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported( - "bulk memory: `memory.copy`".to_string(), - )) + let (func_sig, memory_index, func_idx) = + self.get_memory_copy_func(&mut pos.func, memory_index); + + let memory_index_arg = pos.ins().iconst(I32, memory_index as i64); + + let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx); + + let src_loc = pos.srcloc(); + let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64); + + pos.ins().call_indirect( + func_sig, + func_addr, + &[vmctx, memory_index_arg, dst, src, len, src_loc_arg], + ); + + Ok(()) } fn translate_memory_fill( @@ -1005,7 +1077,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m _len: ir::Value, ) -> WasmResult<()> { Err(WasmError::Unsupported( - "bulk memory: `memory.copy`".to_string(), + "bulk memory: `memory.init`".to_string(), )) } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 4ae120d8f1..395969388a 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -15,6 +15,7 @@ use crate::vmcontext::{ VMTableDefinition, VMTableImport, }; use crate::{TrapDescription, TrapRegistration}; +use backtrace::Backtrace; use memoffset::offset_of; use more_asserts::assert_lt; use std::alloc::{self, Layout}; @@ -601,7 +602,7 @@ impl Instance { source_loc, trap_code: ir::TrapCode::TableOutOfBounds, }, - backtrace: backtrace::Backtrace::new(), + backtrace: Backtrace::new(), }); } @@ -625,6 +626,71 @@ impl Instance { } } + /// Do a `memory.copy` for a locally defined memory. + /// + /// # Errors + /// + /// Returns a `Trap` error when the source or destination ranges are out of + /// bounds. + pub(crate) fn defined_memory_copy( + &self, + memory_index: DefinedMemoryIndex, + dst: u32, + src: u32, + len: u32, + source_loc: ir::SourceLoc, + ) -> Result<(), Trap> { + // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy + + let memory = self.memory(memory_index); + if src + .checked_add(len) + .map_or(true, |n| n as usize > memory.current_length) + || dst + .checked_add(len) + .map_or(true, |m| m as usize > memory.current_length) + { + return Err(Trap::Wasm { + desc: TrapDescription { + source_loc, + trap_code: ir::TrapCode::HeapOutOfBounds, + }, + backtrace: Backtrace::new(), + }); + } + + let dst = isize::try_from(dst).unwrap(); + let src = isize::try_from(src).unwrap(); + + // Bounds and casts are checked above, by this point we know that + // everything is safe. + unsafe { + let dst = memory.base.offset(dst); + let src = memory.base.offset(src); + ptr::copy(src, dst, len as usize); + } + + Ok(()) + } + + /// Perform a `memory.copy` on an imported memory. + pub(crate) fn imported_memory_copy( + &self, + memory_index: MemoryIndex, + dst: u32, + src: u32, + len: u32, + source_loc: ir::SourceLoc, + ) -> Result<(), Trap> { + let import = self.imported_memory(memory_index); + unsafe { + let foreign_instance = (&*import.vmctx).instance(); + let foreign_memory = &*import.from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.defined_memory_copy(foreign_index, dst, src, len, source_loc) + } + } + /// Get a table by index regardless of whether it is locally-defined or an /// imported, foreign table. pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table { diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 3e4c0f93e9..b83627f72b 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -262,3 +262,39 @@ pub unsafe extern "C" fn wasmtime_elem_drop(vmctx: *mut VMContext, elem_index: u let instance = (&mut *vmctx).instance(); instance.elem_drop(elem_index); } + +/// Implementation of `memory.copy` for locally defined memories. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_memory_copy( + vmctx: *mut VMContext, + memory_index: u32, + dst: u32, + src: u32, + len: u32, + source_loc: u32, +) { + let memory_index = DefinedMemoryIndex::from_u32(memory_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + if let Err(trap) = instance.defined_memory_copy(memory_index, dst, src, len, source_loc) { + raise_lib_trap(trap); + } +} + +/// Implementation of `memory.copy` for imported memories. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_imported_memory_copy( + vmctx: *mut VMContext, + memory_index: u32, + dst: u32, + src: u32, + len: u32, + source_loc: u32, +) { + let memory_index = MemoryIndex::from_u32(memory_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + if let Err(trap) = instance.imported_memory_copy(memory_index, dst, src, len, source_loc) { + raise_lib_trap(trap); + } +} diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 613cf9af99..50b02f7b48 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -563,6 +563,11 @@ impl VMBuiltinFunctionsArray { ptrs[BuiltinFunctionIndex::get_elem_drop_index().index() as usize] = wasmtime_elem_drop as usize; + ptrs[BuiltinFunctionIndex::get_memory_copy_index().index() as usize] = + wasmtime_memory_copy as usize; + ptrs[BuiltinFunctionIndex::get_imported_memory_copy_index().index() as usize] = + wasmtime_imported_memory_copy as usize; + debug_assert!(ptrs.iter().cloned().all(|p| p != 0)); Self { ptrs } diff --git a/tests/misc_testsuite/imported-memory-copy.wast b/tests/misc_testsuite/imported-memory-copy.wast new file mode 100644 index 0000000000..4787173cc7 --- /dev/null +++ b/tests/misc_testsuite/imported-memory-copy.wast @@ -0,0 +1,129 @@ +(module $foreign + (memory (export "mem") 1 1) + (data 0 (i32.const 1000) "hello") + (data 0 (i32.const 2000) "olleh")) + +(register "foreign" $foreign) + +(module + (memory (import "foreign" "mem") 1 1) + + (func $is_char (param i32 i32) (result i32) + local.get 0 + i32.load8_u + local.get 1 + i32.eq) + + (func (export "is hello?") (param i32) (result i32) + local.get 0 + i32.const 104 ;; 'h' + call $is_char + + local.get 0 + i32.const 1 + i32.add + i32.const 101 ;; 'e' + call $is_char + + local.get 0 + i32.const 2 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 3 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 4 + i32.add + i32.const 111 ;; 'o' + call $is_char + + i32.and + i32.and + i32.and + i32.and + ) + + (func (export "is olleh?") (param i32) (result i32) + local.get 0 + i32.const 111 ;; 'o' + call $is_char + + local.get 0 + i32.const 1 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 2 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 3 + i32.add + i32.const 101 ;; 'e' + call $is_char + + local.get 0 + i32.const 4 + i32.add + i32.const 104 ;; 'h' + call $is_char + + i32.and + i32.and + i32.and + i32.and + ) + + (func (export "memory.copy") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + memory.copy)) + +;; Our memory has our initial data in the right places. +(assert_return + (invoke "is hello?" (i32.const 1000)) + (i32.const 1)) +(assert_return + (invoke "is olleh?" (i32.const 2000)) + (i32.const 1)) + +;; Non-overlapping memory copy with dst < src. +(invoke "memory.copy" (i32.const 500) (i32.const 1000) (i32.const 5)) +(assert_return + (invoke "is hello?" (i32.const 500)) + (i32.const 1)) + +;; Non-overlapping memory copy with dst > src. +(invoke "memory.copy" (i32.const 1500) (i32.const 1000) (i32.const 5)) +(assert_return + (invoke "is hello?" (i32.const 1500)) + (i32.const 1)) + +;; Overlapping memory copy with dst < src. +(invoke "memory.copy" (i32.const 1998) (i32.const 2000) (i32.const 5)) +(assert_return + (invoke "is olleh?" (i32.const 1998)) + (i32.const 1)) + +;; Overlapping memory copy with dst > src. +(invoke "memory.copy" (i32.const 2000) (i32.const 1998) (i32.const 5)) +(assert_return + (invoke "is olleh?" (i32.const 2000)) + (i32.const 1)) + +;; Overlapping memory copy with dst = src. +(invoke "memory.copy" (i32.const 2000) (i32.const 2000) (i32.const 5)) +(assert_return + (invoke "is olleh?" (i32.const 2000)) + (i32.const 1)) diff --git a/tests/misc_testsuite/memory-copy.wast b/tests/misc_testsuite/memory-copy.wast new file mode 100644 index 0000000000..d339264423 --- /dev/null +++ b/tests/misc_testsuite/memory-copy.wast @@ -0,0 +1,124 @@ +(module + (memory 1 1) + (data 0 (i32.const 1000) "hello") + (data 0 (i32.const 2000) "olleh") + + (func $is_char (param i32 i32) (result i32) + local.get 0 + i32.load8_u + local.get 1 + i32.eq) + + (func (export "is hello?") (param i32) (result i32) + local.get 0 + i32.const 104 ;; 'h' + call $is_char + + local.get 0 + i32.const 1 + i32.add + i32.const 101 ;; 'e' + call $is_char + + local.get 0 + i32.const 2 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 3 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 4 + i32.add + i32.const 111 ;; 'o' + call $is_char + + i32.and + i32.and + i32.and + i32.and + ) + + (func (export "is olleh?") (param i32) (result i32) + local.get 0 + i32.const 111 ;; 'o' + call $is_char + + local.get 0 + i32.const 1 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 2 + i32.add + i32.const 108 ;; 'l' + call $is_char + + local.get 0 + i32.const 3 + i32.add + i32.const 101 ;; 'e' + call $is_char + + local.get 0 + i32.const 4 + i32.add + i32.const 104 ;; 'h' + call $is_char + + i32.and + i32.and + i32.and + i32.and + ) + + (func (export "memory.copy") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + memory.copy)) + +;; Our memory has our initial data in the right places. +(assert_return + (invoke "is hello?" (i32.const 1000)) + (i32.const 1)) +(assert_return + (invoke "is olleh?" (i32.const 2000)) + (i32.const 1)) + +;; Non-overlapping memory copy with dst < src. +(invoke "memory.copy" (i32.const 500) (i32.const 1000) (i32.const 5)) +(assert_return + (invoke "is hello?" (i32.const 500)) + (i32.const 1)) + +;; Non-overlapping memory copy with dst > src. +(invoke "memory.copy" (i32.const 1500) (i32.const 1000) (i32.const 5)) +(assert_return + (invoke "is hello?" (i32.const 1500)) + (i32.const 1)) + +;; Overlapping memory copy with dst < src. +(invoke "memory.copy" (i32.const 1998) (i32.const 2000) (i32.const 5)) +(assert_return + (invoke "is olleh?" (i32.const 1998)) + (i32.const 1)) + +;; Overlapping memory copy with dst > src. +(invoke "memory.copy" (i32.const 2000) (i32.const 1998) (i32.const 5)) +(assert_return + (invoke "is olleh?" (i32.const 2000)) + (i32.const 1)) + +;; Overlapping memory copy with dst = src. +(invoke "memory.copy" (i32.const 2000) (i32.const 2000) (i32.const 5)) +(assert_return + (invoke "is olleh?" (i32.const 2000)) + (i32.const 1)) diff --git a/tests/wast_testsuites.rs b/tests/wast_testsuites.rs index cb7599bc94..5a4ebedfdf 100644 --- a/tests/wast_testsuites.rs +++ b/tests/wast_testsuites.rs @@ -22,6 +22,8 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { || wast.iter().any(|s| s == "table_copy.wast") || wast.iter().any(|s| s == "elem_drop.wast") || wast.iter().any(|s| s == "elem-ref-null.wast") + || wast.iter().any(|s| s == "memory-copy.wast") + || wast.iter().any(|s| s == "imported-memory-copy.wast") || wast .iter() .any(|s| s == "table_copy_on_imported_tables.wast");