diff --git a/Cargo.lock b/Cargo.lock index 98fe9b8192..49f8ed068b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3323,7 +3323,6 @@ dependencies = [ "paste", "psm", "rayon", - "region", "serde", "target-lexicon", "tempfile", @@ -3568,7 +3567,6 @@ dependencies = [ "ittapi-rs", "log", "object", - "region", "rustc-demangle", "rustix", "serde", @@ -3605,7 +3603,6 @@ dependencies = [ "memoffset", "more-asserts", "rand 0.8.5", - "region", "rustix", "thiserror", "wasmtime-environ", diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index cf3be93339..128789dc23 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -14,7 +14,6 @@ edition = "2021" wasmtime-environ = { path = "../environ", version = "=0.40.0" } wasmtime-jit-debug = { path = "../jit-debug", version = "=0.40.0", features = ["perf_jitdump"], optional = true } wasmtime-runtime = { path = "../runtime", version = "=0.40.0" } -region = "2.2.0" thiserror = "1.0.4" target-lexicon = { version = "0.12.0", default-features = false } anyhow = "1.0" diff --git a/crates/jit/src/unwind/systemv.rs b/crates/jit/src/unwind/systemv.rs index 6dc519d0c6..331b0de1e8 100644 --- a/crates/jit/src/unwind/systemv.rs +++ b/crates/jit/src/unwind/systemv.rs @@ -26,7 +26,7 @@ impl UnwindRegistration { unwind_len: usize, ) -> Result { debug_assert_eq!( - unwind_info as usize % region::page::size(), + unwind_info as usize % wasmtime_runtime::page_size(), 0, "The unwind info must always be aligned to a page" ); diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index fc5886eef6..764ef619b7 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -14,7 +14,6 @@ edition = "2021" wasmtime-environ = { path = "../environ", version = "=0.40.0" } wasmtime-fiber = { path = "../fiber", version = "=0.40.0", optional = true } wasmtime-jit-debug = { path = "../jit-debug", version = "=0.40.0", features = ["gdb_jit_int"] } -region = "2.1.0" libc = { version = "0.2.112", default-features = false } log = "0.4.8" memoffset = "0.6.0" @@ -39,6 +38,7 @@ features = [ "Win32_System_Kernel", "Win32_System_Memory", "Win32_System_Diagnostics_Debug", + "Win32_System_SystemInformation", "Win32_Storage_FileSystem", "Win32_Security", ] diff --git a/crates/runtime/src/cow.rs b/crates/runtime/src/cow.rs index 6c37678614..a4364e6f36 100644 --- a/crates/runtime/src/cow.rs +++ b/crates/runtime/src/cow.rs @@ -212,7 +212,7 @@ impl ModuleMemoryImages { _ => return Ok(None), }; let mut memories = PrimaryMap::with_capacity(map.len()); - let page_size = region::page::size() as u32; + let page_size = crate::page_size() as u32; for (memory_index, init) in map { // mmap-based-initialization only works for defined memories with a // known starting point of all zeros, so bail out if the mmeory is @@ -596,7 +596,7 @@ mod test { fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result { // Offset must be page-aligned. - let page_size = region::page::size(); + let page_size = crate::page_size(); assert_eq!(offset & (page_size - 1), 0); let memfd = create_memfd()?; memfd.as_file().write_all(data)?; diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 136a17da99..9879835611 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -30,9 +30,6 @@ cfg_if::cfg_if! { if #[cfg(windows)] { mod windows; use windows as imp; - } else if #[cfg(target_os = "linux")] { - mod linux; - use linux as imp; } else { mod unix; use unix as imp; @@ -217,7 +214,7 @@ impl InstancePool { instance_limits: &InstanceLimits, tunables: &Tunables, ) -> Result { - let page_size = region::page::size(); + let page_size = crate::page_size(); let instance_size = round_up_to_pow2(instance_limits.size, mem::align_of::()); @@ -692,7 +689,7 @@ impl MemoryPool { }; assert!( - memory_size % region::page::size() == 0, + memory_size % crate::page_size() == 0, "memory size {} is not a multiple of system page size", memory_size ); @@ -828,7 +825,7 @@ struct TablePool { impl TablePool { fn new(instance_limits: &InstanceLimits) -> Result { - let page_size = region::page::size(); + let page_size = crate::page_size(); let table_size = round_up_to_pow2( mem::size_of::<*mut u8>() @@ -895,7 +892,9 @@ struct StackPool { #[cfg(all(feature = "async", unix))] impl StackPool { fn new(instance_limits: &InstanceLimits, stack_size: usize) -> Result { - let page_size = region::page::size(); + use rustix::mm::{mprotect, MprotectFlags}; + + let page_size = crate::page_size(); // Add a page to the stack size for the guard page when using fiber stacks let stack_size = if stack_size == 0 { @@ -921,7 +920,7 @@ impl StackPool { for i in 0..max_instances { // Make the stack guard page inaccessible let bottom_of_stack = mapping.as_mut_ptr().add(i * stack_size); - region::protect(bottom_of_stack, page_size, region::Protection::NONE) + mprotect(bottom_of_stack.cast(), page_size, MprotectFlags::empty()) .context("failed to protect stack guard page")?; } } @@ -1298,7 +1297,7 @@ mod test { ..Default::default() })?; - let host_page_size = region::page::size(); + let host_page_size = crate::page_size(); assert_eq!(pool.table_size, host_page_size); assert_eq!(pool.max_tables, 4); @@ -1335,7 +1334,7 @@ mod test { 1, )?; - let native_page_size = region::page::size(); + let native_page_size = crate::page_size(); assert_eq!(pool.stack_size, 2 * native_page_size); assert_eq!(pool.max_instances, 10); assert_eq!(pool.page_size, native_page_size); diff --git a/crates/runtime/src/instance/allocator/pooling/linux.rs b/crates/runtime/src/instance/allocator/pooling/linux.rs deleted file mode 100644 index 26c351855c..0000000000 --- a/crates/runtime/src/instance/allocator/pooling/linux.rs +++ /dev/null @@ -1,56 +0,0 @@ -use anyhow::{Context, Result}; - -fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { - if len == 0 { - return Ok(()); - } - - unsafe { - if protect { - region::protect(addr, len, region::Protection::NONE) - .context("failed to protect memory pages")?; - } - - // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access - rustix::mm::madvise(addr as _, len, rustix::mm::Advice::LinuxDontNeed) - .context("madvise failed to decommit: {}")?; - } - - Ok(()) -} - -pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - // Just change the protection level to READ|WRITE - unsafe { - region::protect(addr, len, region::Protection::READ_WRITE) - .context("failed to make linear memory pages read/write") - } -} - -pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len, true) -} - -pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { - // A no-op as table pages remain READ|WRITE - Ok(()) -} - -pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len, false) -} - -#[cfg(feature = "async")] -pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { - // A no-op as stack pages remain READ|WRITE - Ok(()) -} - -#[cfg(feature = "async")] -pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len, false) -} diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs index aa49e8898d..8a9842e84e 100644 --- a/crates/runtime/src/instance/allocator/pooling/unix.rs +++ b/crates/runtime/src/instance/allocator/pooling/unix.rs @@ -1,26 +1,45 @@ use anyhow::{Context, Result}; +use rustix::mm::{mprotect, MprotectFlags}; fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { if len == 0 { return Ok(()); } - // By creating a new mapping at the same location, this will discard the - // mapping for the pages in the given range. - // The new mapping will be to the CoW zero page, so this effectively - // zeroes the pages. unsafe { - rustix::mm::mmap_anonymous( - addr as _, - len, - if protect { - rustix::mm::ProtFlags::empty() + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + use rustix::mm::{madvise, Advice}; + + if protect { + mprotect(addr.cast(), len, MprotectFlags::empty()) + .context("failed to protect memory pages")?; + } + + // On Linux, this is enough to cause the kernel to initialize + // the pages to 0 on next access + madvise(addr as _, len, Advice::LinuxDontNeed) + .context("madvise failed to decommit: {}")?; } else { - rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE - }, - rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED, - ) - .context("mmap failed to remap pages: {}")?; + use rustix::mm::{mmap_anonymous, ProtFlags, MapFlags}; + + // By creating a new mapping at the same location, this will + // discard the mapping for the pages in the given range. + // The new mapping will be to the CoW zero page, so this + // effectively zeroes the pages. + mmap_anonymous( + addr as _, + len, + if protect { + ProtFlags::empty() + } else { + ProtFlags::READ | ProtFlags::WRITE + }, + MapFlags::PRIVATE | MapFlags::FIXED, + ) + .context("mmap failed to remap pages: {}")?; + } + } } Ok(()) @@ -33,7 +52,7 @@ pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { // Just change the protection level to READ|WRITE unsafe { - region::protect(addr, len, region::Protection::READ_WRITE) + mprotect(addr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE) .context("failed to make linear memory pages read/write") } } diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 4f7567dd35..18ab710cc9 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -21,10 +21,9 @@ )] #![cfg_attr(not(memory_init_cow), allow(unused_variables, unreachable_code))] -use std::sync::atomic::AtomicU64; -use std::sync::Arc; - use anyhow::Error; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; use wasmtime_environ::DefinedFuncIndex; use wasmtime_environ::DefinedMemoryIndex; use wasmtime_environ::FunctionInfo; @@ -200,3 +199,35 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static { /// `VMSharedSignatureIndex` entries corresponding to the `SignatureIndex`. fn signature_ids(&self) -> &[VMSharedSignatureIndex]; } + +/// Returns the host OS page size, in bytes. +pub fn page_size() -> usize { + static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); + + return match PAGE_SIZE.load(Ordering::Relaxed) { + 0 => { + let size = get_page_size(); + assert!(size != 0); + PAGE_SIZE.store(size, Ordering::Relaxed); + size + } + n => n, + }; + + #[cfg(windows)] + fn get_page_size() -> usize { + use std::mem::MaybeUninit; + use windows_sys::Win32::System::SystemInformation::*; + + unsafe { + let mut info = MaybeUninit::uninit(); + GetSystemInfo(info.as_mut_ptr()); + info.assume_init_ref().dwPageSize as usize + } + } + + #[cfg(unix)] + fn get_page_size() -> usize { + unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize } + } +} diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index 45d0a12d5d..19db9be431 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -41,7 +41,8 @@ impl Mmap { /// Create a new `Mmap` pointing to at least `size` bytes of page-aligned accessible memory. pub fn with_at_least(size: usize) -> Result { - let rounded_size = region::page::ceil(size); + let page_size = crate::page_size(); + let rounded_size = (size + (page_size - 1)) & !(page_size - 1); Self::accessible_reserved(rounded_size, rounded_size) } @@ -158,7 +159,7 @@ impl Mmap { /// must be native page-size multiples. #[cfg(not(target_os = "windows"))] pub fn accessible_reserved(accessible_size: usize, mapping_size: usize) -> Result { - let page_size = region::page::size(); + let page_size = crate::page_size(); assert_le!(accessible_size, mapping_size); assert_eq!(mapping_size & (page_size - 1), 0); assert_eq!(accessible_size & (page_size - 1), 0); @@ -226,7 +227,7 @@ impl Mmap { return Ok(Self::new()); } - let page_size = region::page::size(); + let page_size = crate::page_size(); assert_le!(accessible_size, mapping_size); assert_eq!(mapping_size & (page_size - 1), 0); assert_eq!(accessible_size & (page_size - 1), 0); @@ -278,16 +279,22 @@ impl Mmap { /// `self`'s reserved memory. #[cfg(not(target_os = "windows"))] pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<()> { - let page_size = region::page::size(); + use rustix::mm::{mprotect, MprotectFlags}; + + let page_size = crate::page_size(); assert_eq!(start & (page_size - 1), 0); assert_eq!(len & (page_size - 1), 0); assert_le!(len, self.len); assert_le!(start, self.len - len); // Commit the accessible size. - let ptr = self.ptr as *const u8; + let ptr = self.ptr as *mut u8; unsafe { - region::protect(ptr.add(start), len, region::Protection::READ_WRITE)?; + mprotect( + ptr.add(start).cast(), + len, + MprotectFlags::READ | MprotectFlags::WRITE, + )?; } Ok(()) @@ -303,7 +310,7 @@ impl Mmap { use std::io; use windows_sys::Win32::System::Memory::*; - let page_size = region::page::size(); + let page_size = crate::page_size(); assert_eq!(start & (page_size - 1), 0); assert_eq!(len & (page_size - 1), 0); assert_le!(len, self.len); @@ -370,11 +377,11 @@ impl Mmap { assert!(range.end <= self.len()); assert!(range.start <= range.end); assert!( - range.start % region::page::size() == 0, + range.start % crate::page_size() == 0, "changing of protections isn't page-aligned", ); - let base = self.as_ptr().add(range.start); + let base = self.as_ptr().add(range.start) as *mut _; let len = range.end - range.start; // On Windows when we have a file mapping we need to specifically use @@ -385,19 +392,23 @@ impl Mmap { use std::io; use windows_sys::Win32::System::Memory::*; - if self.file.is_some() { - let mut old = 0; - if VirtualProtect(base as *mut _, len, PAGE_WRITECOPY, &mut old) == 0 { - return Err(io::Error::last_os_error()) - .context("failed to change pages to `PAGE_WRITECOPY`"); - } - return Ok(()); + let mut old = 0; + let result = if self.file.is_some() { + VirtualProtect(base, len, PAGE_WRITECOPY, &mut old) + } else { + VirtualProtect(base, len, PAGE_READWRITE, &mut old) + }; + if result == 0 { + return Err(io::Error::last_os_error().into()); } } - // If we're not on Windows or if we're on Windows with an anonymous - // mapping then we can use the `region` crate. - region::protect(base, len, region::Protection::READ_WRITE)?; + #[cfg(not(windows))] + { + use rustix::mm::{mprotect, MprotectFlags}; + mprotect(base, len, MprotectFlags::READ | MprotectFlags::WRITE)?; + } + Ok(()) } @@ -407,15 +418,29 @@ impl Mmap { assert!(range.end <= self.len()); assert!(range.start <= range.end); assert!( - range.start % region::page::size() == 0, + range.start % crate::page_size() == 0, "changing of protections isn't page-aligned", ); + let base = self.as_ptr().add(range.start) as *mut _; + let len = range.end - range.start; + + #[cfg(windows)] + { + use std::io; + use windows_sys::Win32::System::Memory::*; - region::protect( - self.as_ptr().add(range.start), - range.end - range.start, - region::Protection::READ_EXECUTE, - )?; + let mut old = 0; + let result = VirtualProtect(base, len, PAGE_EXECUTE_READ, &mut old); + if result == 0 { + return Err(io::Error::last_os_error().into()); + } + } + + #[cfg(not(windows))] + { + use rustix::mm::{mprotect, MprotectFlags}; + mprotect(base, len, MprotectFlags::READ | MprotectFlags::EXEC)?; + } Ok(()) } diff --git a/crates/runtime/src/traphandlers/unix.rs b/crates/runtime/src/traphandlers/unix.rs index dc0131957f..62262f173f 100644 --- a/crates/runtime/src/traphandlers/unix.rs +++ b/crates/runtime/src/traphandlers/unix.rs @@ -290,7 +290,7 @@ pub fn lazy_per_thread_init() { // ... but failing that we need to allocate our own, so do all that // here. - let page_size: usize = region::page::size(); + let page_size = crate::page_size(); let guard_size = page_size; let alloc_size = guard_size + MIN_STACK_SIZE; diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 9defc87ee6..3c250b36d1 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -23,7 +23,6 @@ wasmtime-component-macro = { path = "../component-macro", version = "=0.40.0", o target-lexicon = { version = "0.12.0", default-features = false } wasmparser = "0.87.0" anyhow = "1.0.19" -region = "2.2.0" libc = "0.2" cfg-if = "1.0" backtrace = { version = "0.3.61" } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 6cd656402a..b12d9cef6f 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1458,7 +1458,7 @@ impl Config { } fn round_up_to_pages(val: u64) -> u64 { - let page_size = region::page::size() as u64; + let page_size = wasmtime_runtime::page_size() as u64; debug_assert!(page_size.is_power_of_two()); val.checked_add(page_size - 1) .map(|val| val & !(page_size - 1)) diff --git a/tests/rlimited-memory.rs b/tests/rlimited-memory.rs index 5acc2f5d8f..0dd9d57a7a 100644 --- a/tests/rlimited-memory.rs +++ b/tests/rlimited-memory.rs @@ -84,7 +84,7 @@ fn custom_limiter_detect_os_oom_failure() -> Result<()> { // The memory_grow_failed hook should show Linux gave OOM: let err_msg = store.data().error.as_ref().unwrap(); assert!( - err_msg.starts_with("System call failed: Cannot allocate memory"), + err_msg.starts_with("Cannot allocate memory"), "unexpected error: {}", err_msg );