diff --git a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs index 79e63f3742..cea735678a 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs @@ -5,7 +5,7 @@ use crate::runtime::vm::VMGcRef; use crate::{ store::{AutoAssertNoGc, StoreOpaque}, ArrayType, AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, RefType, - Result, RootSet, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy, I31, + Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy, I31, }; use core::mem; use core::mem::MaybeUninit; @@ -458,23 +458,11 @@ unsafe impl WasmTy for Rooted { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - let gc_ref = self.inner.try_clone_gc_ref(store)?; - let r64 = gc_ref.as_r64(); - store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); - debug_assert_ne!(r64, 0); - let anyref = u32::try_from(r64).unwrap(); - ptr.write(ValRaw::anyref(anyref)); - Ok(()) + self.wasm_ty_store(store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_anyref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()) - .expect("valid r64") - .expect("non-null"); - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - AnyRef::from_cloned_gc_ref(store, gc_ref) + Self::wasm_ty_load(store, ptr.get_anyref(), AnyRef::from_cloned_gc_ref) } } @@ -514,19 +502,11 @@ unsafe impl WasmTy for Option> { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - match self { - Some(r) => r.store(store, ptr), - None => { - ptr.write(ValRaw::anyref(0)); - Ok(()) - } - } + >::wasm_ty_option_store(self, store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let gc_ref = VMGcRef::from_r64(ptr.get_anyref().into()).expect("valid r64")?; - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - Some(AnyRef::from_cloned_gc_ref(store, gc_ref)) + >::wasm_ty_option_load(store, ptr.get_anyref(), AnyRef::from_cloned_gc_ref) } } @@ -552,28 +532,11 @@ unsafe impl WasmTy for ManuallyRooted { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - let gc_ref = self.inner.try_clone_gc_ref(store)?; - let r64 = gc_ref.as_r64(); - store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); - debug_assert_ne!(r64, 0); - let anyref = u32::try_from(r64).unwrap(); - ptr.write(ValRaw::anyref(anyref)); - Ok(()) + self.wasm_ty_store(store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_anyref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()) - .expect("valid r64") - .expect("non-null"); - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - RootSet::with_lifo_scope(store, |store| { - let rooted = AnyRef::from_cloned_gc_ref(store, gc_ref); - rooted - ._to_manually_rooted(store) - .expect("rooted is in scope") - }) + Self::wasm_ty_load(store, ptr.get_anyref(), AnyRef::from_cloned_gc_ref) } } @@ -614,27 +577,14 @@ unsafe impl WasmTy for Option> { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - match self { - Some(r) => r.store(store, ptr), - None => { - ptr.write(ValRaw::anyref(0)); - Ok(()) - } - } + >::wasm_ty_option_store(self, store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_anyref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()).expect("valid r64")?; - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - RootSet::with_lifo_scope(store, |store| { - let rooted = AnyRef::from_cloned_gc_ref(store, gc_ref); - Some( - rooted - ._to_manually_rooted(store) - .expect("rooted is in scope"), - ) - }) + >::wasm_ty_option_load( + store, + ptr.get_anyref(), + AnyRef::from_cloned_gc_ref, + ) } } diff --git a/crates/wasmtime/src/runtime/gc/enabled/externref.rs b/crates/wasmtime/src/runtime/gc/enabled/externref.rs index f15cc909a8..5a8033f284 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/externref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/externref.rs @@ -5,7 +5,7 @@ use crate::runtime::vm::VMGcRef; use crate::{ store::{AutoAssertNoGc, StoreOpaque}, AsContextMut, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, RefType, - Result, RootSet, Rooted, StoreContext, StoreContextMut, ValRaw, ValType, WasmTy, + Result, Rooted, StoreContext, StoreContextMut, ValRaw, ValType, WasmTy, }; use core::any::Any; use core::mem; @@ -446,23 +446,11 @@ unsafe impl WasmTy for Rooted { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - let gc_ref = self.inner.try_clone_gc_ref(store)?; - let r64 = gc_ref.as_r64(); - store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); - debug_assert_ne!(r64, 0); - let externref = u32::try_from(r64).unwrap(); - ptr.write(ValRaw::externref(externref)); - Ok(()) + self.wasm_ty_store(store, ptr, ValRaw::externref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_externref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()) - .expect("valid r64") - .expect("non-null"); - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - ExternRef::from_cloned_gc_ref(store, gc_ref) + Self::wasm_ty_load(store, ptr.get_externref(), ExternRef::from_cloned_gc_ref) } } @@ -488,19 +476,15 @@ unsafe impl WasmTy for Option> { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - match self { - Some(r) => r.store(store, ptr), - None => { - ptr.write(ValRaw::externref(0)); - Ok(()) - } - } + >::wasm_ty_option_store(self, store, ptr, ValRaw::externref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let gc_ref = VMGcRef::from_r64(ptr.get_externref().into()).expect("valid r64")?; - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - Some(ExternRef::from_cloned_gc_ref(store, gc_ref)) + >::wasm_ty_option_load( + store, + ptr.get_externref(), + ExternRef::from_cloned_gc_ref, + ) } } @@ -526,28 +510,11 @@ unsafe impl WasmTy for ManuallyRooted { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - let gc_ref = self.inner.try_clone_gc_ref(store)?; - let r64 = gc_ref.as_r64(); - store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); - debug_assert_ne!(r64, 0); - let externref = u32::try_from(r64).unwrap(); - ptr.write(ValRaw::externref(externref)); - Ok(()) + self.wasm_ty_store(store, ptr, ValRaw::externref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_externref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()) - .expect("valid r64") - .expect("non-null"); - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - RootSet::with_lifo_scope(store, |store| { - let rooted = ExternRef::from_cloned_gc_ref(store, gc_ref); - rooted - ._to_manually_rooted(store) - .expect("rooted is in scope") - }) + Self::wasm_ty_load(store, ptr.get_externref(), ExternRef::from_cloned_gc_ref) } } @@ -574,27 +541,14 @@ unsafe impl WasmTy for Option> { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - match self { - Some(r) => r.store(store, ptr), - None => { - ptr.write(ValRaw::externref(0)); - Ok(()) - } - } + >::wasm_ty_option_store(self, store, ptr, ValRaw::externref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_externref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()).expect("valid r64")?; - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - RootSet::with_lifo_scope(store, |store| { - let rooted = ExternRef::from_cloned_gc_ref(store, gc_ref); - Some( - rooted - ._to_manually_rooted(store) - .expect("rooted is in scope"), - ) - }) + >::wasm_ty_option_load( + store, + ptr.get_externref(), + ExternRef::from_cloned_gc_ref, + ) } } diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs index c216b90f6f..380539fe31 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs @@ -102,15 +102,15 @@ //! can. However, if you really must, consider also using an `AutoAssertNoGc` //! across the block of code that is manipulating raw GC references. -use crate::prelude::*; use crate::runtime::vm::{GcRootsList, GcStore, VMGcRef}; +use crate::{prelude::*, ValRaw}; use crate::{ store::{AutoAssertNoGc, StoreId, StoreOpaque}, AsContext, AsContextMut, GcRef, Result, RootedGcRef, }; use core::any; use core::marker; -use core::mem; +use core::mem::{self, MaybeUninit}; use core::num::NonZeroU64; use core::{ fmt::{self, Debug}, @@ -952,6 +952,64 @@ impl Rooted { pub(crate) fn unchecked_cast(self) -> Rooted { Rooted::from_gc_root_index(self.inner) } + + /// Common implementation of the `WasmTy::store` trait method for all + /// `Rooted`s. + pub(super) fn wasm_ty_store( + self, + store: &mut AutoAssertNoGc<'_>, + ptr: &mut MaybeUninit, + val_raw: impl Fn(u32) -> ValRaw, + ) -> Result<()> { + let gc_ref = self.inner.try_clone_gc_ref(store)?; + let raw = gc_ref.as_raw_u32(); + debug_assert_ne!(raw, 0); + store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); + ptr.write(val_raw(raw)); + Ok(()) + } + + /// Common implementation of the `WasmTy::load` trait method for all + /// `Rooted`s. + pub(super) fn wasm_ty_load( + store: &mut AutoAssertNoGc<'_>, + raw_gc_ref: u32, + from_cloned_gc_ref: impl Fn(&mut AutoAssertNoGc<'_>, VMGcRef) -> Self, + ) -> Self { + debug_assert_ne!(raw_gc_ref, 0); + let gc_ref = VMGcRef::from_raw_u32(raw_gc_ref).expect("non-null"); + let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); + from_cloned_gc_ref(store, gc_ref) + } + + /// Common implementation of the `WasmTy::store` trait method for all + /// `Option>`s. + pub(super) fn wasm_ty_option_store( + me: Option, + store: &mut AutoAssertNoGc<'_>, + ptr: &mut MaybeUninit, + val_raw: impl Fn(u32) -> ValRaw, + ) -> Result<()> { + match me { + Some(me) => me.wasm_ty_store(store, ptr, val_raw), + None => { + ptr.write(val_raw(0)); + Ok(()) + } + } + } + + /// Common implementation of the `WasmTy::load` trait method for all + /// `Option>`s. + pub(super) fn wasm_ty_option_load( + store: &mut AutoAssertNoGc<'_>, + raw_gc_ref: u32, + from_cloned_gc_ref: impl Fn(&mut AutoAssertNoGc<'_>, VMGcRef) -> Self, + ) -> Option { + let gc_ref = VMGcRef::from_raw_u32(raw_gc_ref)?; + let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); + Some(from_cloned_gc_ref(store, gc_ref)) + } } /// Nested rooting scopes. @@ -1601,6 +1659,76 @@ where core::mem::forget(self); u } + + /// Common implementation of the `WasmTy::store` trait method for all + /// `ManuallyRooted`s. + pub(super) fn wasm_ty_store( + self, + store: &mut AutoAssertNoGc<'_>, + ptr: &mut MaybeUninit, + val_raw: impl Fn(u32) -> ValRaw, + ) -> Result<()> { + let gc_ref = self.try_clone_gc_ref(store)?; + let raw = gc_ref.as_raw_u32(); + debug_assert_ne!(raw, 0); + store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); + ptr.write(val_raw(raw)); + Ok(()) + } + + /// Common implementation of the `WasmTy::load` trait method for all + /// `ManuallyRooted`s. + pub(super) fn wasm_ty_load( + store: &mut AutoAssertNoGc<'_>, + raw_gc_ref: u32, + from_cloned_gc_ref: impl Fn(&mut AutoAssertNoGc<'_>, VMGcRef) -> Rooted, + ) -> Self { + debug_assert_ne!(raw_gc_ref, 0); + let gc_ref = VMGcRef::from_raw_u32(raw_gc_ref).expect("non-null"); + let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); + RootSet::with_lifo_scope(store, |store| { + let rooted = from_cloned_gc_ref(store, gc_ref); + rooted + ._to_manually_rooted(store) + .expect("rooted is in scope") + }) + } + + /// Common implementation of the `WasmTy::store` trait method for all + /// `Option>`s. + pub(super) fn wasm_ty_option_store( + me: Option, + store: &mut AutoAssertNoGc<'_>, + ptr: &mut MaybeUninit, + val_raw: impl Fn(u32) -> ValRaw, + ) -> Result<()> { + match me { + Some(me) => me.wasm_ty_store(store, ptr, val_raw), + None => { + ptr.write(val_raw(0)); + Ok(()) + } + } + } + + /// Common implementation of the `WasmTy::load` trait method for all + /// `Option>`s. + pub(super) fn wasm_ty_option_load( + store: &mut AutoAssertNoGc<'_>, + raw_gc_ref: u32, + from_cloned_gc_ref: impl Fn(&mut AutoAssertNoGc<'_>, VMGcRef) -> Rooted, + ) -> Option { + let gc_ref = VMGcRef::from_raw_u32(raw_gc_ref)?; + let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); + RootSet::with_lifo_scope(store, |store| { + let rooted = from_cloned_gc_ref(store, gc_ref); + Some( + rooted + ._to_manually_rooted(store) + .expect("rooted is in scope"), + ) + }) + } } impl RootedGcRefImpl for ManuallyRooted { diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rs index 66c467be20..12cdc797a2 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs @@ -7,7 +7,7 @@ use crate::{ prelude::*, store::{AutoAssertNoGc, StoreContextMut, StoreOpaque}, AsContext, AsContextMut, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, - RefType, RootSet, Rooted, StructType, Val, ValRaw, ValType, WasmTy, + RefType, Rooted, StructType, Val, ValRaw, ValType, WasmTy, }; use crate::{AnyRef, FieldType}; use core::mem::{self, MaybeUninit}; @@ -592,23 +592,11 @@ unsafe impl WasmTy for Rooted { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - let gc_ref = self.inner.try_clone_gc_ref(store)?; - let r64 = gc_ref.as_r64(); - store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); - debug_assert_ne!(r64, 0); - let anyref = u32::try_from(r64).unwrap(); - ptr.write(ValRaw::anyref(anyref)); - Ok(()) + self.wasm_ty_store(store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_anyref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()) - .expect("valid r64") - .expect("non-null"); - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - StructRef::from_cloned_gc_ref(store, gc_ref) + Self::wasm_ty_load(store, ptr.get_anyref(), StructRef::from_cloned_gc_ref) } } @@ -648,19 +636,15 @@ unsafe impl WasmTy for Option> { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - match self { - Some(r) => r.store(store, ptr), - None => { - ptr.write(ValRaw::anyref(0)); - Ok(()) - } - } + >::wasm_ty_option_store(self, store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let gc_ref = VMGcRef::from_r64(ptr.get_anyref().into()).expect("valid r64")?; - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - Some(StructRef::from_cloned_gc_ref(store, gc_ref)) + >::wasm_ty_option_load( + store, + ptr.get_anyref(), + StructRef::from_cloned_gc_ref, + ) } } @@ -702,28 +686,11 @@ unsafe impl WasmTy for ManuallyRooted { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - let gc_ref = self.inner.try_clone_gc_ref(store)?; - let r64 = gc_ref.as_r64(); - store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref); - debug_assert_ne!(r64, 0); - let anyref = u32::try_from(r64).unwrap(); - ptr.write(ValRaw::anyref(anyref)); - Ok(()) + self.wasm_ty_store(store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_anyref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()) - .expect("valid r64") - .expect("non-null"); - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - RootSet::with_lifo_scope(store, |store| { - let rooted = StructRef::from_cloned_gc_ref(store, gc_ref); - rooted - ._to_manually_rooted(store) - .expect("rooted is in scope") - }) + Self::wasm_ty_load(store, ptr.get_anyref(), StructRef::from_cloned_gc_ref) } } @@ -766,27 +733,14 @@ unsafe impl WasmTy for Option> { } fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit) -> Result<()> { - match self { - Some(r) => r.store(store, ptr), - None => { - ptr.write(ValRaw::anyref(0)); - Ok(()) - } - } + >::wasm_ty_option_store(self, store, ptr, ValRaw::anyref) } unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self { - let raw = ptr.get_anyref(); - debug_assert_ne!(raw, 0); - let gc_ref = VMGcRef::from_r64(raw.into()).expect("valid r64")?; - let gc_ref = store.unwrap_gc_store_mut().clone_gc_ref(&gc_ref); - RootSet::with_lifo_scope(store, |store| { - let rooted = StructRef::from_cloned_gc_ref(store, gc_ref); - Some( - rooted - ._to_manually_rooted(store) - .expect("rooted is in scope"), - ) - }) + >::wasm_ty_option_load( + store, + ptr.get_anyref(), + StructRef::from_cloned_gc_ref, + ) } }