From b3636ff6e5c669866e5006b9a3a678191182490b Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 18 Jun 2024 08:44:38 -0700 Subject: [PATCH] Introduce the `cranelift-bitset` crate; use it for stack maps in both Cranelift and Wasmtime (#8826) * Introduce the `cranelift-bitset` crate The eventual goal is to deduplicate bitset types between Cranelift and Wasmtime, especially their use in stack maps. * Use the `cranelift-bitset` crate inside both Cranelift and Wasmtime Mostly for stack maps, also for a variety of other random things where `cranelift_codegen::bitset::BitSet` was previously used. * Fix stack maps unit test in cranelift-codegen * Uncomment `no_std` declaration * Fix `CompountBitSet::reserve` method * Fix `CompoundBitSet::insert` method * Keep track of the max in a `CompoundBitSet` Makes a bunch of other stuff easier, and will be needed for replacing `cranelift_entity::EntitySet`'s bitset with this thing anyways. * Add missing parens * Fix a bug around insert and reserve * Implement `with_capacity` in terms of `new` and `reserve` * Rename `reserve` to `ensure_capacity` --- Cargo.lock | 10 + Cargo.toml | 1 + cranelift/bitset/Cargo.toml | 20 + cranelift/bitset/src/compound.rs | 495 ++++++++++++++++++ cranelift/bitset/src/lib.rs | 19 + cranelift/bitset/src/scalar.rs | 575 +++++++++++++++++++++ cranelift/bitset/tests/bitset.rs | 78 +++ cranelift/codegen/Cargo.toml | 2 + cranelift/codegen/meta/src/gen_inst.rs | 2 +- cranelift/codegen/src/binemit/stack_map.rs | 66 +-- cranelift/codegen/src/bitset.rs | 187 ------- cranelift/codegen/src/context.rs | 6 + cranelift/codegen/src/ir/instructions.rs | 18 +- cranelift/codegen/src/lib.rs | 2 +- cranelift/codegen/src/machinst/buffer.rs | 5 + crates/cranelift/src/compiler.rs | 21 +- crates/environ/Cargo.toml | 1 + crates/environ/src/stack_map.rs | 16 +- scripts/publish.rs | 2 + 19 files changed, 1267 insertions(+), 259 deletions(-) create mode 100644 cranelift/bitset/Cargo.toml create mode 100644 cranelift/bitset/src/compound.rs create mode 100644 cranelift/bitset/src/lib.rs create mode 100644 cranelift/bitset/src/scalar.rs create mode 100644 cranelift/bitset/tests/bitset.rs delete mode 100644 cranelift/codegen/src/bitset.rs diff --git a/Cargo.lock b/Cargo.lock index f27afc5227..a70051bfed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,6 +596,14 @@ dependencies = [ "cranelift-entity", ] +[[package]] +name = "cranelift-bitset" +version = "0.110.0" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "cranelift-codegen" version = "0.110.0" @@ -604,6 +612,7 @@ dependencies = [ "bumpalo", "capstone", "cranelift-bforest", + "cranelift-bitset", "cranelift-codegen-meta", "cranelift-codegen-shared", "cranelift-control", @@ -3644,6 +3653,7 @@ dependencies = [ "anyhow", "clap", "cpp_demangle", + "cranelift-bitset", "cranelift-entity", "env_logger", "gimli", diff --git a/Cargo.toml b/Cargo.toml index 9d5946b2e9..9b8a02f4f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -224,6 +224,7 @@ cranelift-object = { path = "cranelift/object", version = "0.110.0" } cranelift-jit = { path = "cranelift/jit", version = "0.110.0" } cranelift-fuzzgen = { path = "cranelift/fuzzgen" } cranelift-bforest = { path = "cranelift/bforest", version = "0.110.0" } +cranelift-bitset = { path = "cranelift/bitset", version = "0.110.0" } cranelift-control = { path = "cranelift/control", version = "0.110.0" } cranelift = { path = "cranelift/umbrella", version = "0.110.0" } diff --git a/cranelift/bitset/Cargo.toml b/cranelift/bitset/Cargo.toml new file mode 100644 index 0000000000..f0958cc5e9 --- /dev/null +++ b/cranelift/bitset/Cargo.toml @@ -0,0 +1,20 @@ +[package] +authors = ["The Cranelift Project Developers"] +name = "cranelift-bitset" +version = "0.110.0" +description = "Various bitset stuff for use inside Cranelift" +license = "Apache-2.0 WITH LLVM-exception" +documentation = "https://docs.rs/cranelift-bitset" +repository = "https://github.com/bytecodealliance/wasmtime" +readme = "README.md" +edition.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true, optional = true } +serde_derive = { workspace = true, optional = true } + +[features] +enable-serde = ["dep:serde", "dep:serde_derive"] diff --git a/cranelift/bitset/src/compound.rs b/cranelift/bitset/src/compound.rs new file mode 100644 index 0000000000..bbd3f3d04f --- /dev/null +++ b/cranelift/bitset/src/compound.rs @@ -0,0 +1,495 @@ +//! Compound bit sets. + +use crate::scalar::{self, ScalarBitSet}; +use alloc::{vec, vec::Vec}; +use core::mem; + +/// A large bit set backed by dynamically-sized storage. +/// +/// # Example +/// +/// ``` +/// use cranelift_bitset::CompoundBitSet; +/// +/// // Create a new bitset. +/// let mut bitset = CompoundBitSet::new(); +/// +/// // Bitsets are initially empty. +/// assert!(bitset.is_empty()); +/// assert_eq!(bitset.len(), 0); +/// +/// // Insert into the bitset. +/// bitset.insert(444); +/// bitset.insert(555); +/// bitset.insert(666); +/// +/// // Now the bitset is not empty. +/// assert_eq!(bitset.len(), 3); +/// assert!(!bitset.is_empty()); +/// assert!(bitset.contains(444)); +/// assert!(bitset.contains(555)); +/// assert!(bitset.contains(666)); +/// +/// // Remove an element from the bitset. +/// let was_present = bitset.remove(666); +/// assert!(was_present); +/// assert!(!bitset.contains(666)); +/// assert_eq!(bitset.len(), 2); +/// +/// // Can iterate over the elements in the set. +/// let elems: Vec<_> = bitset.iter().collect(); +/// assert_eq!(elems, [444, 555]); +/// ``` +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub struct CompoundBitSet { + elems: Vec>, + len: usize, + max: Option, +} + +impl core::fmt::Debug for CompoundBitSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "CompoundBitSet ")?; + f.debug_set().entries(self.iter()).finish() + } +} + +impl Default for CompoundBitSet { + #[inline] + fn default() -> Self { + Self::new() + } +} + +const BITS_PER_WORD: usize = mem::size_of::() * 8; + +impl CompoundBitSet { + /// Construct a new, empty bit set. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let bitset = CompoundBitSet::new(); + /// + /// assert!(bitset.is_empty()); + /// ``` + #[inline] + pub fn new() -> Self { + CompoundBitSet { + elems: vec![], + len: 0, + max: None, + } + } + + /// Construct a new, empty bit set with space reserved to store any element + /// `x` such that `x < capacity`. + /// + /// The actual capacity reserved may be greater than that requested. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let bitset = CompoundBitSet::with_capacity(4096); + /// + /// assert!(bitset.is_empty()); + /// assert!(bitset.capacity() >= 4096); + /// ``` + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + let mut bitset = Self::new(); + bitset.ensure_capacity(capacity); + bitset + } + + /// Get the number of elements in this bitset. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// assert_eq!(bitset.len(), 0); + /// + /// bitset.insert(24); + /// bitset.insert(130); + /// bitset.insert(3600); + /// + /// assert_eq!(bitset.len(), 3); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Get `n + 1` where `n` is the largest value that can be stored inside + /// this set without growing the backing storage. + /// + /// That is, this set can store any value `x` such that `x < + /// bitset.capacity()` without growing the backing storage. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// // New bitsets have zero capacity -- they allocate lazily. + /// assert_eq!(bitset.capacity(), 0); + /// + /// // Insert into the bitset, growing its capacity. + /// bitset.insert(999); + /// + /// // The bitset must now have capacity for at least `999` elements, + /// // perhaps more. + /// assert!(bitset.capacity() >= 999); + ///``` + pub fn capacity(&self) -> usize { + self.elems.capacity() * BITS_PER_WORD + } + + /// Is this bitset empty? + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// assert!(bitset.is_empty()); + /// + /// bitset.insert(1234); + /// + /// assert!(!bitset.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Convert an element `i` into the `word` that can be used to index into + /// `self.elems` and the `bit` that can be tested in the + /// `ScalarBitSet` at `self.elems[word]`. + #[inline] + fn word_and_bit(i: usize) -> (usize, u8) { + let word = i / BITS_PER_WORD; + let bit = i % BITS_PER_WORD; + let bit = u8::try_from(bit).unwrap(); + (word, bit) + } + + /// The opposite of `word_and_bit`: convert the pair of an index into + /// `self.elems` and associated bit index into a set element. + #[inline] + fn elem(word: usize, bit: u8) -> usize { + let bit = usize::from(bit); + debug_assert!(bit < BITS_PER_WORD); + word * BITS_PER_WORD + bit + } + + /// Is `i` contained in this bitset? + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// assert!(!bitset.contains(666)); + /// + /// bitset.insert(666); + /// + /// assert!(bitset.contains(666)); + /// ``` + #[inline] + pub fn contains(&self, i: usize) -> bool { + let (word, bit) = Self::word_and_bit(i); + if word < self.elems.len() { + self.elems[word].contains(bit) + } else { + false + } + } + + /// Ensure there is space in this bitset for the values `0..n`, growing the + /// backing storage if necessary. + /// + /// After calling `bitset.ensure_capacity(n)`, inserting any element `i` + /// where `i < n` is guaranteed to succeed without growing the bitset's + /// backing storage. + /// + /// # Example + /// + /// ``` + /// # use cranelift_bitset::CompoundBitSet; + /// # let mut bitset = CompoundBitSet::new(); + /// // We are going to do a series of inserts where `1000` will be the + /// // maximum value inserted. Make sure that our bitset has capacity for + /// // these elements once up front, to avoid growing the backing storage + /// // multiple times incrementally. + /// bitset.ensure_capacity(1001); + /// + /// for i in 0..=1000 { + /// if i % 2 == 0 { + /// // Inserting this value should not require growing the backing + /// // storage. + /// assert!(bitset.capacity() > i); + /// bitset.insert(i); + /// } + /// } + /// ``` + #[inline] + pub fn ensure_capacity(&mut self, n: usize) { + let (word, _bit) = Self::word_and_bit(n); + if word >= self.elems.len() { + self.elems.resize_with(word + 1, ScalarBitSet::new); + } + } + + /// Insert `i` into this bitset. + /// + /// Returns whether the value was newly inserted. That is: + /// + /// * If the set did not previously contain `i` then `true` is returned. + /// + /// * If the set already contained `i` then `false` is returned. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// // When an element is inserted that was not already present in the set, + /// // then `true` is returned. + /// let is_new = bitset.insert(1234); + /// assert!(is_new); + /// + /// // The element is now present in the set. + /// assert!(bitset.contains(1234)); + /// + /// // And when the element is already in the set, `false` is returned from + /// // `insert`. + /// let is_new = bitset.insert(1234); + /// assert!(!is_new); + /// ``` + #[inline] + pub fn insert(&mut self, i: usize) -> bool { + self.ensure_capacity(i + 1); + let (word, bit) = Self::word_and_bit(i); + let is_new = self.elems[word].insert(bit); + self.len += is_new as usize; + self.max = self.max.map(|max| core::cmp::max(max, i)).or(Some(i)); + is_new + } + + /// Remove `i` from this bitset. + /// + /// Returns whether `i` was previously in this set or not. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// // Removing an element that was not present in the set returns `false`. + /// let was_present = bitset.remove(456); + /// assert!(!was_present); + /// + /// // And when the element was in the set, `true` is returned. + /// bitset.insert(456); + /// let was_present = bitset.remove(456); + /// assert!(was_present); + /// ``` + #[inline] + pub fn remove(&mut self, i: usize) -> bool { + let (word, bit) = Self::word_and_bit(i); + if word < self.elems.len() { + let sub = &mut self.elems[word]; + let was_present = sub.remove(bit); + self.len -= was_present as usize; + if was_present && self.max == Some(i) { + self.update_max(word); + } + was_present + } else { + false + } + } + + /// Update the `self.max` field, based on the old word index of `self.max`. + fn update_max(&mut self, word_of_old_max: usize) { + self.max = self.elems[0..word_of_old_max + 1] + .iter() + .enumerate() + .rev() + .filter_map(|(word, sub)| { + let bit = sub.max()?; + Some(Self::elem(word, bit)) + }) + .next(); + } + + /// Get the largest value in this set, or `None` if this set is empty. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// // Returns `None` if the bitset is empty. + /// assert!(bitset.max().is_none()); + /// + /// bitset.insert(123); + /// bitset.insert(987); + /// bitset.insert(999); + /// + /// // Otherwise, it returns the largest value in the set. + /// assert_eq!(bitset.max(), Some(999)); + /// ``` + #[inline] + pub fn max(&self) -> Option { + self.max + } + + /// Removes and returns the largest value in this set. + /// + /// Returns `None` if this set is empty. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// bitset.insert(111); + /// bitset.insert(222); + /// bitset.insert(333); + /// bitset.insert(444); + /// bitset.insert(555); + /// + /// assert_eq!(bitset.pop(), Some(555)); + /// assert_eq!(bitset.pop(), Some(444)); + /// assert_eq!(bitset.pop(), Some(333)); + /// assert_eq!(bitset.pop(), Some(222)); + /// assert_eq!(bitset.pop(), Some(111)); + /// assert_eq!(bitset.pop(), None); + /// ``` + #[inline] + pub fn pop(&mut self) -> Option { + let max = self.max?; + self.remove(max); + Some(max) + } + + /// Remove all elements from this bitset. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// bitset.insert(100); + /// bitset.insert(200); + /// bitset.insert(300); + /// + /// bitset.clear(); + /// + /// assert!(bitset.is_empty()); + /// ``` + #[inline] + pub fn clear(&mut self) { + self.elems.clear(); + self.len = 0; + self.max = None; + } + + /// Iterate over the elements in this bitset. + /// + /// The elements are always yielded in sorted order. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::CompoundBitSet; + /// + /// let mut bitset = CompoundBitSet::new(); + /// + /// bitset.insert(0); + /// bitset.insert(4096); + /// bitset.insert(123); + /// bitset.insert(456); + /// bitset.insert(789); + /// + /// assert_eq!( + /// bitset.iter().collect::>(), + /// [0, 123, 456, 789, 4096], + /// ); + /// ``` + #[inline] + pub fn iter(&self) -> Iter<'_> { + Iter { + bitset: self, + word: 0, + sub: None, + } + } +} + +impl<'a> IntoIterator for &'a CompoundBitSet { + type Item = usize; + + type IntoIter = Iter<'a>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// An iterator over the elements in a [`CompoundBitSet`]. +pub struct Iter<'a> { + bitset: &'a CompoundBitSet, + word: usize, + sub: Option>, +} + +impl Iterator for Iter<'_> { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + loop { + if let Some(sub) = &mut self.sub { + if let Some(bit) = sub.next() { + return Some(CompoundBitSet::elem(self.word, bit)); + } else { + self.word += 1; + } + } + + self.sub = Some(self.bitset.elems.get(self.word)?.iter()); + } + } +} diff --git a/cranelift/bitset/src/lib.rs b/cranelift/bitset/src/lib.rs new file mode 100644 index 0000000000..5a9fd354aa --- /dev/null +++ b/cranelift/bitset/src/lib.rs @@ -0,0 +1,19 @@ +//! Bitsets for Cranelift. +//! +//! This module provides two bitset implementations: +//! +//! 1. [`ScalarBitSet`]: A small bitset built on top of a single integer. +//! +//! 2. [`CompoundBitSet`]: A bitset that can store more bits than fit in a +//! single integer, but which internally has heap allocations. + +#![deny(missing_docs)] +#![no_std] + +extern crate alloc; + +pub mod compound; +pub mod scalar; + +pub use compound::CompoundBitSet; +pub use scalar::ScalarBitSet; diff --git a/cranelift/bitset/src/scalar.rs b/cranelift/bitset/src/scalar.rs new file mode 100644 index 0000000000..2d959016ca --- /dev/null +++ b/cranelift/bitset/src/scalar.rs @@ -0,0 +1,575 @@ +//! Scalar bitsets. + +use core::mem::size_of; +use core::ops::{Add, BitAnd, BitOr, Not, Shl, Shr, Sub}; + +/// A small bitset built on top of a single primitive integer type. +/// +/// # Example +/// +/// ``` +/// use cranelift_bitset::ScalarBitSet; +/// +/// // Create a new bitset backed with a `u32`. +/// let mut bitset = ScalarBitSet::::new(); +/// +/// // Bitsets are initially empty. +/// assert!(bitset.is_empty()); +/// assert_eq!(bitset.len(), 0); +/// +/// // Insert into the bitset. +/// bitset.insert(4); +/// bitset.insert(5); +/// bitset.insert(6); +/// +/// // Now the bitset is not empty. +/// assert_eq!(bitset.len(), 3); +/// assert!(!bitset.is_empty()); +/// assert!(bitset.contains(4)); +/// assert!(bitset.contains(5)); +/// assert!(bitset.contains(6)); +/// +/// // Remove an element from the bitset. +/// let was_present = bitset.remove(6); +/// assert!(was_present); +/// assert!(!bitset.contains(6)); +/// assert_eq!(bitset.len(), 2); +/// +/// // Can iterate over the elements in the set. +/// let elems: Vec<_> = bitset.iter().collect(); +/// assert_eq!(elems, [4, 5]); +/// ``` +#[derive(Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub struct ScalarBitSet(pub T); + +impl core::fmt::Debug for ScalarBitSet +where + T: ScalarBitSetStorage, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut s = f.debug_struct(core::any::type_name::()); + for i in 0..Self::capacity() { + use alloc::string::ToString; + let i = u8::try_from(i).unwrap(); + s.field(&i.to_string(), &self.contains(i)); + } + s.finish() + } +} + +impl Default for ScalarBitSet +where + T: ScalarBitSetStorage, +{ + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl ScalarBitSet +where + T: ScalarBitSetStorage, +{ + /// Create a new, empty bitset. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let bitset = ScalarBitSet::::new(); + /// + /// assert!(bitset.is_empty()); + /// ``` + #[inline] + pub fn new() -> Self { + Self(T::from(0)) + } + + /// Construct a bitset with the half-open range `[lo, hi)` inserted. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let bitset = ScalarBitSet::::from_range(3, 6); + /// + /// assert_eq!(bitset.len(), 3); + /// + /// assert!(bitset.contains(3)); + /// assert!(bitset.contains(4)); + /// assert!(bitset.contains(5)); + /// ``` + /// + /// # Panics + /// + /// Panics if `lo > hi` or if `hi > Self::capacity()`. + /// + /// ```should_panic + /// use cranelift_bitset::ScalarBitSet; + /// + /// // The lower bound may not be larger than the upper bound. + /// let bitset = ScalarBitSet::::from_range(6, 3); + /// ``` + /// + /// ```should_panic + /// use cranelift_bitset::ScalarBitSet; + /// + /// // The bounds must fit within the backing scalar type. + /// let bitset = ScalarBitSet::::from_range(3, 69); + /// ``` + #[inline] + pub fn from_range(lo: u8, hi: u8) -> Self { + assert!(lo <= hi); + assert!(hi <= Self::capacity()); + + let one = T::from(1); + + // We can't just do (one << hi) - one here as the shift may overflow + let hi_rng = if hi >= 1 { + (one << (hi - 1)) + ((one << (hi - 1)) - one) + } else { + T::from(0) + }; + + let lo_rng = (one << lo) - one; + + Self(hi_rng - lo_rng) + } + + /// The maximum number of bits that can be stored in this bitset. + /// + /// If you need more bits than this, use a + /// [`CompoundBitSet`][crate::CompoundBitSet] instead of a `ScalarBitSet`. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// assert_eq!(ScalarBitSet::::capacity(), 8); + /// assert_eq!(ScalarBitSet::::capacity(), 64); + /// ``` + #[inline] + pub fn capacity() -> u8 { + u8::try_from(size_of::()).unwrap() * 8 + } + + /// Get the number of elements in this set. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// assert_eq!(bitset.len(), 0); + /// + /// bitset.insert(24); + /// bitset.insert(13); + /// bitset.insert(36); + /// + /// assert_eq!(bitset.len(), 3); + /// ``` + #[inline] + pub fn len(&self) -> u8 { + self.0.count_ones() + } + + /// Is this bitset empty? + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// assert!(bitset.is_empty()); + /// + /// bitset.insert(10); + /// + /// assert!(!bitset.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.0 == T::from(0) + } + + /// Check whether this bitset contains `i`. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// assert!(!bitset.contains(7)); + /// + /// bitset.insert(7); + /// + /// assert!(bitset.contains(7)); + /// ``` + /// + /// # Panics + /// + /// Panics if `i` is greater than or equal to [`Self::capacity()`][Self::capacity]. + /// + /// ```should_panic + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // A `ScalarBitSet` can only hold the elements `0..=7`, so `8` is + /// // out of bounds and will trigger a panic. + /// bitset.contains(8); + /// ``` + #[inline] + pub fn contains(&self, i: u8) -> bool { + assert!(i < Self::capacity()); + self.0 & (T::from(1) << i) != T::from(0) + } + + /// Insert `i` into this bitset. + /// + /// Returns whether the value was newly inserted. That is: + /// + /// * If the set did not previously contain `i` then `true` is returned. + /// + /// * If the set already contained `i` then `false` is returned. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // When an element is inserted that was not already present in the set, + /// // then `true` is returned. + /// let is_new = bitset.insert(7); + /// assert!(is_new); + /// + /// // The element is now present in the set. + /// assert!(bitset.contains(7)); + /// + /// // And when the element is already in the set, `false` is returned from + /// // `insert`. + /// let is_new = bitset.insert(7); + /// assert!(!is_new); + /// ``` + /// + /// # Panics + /// + /// Panics if `i` is greater than or equal to [`Self::capacity()`][Self::capacity]. + /// + /// ```should_panic + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // A `ScalarBitSet` can only hold the elements `0..=31`, so `42` is + /// // out of bounds and will trigger a panic. + /// bitset.insert(42); + /// ``` + #[inline] + pub fn insert(&mut self, i: u8) -> bool { + let is_new = !self.contains(i); + self.0 = self.0 | (T::from(1) << i); + is_new + } + + /// Remove `i` from this bitset. + /// + /// Returns whether `i` was previously in this set or not. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // Removing an element that was not present in the set returns `false`. + /// let was_present = bitset.remove(100); + /// assert!(!was_present); + /// + /// // And when the element was in the set, `true` is returned. + /// bitset.insert(100); + /// let was_present = bitset.remove(100); + /// assert!(was_present); + /// ``` + /// + /// # Panics + /// + /// Panics if `i` is greater than or equal to [`Self::capacity()`][Self::capacity]. + /// + /// ```should_panic + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // A `ScalarBitSet` can only hold the elements `0..=15`, so `20` is + /// // out of bounds and will trigger a panic. + /// bitset.remove(20); + /// ``` + #[inline] + pub fn remove(&mut self, i: u8) -> bool { + let was_present = self.contains(i); + self.0 = self.0 & !(T::from(1) << i); + was_present + } + + /// Remove all entries from this bitset. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// bitset.insert(10); + /// bitset.insert(20); + /// bitset.insert(30); + /// + /// bitset.clear(); + /// + /// assert!(bitset.is_empty()); + /// ``` + #[inline] + pub fn clear(&mut self) { + self.0 = T::from(0); + } + + /// Remove and return the largest value in the bitset. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// bitset.insert(0); + /// bitset.insert(24); + /// bitset.insert(13); + /// bitset.insert(36); + /// + /// assert_eq!(bitset.pop(), Some(36)); + /// assert_eq!(bitset.pop(), Some(24)); + /// assert_eq!(bitset.pop(), Some(13)); + /// assert_eq!(bitset.pop(), Some(0)); + /// assert_eq!(bitset.pop(), None); + /// ``` + #[inline] + pub fn pop(&mut self) -> Option { + let max = self.max()?; + self.remove(max); + Some(max) + } + + /// Return the smallest number contained in this bitset or `None` if this + /// bitset is empty. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // When the bitset is empty, `min` returns `None`. + /// assert_eq!(bitset.min(), None); + /// + /// bitset.insert(28); + /// bitset.insert(1); + /// bitset.insert(63); + /// + /// // When the bitset is not empty, it returns the smallest element. + /// assert_eq!(bitset.min(), Some(1)); + /// ``` + #[inline] + pub fn min(&self) -> Option { + if self.0 == T::from(0) { + None + } else { + Some(self.0.trailing_zeros()) + } + } + + /// Return the largest number contained in the bitset or None if this bitset + /// is empty + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// // When the bitset is empty, `max` returns `None`. + /// assert_eq!(bitset.max(), None); + /// + /// bitset.insert(0); + /// bitset.insert(36); + /// bitset.insert(49); + /// + /// // When the bitset is not empty, it returns the smallest element. + /// assert_eq!(bitset.max(), Some(49)); + /// ``` + #[inline] + pub fn max(&self) -> Option { + if self.0 == T::from(0) { + None + } else { + let leading_zeroes = self.0.leading_zeros(); + Some(Self::capacity() - leading_zeroes - 1) + } + } + + /// Iterate over the items in this set. + /// + /// Items are always yielded in sorted order. + /// + /// # Example + /// + /// ``` + /// use cranelift_bitset::ScalarBitSet; + /// + /// let mut bitset = ScalarBitSet::::new(); + /// + /// bitset.insert(19); + /// bitset.insert(3); + /// bitset.insert(63); + /// bitset.insert(0); + /// + /// assert_eq!( + /// bitset.iter().collect::>(), + /// [0, 3, 19, 63], + /// ); + /// ``` + #[inline] + pub fn iter(&self) -> Iter { + Iter { + value: self.0, + index: 0, + } + } +} + +impl IntoIterator for ScalarBitSet +where + T: ScalarBitSetStorage, +{ + type Item = u8; + + type IntoIter = Iter; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, T> IntoIterator for &'a ScalarBitSet +where + T: ScalarBitSetStorage, +{ + type Item = u8; + + type IntoIter = Iter; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// A trait implemented by all integers that can be used as the backing storage +/// for a [`ScalarBitSet`]. +/// +/// You shouldn't have to implement this yourself, it is already implemented for +/// `u{8,16,32,64,128}` and if you need more bits than that, then use +/// [`CompoundBitSet`][crate::CompoundBitSet] instead. +pub trait ScalarBitSetStorage: + Default + + From + + Shl + + Shr + + BitAnd + + BitOr + + Not + + Sub + + Add + + PartialEq + + Copy +{ + /// Count the number of leading zeros. + fn leading_zeros(self) -> u8; + + /// Count the number of trailing zeros. + fn trailing_zeros(self) -> u8; + + /// Count the number of bits set in this integer. + fn count_ones(self) -> u8; +} + +macro_rules! impl_storage { + ( $int:ty ) => { + impl ScalarBitSetStorage for $int { + fn leading_zeros(self) -> u8 { + u8::try_from(self.leading_zeros()).unwrap() + } + + fn trailing_zeros(self) -> u8 { + u8::try_from(self.trailing_zeros()).unwrap() + } + + fn count_ones(self) -> u8 { + u8::try_from(self.count_ones()).unwrap() + } + } + }; +} + +impl_storage!(u8); +impl_storage!(u16); +impl_storage!(u32); +impl_storage!(u64); +impl_storage!(u128); +impl_storage!(usize); + +/// An iterator over the elements in a [`ScalarBitSet`]. +pub struct Iter { + value: T, + index: u8, +} + +impl Iterator for Iter +where + T: ScalarBitSetStorage, +{ + type Item = u8; + + #[inline] + fn next(&mut self) -> Option { + if self.value == T::from(0) { + None + } else { + let trailing_zeros = self.value.trailing_zeros(); + let elem = self.index + trailing_zeros; + self.index += trailing_zeros + 1; + self.value = self.value >> (trailing_zeros + 1); + Some(elem) + } + } +} diff --git a/cranelift/bitset/tests/bitset.rs b/cranelift/bitset/tests/bitset.rs new file mode 100644 index 0000000000..d0f5aedd64 --- /dev/null +++ b/cranelift/bitset/tests/bitset.rs @@ -0,0 +1,78 @@ +use cranelift_bitset::*; + +#[test] +fn contains() { + let s = ScalarBitSet::(255); + for i in 0..7 { + assert!(s.contains(i)); + } + + let s1 = ScalarBitSet::(0); + for i in 0..7 { + assert!(!s1.contains(i)); + } + + let s2 = ScalarBitSet::(127); + for i in 0..6 { + assert!(s2.contains(i)); + } + assert!(!s2.contains(7)); + + let s3 = ScalarBitSet::(2 | 4 | 64); + assert!(!s3.contains(0) && !s3.contains(3) && !s3.contains(4)); + assert!(!s3.contains(5) && !s3.contains(7)); + assert!(s3.contains(1) && s3.contains(2) && s3.contains(6)); + + let s4 = ScalarBitSet::(4 | 8 | 256 | 1024); + assert!( + !s4.contains(0) + && !s4.contains(1) + && !s4.contains(4) + && !s4.contains(5) + && !s4.contains(6) + && !s4.contains(7) + && !s4.contains(9) + && !s4.contains(11) + ); + assert!(s4.contains(2) && s4.contains(3) && s4.contains(8) && s4.contains(10)); +} + +#[test] +fn minmax() { + let s = ScalarBitSet::(255); + assert_eq!(s.min(), Some(0)); + assert_eq!(s.max(), Some(7)); + assert!(s.min() == Some(0) && s.max() == Some(7)); + let s1 = ScalarBitSet::(0); + assert!(s1.min() == None && s1.max() == None); + let s2 = ScalarBitSet::(127); + assert!(s2.min() == Some(0) && s2.max() == Some(6)); + let s3 = ScalarBitSet::(2 | 4 | 64); + assert!(s3.min() == Some(1) && s3.max() == Some(6)); + let s4 = ScalarBitSet::(4 | 8 | 256 | 1024); + assert!(s4.min() == Some(2) && s4.max() == Some(10)); +} + +#[test] +fn from_range() { + let s = ScalarBitSet::::from_range(5, 5); + assert!(s.0 == 0); + + let s = ScalarBitSet::::from_range(0, 8); + assert!(s.0 == 255); + + let s = ScalarBitSet::::from_range(0, 8); + assert!(s.0 == 255u16); + + let s = ScalarBitSet::::from_range(0, 16); + assert!(s.0 == 65535u16); + + let s = ScalarBitSet::::from_range(5, 6); + assert!(s.0 == 32u8); + + let s = ScalarBitSet::::from_range(3, 7); + assert!(s.0 == 8 | 16 | 32 | 64); + + let s = ScalarBitSet::::from_range(5, 11); + assert!(s.0 == 32 | 64 | 128 | 256 | 512 | 1024); +} diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 1750fdbdb9..68744d2bbd 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -22,6 +22,7 @@ capstone = { workspace = true, optional = true } cranelift-codegen-shared = { path = "./shared", version = "0.110.0" } cranelift-entity = { workspace = true } cranelift-bforest = { workspace = true } +cranelift-bitset = { workspace = true } cranelift-control = { workspace = true } hashbrown = { workspace = true, features = ["raw"] } target-lexicon = { workspace = true } @@ -95,6 +96,7 @@ enable-serde = [ "serde", "serde_derive", "cranelift-entity/enable-serde", + "cranelift-bitset/enable-serde", "regalloc2/enable-serde", "smallvec/serde" ] diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index 3ad9bd544f..82a607ef60 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -725,7 +725,7 @@ fn gen_bitset<'a, T: IntoIterator>( assert!(u32::from(*x) < (1 << u32::from(field_size))); acc | x }); - fmtln!(fmt, "{}: BitSet::({}),", name, field_size, bits); + fmtln!(fmt, "{}: ScalarBitSet::({}),", name, field_size, bits); } fn iterable_to_string>(iterable: T) -> String { diff --git a/cranelift/codegen/src/binemit/stack_map.rs b/cranelift/codegen/src/binemit/stack_map.rs index 037611e6ea..b5d1b694d7 100644 --- a/cranelift/codegen/src/binemit/stack_map.rs +++ b/cranelift/codegen/src/binemit/stack_map.rs @@ -1,8 +1,4 @@ -use crate::bitset::BitSet; -use alloc::vec::Vec; - -type Num = u32; -const NUM_BITS: usize = core::mem::size_of::() * 8; +use cranelift_bitset::CompoundBitSet; /// Stack maps record which words in a stack frame contain live GC references at /// a given instruction pointer. @@ -72,43 +68,33 @@ const NUM_BITS: usize = core::mem::size_of::() * 8; derive(serde_derive::Deserialize, serde_derive::Serialize) )] pub struct StackMap { - bitmap: Vec>, + bitset: CompoundBitSet, mapped_words: u32, } impl StackMap { - /// Create a vec of Bitsets from a slice of bools. - pub fn from_slice(vec: &[bool]) -> Self { - let len = vec.len(); - let num_word = len / NUM_BITS + (len % NUM_BITS != 0) as usize; - let mut bitmap = Vec::with_capacity(num_word); - - for segment in vec.chunks(NUM_BITS) { - let mut curr_word = 0; - for (i, set) in segment.iter().enumerate() { - if *set { - curr_word |= 1 << i; - } + /// Create a stack map from a slice of booleans. + pub fn from_slice(bools: &[bool]) -> Self { + let mut bitset = CompoundBitSet::with_capacity(bools.len()); + for (i, b) in bools.iter().enumerate() { + if *b { + bitset.insert(i); } - bitmap.push(BitSet(curr_word)); } Self { - mapped_words: len as u32, - bitmap, + mapped_words: u32::try_from(bools.len()).unwrap(), + bitset, } } /// Returns a specified bit. pub fn get_bit(&self, bit_index: usize) -> bool { - assert!(bit_index < NUM_BITS * self.bitmap.len()); - let word_index = bit_index / NUM_BITS; - let word_offset = bit_index % NUM_BITS; - self.bitmap[word_index].contains(word_offset as u32) + self.bitset.contains(bit_index) } /// Returns the raw bitmap that represents this stack map. - pub fn as_slice(&self) -> &[BitSet] { - &self.bitmap + pub fn into_bitset(self) -> CompoundBitSet { + self.bitset } /// Returns the number of words represented by this stack map. @@ -120,11 +106,15 @@ impl StackMap { #[cfg(test)] mod tests { use super::*; + use alloc::vec::Vec; + + type Num = u32; + const NUM_BITS: usize = core::mem::size_of::() * 8; #[test] fn stack_maps() { let vec: Vec = Vec::new(); - assert!(StackMap::from_slice(&vec).bitmap.is_empty()); + assert!(StackMap::from_slice(&vec).bitset.is_empty()); let mut vec: [bool; NUM_BITS] = Default::default(); let set_true_idx = [5, 7, 24, 31]; @@ -134,22 +124,18 @@ mod tests { } let mut vec = vec.to_vec(); - assert_eq!( - vec![BitSet::(2164261024)], - StackMap::from_slice(&vec).bitmap - ); + let stack_map = StackMap::from_slice(&vec); + for idx in 0..32 { + assert_eq!(stack_map.get_bit(idx), set_true_idx.contains(&idx)); + } vec.push(false); vec.push(true); let res = StackMap::from_slice(&vec); - assert_eq!( - vec![BitSet::(2164261024), BitSet::(2)], - res.bitmap - ); - - assert!(res.get_bit(5)); - assert!(res.get_bit(31)); + for idx in 0..32 { + assert_eq!(stack_map.get_bit(idx), set_true_idx.contains(&idx)); + } + assert!(!res.get_bit(32)); assert!(res.get_bit(33)); - assert!(!res.get_bit(1)); } } diff --git a/cranelift/codegen/src/bitset.rs b/cranelift/codegen/src/bitset.rs deleted file mode 100644 index 70c86d7222..0000000000 --- a/cranelift/codegen/src/bitset.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! Small Bitset -//! -//! This module defines a struct `BitSet` encapsulating a bitset built over the type T. -//! T is intended to be a primitive unsigned type. Currently it can be any type between u8 and u32 -//! -//! If you would like to add support for larger bitsets in the future, you need to change the trait -//! bound `Into` and the `u32` in the implementation of `max_bits()`. - -use core::mem::size_of; -use core::ops::{Add, BitOr, Shl, Sub}; - -/// A small bitset built on a single primitive integer type -#[derive(Clone, Copy, Default, PartialEq, Eq)] -#[cfg_attr( - feature = "enable-serde", - derive(serde_derive::Serialize, serde_derive::Deserialize) -)] -pub struct BitSet(pub T); - -impl std::fmt::Debug for BitSet -where - T: Into - + From - + BitOr - + Shl - + Sub - + Add - + PartialEq - + Copy, -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut s = f.debug_struct(std::any::type_name::()); - for i in 0..Self::bits() { - use std::string::ToString; - let i = u32::try_from(i).unwrap(); - s.field(&i.to_string(), &self.contains(i)); - } - s.finish() - } -} - -impl BitSet -where - T: Into - + From - + BitOr - + Shl - + Sub - + Add - + PartialEq - + Copy, -{ - /// Maximum number of bits supported by this BitSet instance - pub fn bits() -> usize { - size_of::() * 8 - } - - /// Maximum number of bits supported by any bitset instance atm. - pub fn max_bits() -> usize { - size_of::() * 8 - } - - /// Check if this BitSet contains the number num - pub fn contains(&self, num: u32) -> bool { - debug_assert!((num as usize) < Self::bits()); - debug_assert!((num as usize) < Self::max_bits()); - self.0.into() & (1 << num) != 0 - } - - /// Return the smallest number contained in the bitset or None if empty - pub fn min(&self) -> Option { - if self.0.into() == 0 { - None - } else { - Some(self.0.into().trailing_zeros() as u8) - } - } - - /// Return the largest number contained in the bitset or None if empty - pub fn max(&self) -> Option { - if self.0.into() == 0 { - None - } else { - let leading_zeroes = self.0.into().leading_zeros() as usize; - Some((Self::max_bits() - leading_zeroes - 1) as u8) - } - } - - /// Construct a BitSet with the half-open range [lo,hi) filled in - pub fn from_range(lo: u8, hi: u8) -> Self { - debug_assert!(lo <= hi); - debug_assert!((hi as usize) <= Self::bits()); - let one: T = T::from(1); - // I can't just do (one << hi) - one here as the shift may overflow - let hi_rng = if hi >= 1 { - (one << (hi - 1)) + ((one << (hi - 1)) - one) - } else { - T::from(0) - }; - - let lo_rng = (one << lo) - one; - - Self(hi_rng - lo_rng) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn contains() { - let s = BitSet::(255); - for i in 0..7 { - assert!(s.contains(i)); - } - - let s1 = BitSet::(0); - for i in 0..7 { - assert!(!s1.contains(i)); - } - - let s2 = BitSet::(127); - for i in 0..6 { - assert!(s2.contains(i)); - } - assert!(!s2.contains(7)); - - let s3 = BitSet::(2 | 4 | 64); - assert!(!s3.contains(0) && !s3.contains(3) && !s3.contains(4)); - assert!(!s3.contains(5) && !s3.contains(7)); - assert!(s3.contains(1) && s3.contains(2) && s3.contains(6)); - - let s4 = BitSet::(4 | 8 | 256 | 1024); - assert!( - !s4.contains(0) - && !s4.contains(1) - && !s4.contains(4) - && !s4.contains(5) - && !s4.contains(6) - && !s4.contains(7) - && !s4.contains(9) - && !s4.contains(11) - ); - assert!(s4.contains(2) && s4.contains(3) && s4.contains(8) && s4.contains(10)); - } - - #[test] - fn minmax() { - let s = BitSet::(255); - assert_eq!(s.min(), Some(0)); - assert_eq!(s.max(), Some(7)); - assert!(s.min() == Some(0) && s.max() == Some(7)); - let s1 = BitSet::(0); - assert!(s1.min() == None && s1.max() == None); - let s2 = BitSet::(127); - assert!(s2.min() == Some(0) && s2.max() == Some(6)); - let s3 = BitSet::(2 | 4 | 64); - assert!(s3.min() == Some(1) && s3.max() == Some(6)); - let s4 = BitSet::(4 | 8 | 256 | 1024); - assert!(s4.min() == Some(2) && s4.max() == Some(10)); - } - - #[test] - fn from_range() { - let s = BitSet::::from_range(5, 5); - assert!(s.0 == 0); - - let s = BitSet::::from_range(0, 8); - assert!(s.0 == 255); - - let s = BitSet::::from_range(0, 8); - assert!(s.0 == 255u16); - - let s = BitSet::::from_range(0, 16); - assert!(s.0 == 65535u16); - - let s = BitSet::::from_range(5, 6); - assert!(s.0 == 32u8); - - let s = BitSet::::from_range(3, 7); - assert!(s.0 == 8 | 16 | 32 | 64); - - let s = BitSet::::from_range(5, 11); - assert!(s.0 == 32 | 64 | 128 | 256 | 512 | 1024); - } -} diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index 856bb40856..d95e985a48 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -96,6 +96,12 @@ impl Context { self.compiled_code.as_ref() } + /// Returns the compilation result for this function, available after any `compile` function + /// has been called. + pub fn take_compiled_code(&mut self) -> Option { + self.compiled_code.take() + } + /// Set the flag to request a disassembly when compiling with a /// `MachBackend` backend. pub fn set_disasm(&mut self, val: bool) { diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index 9a9254264f..fd5208070f 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -15,7 +15,7 @@ use core::str::FromStr; #[cfg(feature = "enable-serde")] use serde_derive::{Deserialize, Serialize}; -use crate::bitset::BitSet; +use crate::bitset::ScalarBitSet; use crate::entity; use crate::ir::{ self, @@ -619,8 +619,8 @@ impl OpcodeConstraints { } } -type BitSet8 = BitSet; -type BitSet16 = BitSet; +type BitSet8 = ScalarBitSet; +type BitSet16 = ScalarBitSet; /// A value type set describes the permitted set of types for a type variable. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -642,7 +642,7 @@ impl ValueTypeSet { /// /// Note that the base type set does not have to be included in the type set proper. fn is_base_type(self, scalar: Type) -> bool { - let l2b = scalar.log2_lane_bits(); + let l2b = u8::try_from(scalar.log2_lane_bits()).unwrap(); if scalar.is_int() { self.ints.contains(l2b) } else if scalar.is_float() { @@ -657,10 +657,10 @@ impl ValueTypeSet { /// Does `typ` belong to this set? pub fn contains(self, typ: Type) -> bool { if typ.is_dynamic_vector() { - let l2l = typ.log2_min_lane_count(); + let l2l = u8::try_from(typ.log2_min_lane_count()).unwrap(); self.dynamic_lanes.contains(l2l) && self.is_base_type(typ.lane_type()) } else { - let l2l = typ.log2_lane_count(); + let l2l = u8::try_from(typ.log2_lane_count()).unwrap(); self.lanes.contains(l2l) && self.is_base_type(typ.lane_type()) } } @@ -786,7 +786,7 @@ impl OperandConstraint { let mut tys = ValueTypeSet::default(); // We're testing scalar values, only. - tys.lanes = BitSet::from_range(0, 1); + tys.lanes = ScalarBitSet::from_range(0, 1); if ctrl_type.is_int() { // The upper bound in from_range is exclusive, and we want to exclude the @@ -806,7 +806,7 @@ impl OperandConstraint { let mut tys = ValueTypeSet::default(); // We're testing scalar values, only. - tys.lanes = BitSet::from_range(0, 1); + tys.lanes = ScalarBitSet::from_range(0, 1); if ctrl_type.is_int() { let lower_bound = ctrl_type_bits as u8 + 1; @@ -815,7 +815,7 @@ impl OperandConstraint { // lower bound would overflow as 2^8 doesn't fit in a u8, but this would // already describe the empty set so instead we leave `ints` in its default // empty state. - if lower_bound < BitSet8::bits() as u8 { + if lower_bound < BitSet8::capacity() { // The interval should include all types wider than `ctrl_type`, so we use // `2^8` as the upper bound, and add one to the bits of `ctrl_type` to define // the interval `(ctrl_type, I128]`. diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 017122a8c7..a5b8a1051c 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -28,6 +28,7 @@ pub use crate::verifier::verify_function; pub use crate::write::write_function; pub use cranelift_bforest as bforest; +pub use cranelift_bitset as bitset; pub use cranelift_control as control; pub use cranelift_entity as entity; #[cfg(feature = "unwind")] @@ -65,7 +66,6 @@ pub use crate::machinst::{ }; mod alias_analysis; -mod bitset; mod constant_hash; mod context; mod ctxhash; diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index c51fcbe669..c4ffbb2f89 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -1722,6 +1722,11 @@ impl MachBufferFinalized { &self.stack_maps[..] } + /// Take this buffer's stack map metadata. + pub fn take_stack_maps(&mut self) -> SmallVec<[MachStackMap; 8]> { + mem::take(&mut self.stack_maps) + } + /// Get the list of call sites for this code. pub fn call_sites(&self) -> &[MachCallSite] { &self.call_sites[..] diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index bb44575bda..80075c6487 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -815,7 +815,7 @@ impl FunctionCompiler<'_> { let isa = &*self.compiler.isa; let (_, _code_buf) = compile_maybe_cached(context, isa, self.cx.incremental_cache_ctx.as_mut())?; - let compiled_code = context.compiled_code().unwrap(); + let mut compiled_code = context.take_compiled_code().unwrap(); // Give wasm functions, user defined code, a "preferred" alignment // instead of the minimum alignment as this can help perf in niche @@ -875,7 +875,8 @@ impl FunctionCompiler<'_> { } } - let stack_maps = mach_stack_maps_to_stack_maps(compiled_code.buffer.stack_maps()); + let stack_maps = + mach_stack_maps_to_stack_maps(compiled_code.buffer.take_stack_maps().into_iter()); compiled_function .set_sized_stack_slots(std::mem::take(&mut context.func.sized_stack_slots)); self.compiler.contexts.lock().unwrap().push(self.cx); @@ -890,21 +891,21 @@ impl FunctionCompiler<'_> { } } -fn mach_stack_maps_to_stack_maps(mach_stack_maps: &[MachStackMap]) -> Vec { +fn mach_stack_maps_to_stack_maps( + mach_stack_maps: impl ExactSizeIterator, +) -> Vec { // This is converting from Cranelift's representation of a stack map to // Wasmtime's representation. They happen to align today but that may // not always be true in the future. - let mut stack_maps = Vec::new(); - for &MachStackMap { + let mut stack_maps = Vec::with_capacity(mach_stack_maps.len()); + for MachStackMap { offset_end, - ref stack_map, + stack_map, .. } in mach_stack_maps { - let stack_map = wasmtime_environ::StackMap::new( - stack_map.mapped_words(), - stack_map.as_slice().iter().map(|a| a.0), - ); + let mapped_words = stack_map.mapped_words(); + let stack_map = wasmtime_environ::StackMap::new(mapped_words, stack_map.into_bitset()); stack_maps.push(StackMapInformation { code_offset: offset_end, stack_map, diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 5cef16b18e..ede92838df 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } postcard = { workspace = true } cpp_demangle = { version = "0.4.3", optional = true } cranelift-entity = { workspace = true } +cranelift-bitset = { workspace = true, features = ['enable-serde'] } wasmtime-types = { workspace = true } wasmparser = { workspace = true, features = ['validate', 'serde'] } indexmap = { workspace = true, features = ["serde"] } diff --git a/crates/environ/src/stack_map.rs b/crates/environ/src/stack_map.rs index 428dc39936..023d6b0ec4 100644 --- a/crates/environ/src/stack_map.rs +++ b/crates/environ/src/stack_map.rs @@ -1,4 +1,4 @@ -use crate::prelude::*; +use cranelift_bitset::CompoundBitSet; use serde_derive::{Deserialize, Serialize}; /// A map for determining where live GC references live in a stack frame. @@ -8,26 +8,20 @@ use serde_derive::{Deserialize, Serialize}; /// the docs over there. #[derive(Debug, Serialize, Deserialize)] pub struct StackMap { - bits: Box<[u32]>, + bits: CompoundBitSet, mapped_words: u32, } impl StackMap { /// Creates a new `StackMap`, typically from a preexisting /// `binemit::StackMap`. - pub fn new(mapped_words: u32, bits: impl Iterator) -> StackMap { - StackMap { - bits: bits.collect(), - mapped_words, - } + pub fn new(mapped_words: u32, bits: CompoundBitSet) -> StackMap { + StackMap { bits, mapped_words } } /// Returns a specified bit. pub fn get_bit(&self, bit_index: usize) -> bool { - assert!(bit_index < 32 * self.bits.len()); - let word_index = bit_index / 32; - let word_offset = bit_index % 32; - (self.bits[word_index] & (1 << word_offset)) != 0 + self.bits.contains(bit_index) } /// Returns the number of words represented by this stack map. diff --git a/scripts/publish.rs b/scripts/publish.rs index 7927936c83..eb92c648c5 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -22,6 +22,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "cranelift-entity", "wasmtime-types", "cranelift-bforest", + "cranelift-bitset", "cranelift-codegen-shared", "cranelift-codegen-meta", "cranelift-egraph", @@ -93,6 +94,7 @@ const PUBLIC_CRATES: &[&str] = &[ // have breaking API changes in patch releases "cranelift-entity", "cranelift-bforest", + "cranelift-bitset", "cranelift-codegen-shared", "cranelift-codegen-meta", "cranelift-egraph",